diff --git a/.gitignore b/.gitignore index 434d4a5107..e455846d68 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,6 @@ dist .vscode .tags* *.sw? -config* openbazaar-go* # macOS diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index a773c94c1d..50120764fa 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -11,10 +11,82 @@ "ImportPath": "github.com/OpenBazaar/go-blockstackclient", "Rev": "d805c545cd74f4ae0cca2b64a739771a14dfbaaf" }, + { + "ImportPath": "github.com/OpenBazaar/golang-socketio", + "Rev": "9fab7b0eb211a7134cea662b8de11496a41de94e" + }, + { + "ImportPath": "github.com/OpenBazaar/golang-socketio/protocol", + "Rev": "9fab7b0eb211a7134cea662b8de11496a41de94e" + }, + { + "ImportPath": "github.com/OpenBazaar/golang-socketio/transport", + "Rev": "9fab7b0eb211a7134cea662b8de11496a41de94e" + }, { "ImportPath": "github.com/OpenBazaar/jsonpb", "Rev": "37d32ddf4eefaab6c19a01c99f4e00df9b1be48f" }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/bitcoin", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/bitcoincash", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/client", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/client/transport", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/config", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/datastore", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/keys", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/litecoin", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/litecoin/address", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/litecoin/params", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/service", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/util", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/zcash", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, + { + "ImportPath": "github.com/OpenBazaar/multiwallet/zcash/address", + "Rev": "dd770ddb47476642c3951ea26e84f2d46f42c96e" + }, { "ImportPath": "github.com/OpenBazaar/spvwallet", "Rev": "cc9f64c4f5ab77fde286913e9aa347dd2c4c5db1" @@ -160,6 +232,10 @@ "ImportPath": "github.com/btcsuite/go-socks/socks", "Rev": "4720035b7bfd2a9bb130b1c184f8bbe41b6f0d0f" }, + { + "ImportPath": "github.com/btcsuite/golangcrypto/ripemd160", + "Rev": "53f62d9b43e87a6c56975cf862af7edf33a8d0df" + }, { "ImportPath": "github.com/btcsuite/goleveldb/leveldb/errors", "Rev": "7834afc9e8cd15233b6c3d97e12674a31ca24602" @@ -274,368 +350,368 @@ }, { "ImportPath": "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/mitchellh/go-homedir", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/assets", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/blocks/blockstoreutil", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/blockservice", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/commands", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/commands/legacy", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/commands", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/commands/dag", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/commands/e", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/commands/object", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/commands/unixfs", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/coreapi", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/coreapi/interface", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/coreapi/interface/options", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/coredag", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/corehttp", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/corerepo", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/coreunix", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/core/mock", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/exchange/bitswap", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/exchange/bitswap/decision", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/exchange/bitswap/message", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/exchange/bitswap/message/pb", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/exchange/bitswap/network", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/exchange/bitswap/notifications", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/exchange/bitswap/testnet", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/exchange/reprovide", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/filestore", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/filestore/pb", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/fuse/ipns", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/fuse/mount", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/fuse/node", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/fuse/readonly", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/importer", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/importer/balanced", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/importer/helpers", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/importer/trickle", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/keystore", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/merkledag", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/merkledag/pb", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/merkledag/test", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/merkledag/utils", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/mfs", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/namesys", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/namesys/opts", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/namesys/pb", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/namesys/republisher", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/p2p", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/path", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/path/resolver", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/pin", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/pin/gc", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/pin/internal/pb", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/repo", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/repo/common", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/repo/config", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/repo/fsrepo", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/repo/fsrepo/migrations", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/repo/fsrepo/serialize", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/tar", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/thirdparty/dir", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/thirdparty/math2", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/thirdparty/verifbs", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/thirdparty/verifcid", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/unixfs", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/unixfs/archive", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/unixfs/archive/tar", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/unixfs/hamt", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/unixfs/io", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/unixfs/mod", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/ipfs/go-ipfs/unixfs/pb", - "Comment": "v0.4.11-84-ga798c8ab7", - "Rev": "a798c8ab7bac5c2173cc1b59a40d701536288c87" + "Comment": "v0.4.11-86-ge6033d5e2", + "Rev": "e6033d5e21b7569ff3696b190c0e94af5cd12742" }, { "ImportPath": "github.com/jbenet/go-context/io", @@ -651,6 +727,26 @@ "Comment": "v1.2.3", "Rev": "2e6820834a1f36c626bf19a253b7d3cc060e9b8b" }, + { + "ImportPath": "github.com/ltcsuite/ltcd/chaincfg", + "Comment": "BTCD_0_8_0_BETA-2231-g92a0feb9", + "Rev": "92a0feb9b86f5971655013115ea4f95949403592" + }, + { + "ImportPath": "github.com/ltcsuite/ltcd/chaincfg/chainhash", + "Comment": "BTCD_0_8_0_BETA-2231-g92a0feb9", + "Rev": "92a0feb9b86f5971655013115ea4f95949403592" + }, + { + "ImportPath": "github.com/ltcsuite/ltcd/wire", + "Comment": "BTCD_0_8_0_BETA-2231-g92a0feb9", + "Rev": "92a0feb9b86f5971655013115ea4f95949403592" + }, + { + "ImportPath": "github.com/ltcsuite/ltcutil/base58", + "Comment": "BTCUTIL_0_4_0-109-ga88d7df", + "Rev": "a88d7dfb1c02af5dffe005ad21f40c42d9fd5ad0" + }, { "ImportPath": "github.com/mattn/go-colorable", "Comment": "v0.0.9-14-gefa5899", @@ -671,7 +767,7 @@ }, { "ImportPath": "github.com/mutecomm/go-sqlcipher", - "Rev": "f799951b4ab269c2ce24913961af2cb5d681cc43" + "Rev": "34287a64fbe7fc03ab5a0e6cbc35618f9453b8d2" }, { "ImportPath": "github.com/natefinch/lumberjack", @@ -739,6 +835,10 @@ "ImportPath": "golang.org/x/crypto/salsa20/salsa", "Rev": "a49355c7e3f8fe157a85be2f77e6e269a0f89602" }, + { + "ImportPath": "golang.org/x/crypto/scrypt", + "Rev": "a49355c7e3f8fe157a85be2f77e6e269a0f89602" + }, { "ImportPath": "golang.org/x/crypto/ssh/terminal", "Rev": "a49355c7e3f8fe157a85be2f77e6e269a0f89602" diff --git a/cmd/start.go b/cmd/start.go index 64af4fa920..b34dfa3f89 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -20,9 +20,6 @@ import ( "github.com/OpenBazaar/bitcoind-wallet" bstk "github.com/OpenBazaar/go-blockstackclient" "github.com/OpenBazaar/openbazaar-go/api" - "github.com/OpenBazaar/openbazaar-go/bitcoin" - lis "github.com/OpenBazaar/openbazaar-go/bitcoin/listeners" - "github.com/OpenBazaar/openbazaar-go/bitcoin/resync" "github.com/OpenBazaar/openbazaar-go/core" "github.com/OpenBazaar/openbazaar-go/ipfs" obns "github.com/OpenBazaar/openbazaar-go/namesys" @@ -35,9 +32,12 @@ import ( sto "github.com/OpenBazaar/openbazaar-go/storage" "github.com/OpenBazaar/openbazaar-go/storage/dropbox" "github.com/OpenBazaar/openbazaar-go/storage/selfhosted" + "github.com/OpenBazaar/openbazaar-go/wallet" + lis "github.com/OpenBazaar/openbazaar-go/wallet/listeners" + "github.com/OpenBazaar/openbazaar-go/wallet/resync" "github.com/OpenBazaar/spvwallet" exchange "github.com/OpenBazaar/spvwallet/exchangerates" - "github.com/OpenBazaar/wallet-interface" + wi "github.com/OpenBazaar/wallet-interface" "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcutil/base58" "github.com/cpacia/BitcoinCash-Wallet" @@ -262,12 +262,12 @@ func (x *Start) Execute(args []string) error { walletCfg.Binary = x.ZCash } - ct := wallet.Bitcoin + ct := wi.Bitcoin switch walletCfg.Type { case "bitcoincash": - ct = wallet.BitcoinCash + ct = wi.BitcoinCash case "zcashd": - ct = wallet.Zcash + ct = wi.Zcash } migrations.WalletCoinType = ct @@ -538,7 +538,7 @@ func (x *Start) Execute(args []string) error { walletCfg.Type = "zcashd" walletCfg.Binary = x.ZCash } - var exchangeRates wallet.ExchangeRates + var exchangeRates wi.ExchangeRates if !x.DisableExchangeRates { exchangeRates = exchange.NewBitcoinPriceFetcher(torDialer) } @@ -558,7 +558,7 @@ func (x *Start) Execute(args []string) error { ml := logging.MultiLogger(bitcoinFileFormatter) var resyncManager *resync.ResyncManager - var cryptoWallet wallet.Wallet + var cryptoWallet wi.Wallet var walletTypeStr string switch strings.ToLower(walletCfg.Type) { case "spvwallet": @@ -857,7 +857,7 @@ func (x *Start) Execute(args []string) error { cryptoWallet.AddTransactionListener(TL.OnTransactionReceived) cryptoWallet.AddTransactionListener(WL.OnTransactionReceived) log.Infof("Starting %s wallet\n", walletTypeStr) - su := bitcoin.NewStatusUpdater(cryptoWallet, core.Node.Broadcast, nd.Context()) + su := wallet.NewStatusUpdater(cryptoWallet, core.Node.Broadcast, nd.Context()) go su.Start() go cryptoWallet.Start() if resyncManager != nil { @@ -1084,7 +1084,7 @@ func serveHTTPApi(cctx *commands.Context) (<-chan error, error) { return errc, nil } -func InitializeRepo(dataDir, password, mnemonic string, testnet bool, creationDate time.Time, coinType wallet.CoinType) (*db.SQLiteDatastore, error) { +func InitializeRepo(dataDir, password, mnemonic string, testnet bool, creationDate time.Time, coinType wi.CoinType) (*db.SQLiteDatastore, error) { // Database sqliteDB, err := db.Create(dataDir, password, testnet, coinType) if err != nil { diff --git a/mobile/node.go b/mobile/node.go index c714949dde..1ca912eea3 100644 --- a/mobile/node.go +++ b/mobile/node.go @@ -13,15 +13,14 @@ import ( ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore" - lis "github.com/OpenBazaar/openbazaar-go/bitcoin/listeners" rep "github.com/OpenBazaar/openbazaar-go/net/repointer" ret "github.com/OpenBazaar/openbazaar-go/net/retriever" "github.com/OpenBazaar/openbazaar-go/net/service" + lis "github.com/OpenBazaar/openbazaar-go/wallet/listeners" "errors" "fmt" bstk "github.com/OpenBazaar/go-blockstackclient" - "github.com/OpenBazaar/openbazaar-go/bitcoin" "github.com/OpenBazaar/openbazaar-go/core" "github.com/OpenBazaar/openbazaar-go/ipfs" obnet "github.com/OpenBazaar/openbazaar-go/net" @@ -29,6 +28,7 @@ import ( "github.com/OpenBazaar/openbazaar-go/repo/migrations" "github.com/OpenBazaar/openbazaar-go/schema" "github.com/OpenBazaar/openbazaar-go/storage/selfhosted" + "github.com/OpenBazaar/openbazaar-go/wallet" "github.com/OpenBazaar/spvwallet" "github.com/OpenBazaar/spvwallet/exchangerates" wi "github.com/OpenBazaar/wallet-interface" @@ -378,7 +378,7 @@ func (n *Node) Start() error { WL := lis.NewWalletListener(n.OpenBazaarNode.Datastore, n.OpenBazaarNode.Broadcast) n.OpenBazaarNode.Wallet.AddTransactionListener(TL.OnTransactionReceived) n.OpenBazaarNode.Wallet.AddTransactionListener(WL.OnTransactionReceived) - su := bitcoin.NewStatusUpdater(n.OpenBazaarNode.Wallet, n.OpenBazaarNode.Broadcast, n.OpenBazaarNode.IpfsNode.Context()) + su := wallet.NewStatusUpdater(n.OpenBazaarNode.Wallet, n.OpenBazaarNode.Broadcast, n.OpenBazaarNode.IpfsNode.Context()) go su.Start() go n.OpenBazaarNode.Wallet.Start() } diff --git a/repo/db/db.go b/repo/db/db.go index bae8947e05..5d9f457697 100644 --- a/repo/db/db.go +++ b/repo/db/db.go @@ -86,6 +86,15 @@ func NewSQLiteDatastore(db *sql.DB, l *sync.Mutex, coinType wallet.CoinType) *SQ } } +type DB struct { + SqlDB *sql.DB + Lock *sync.Mutex +} + +func (d *SQLiteDatastore) DB() *DB { + return &DB{d.db, d.lock} +} + func (d *SQLiteDatastore) Ping() error { return d.db.Ping() } diff --git a/repo/init.go b/repo/init.go index 7105e05930..3466ee7708 100644 --- a/repo/init.go +++ b/repo/init.go @@ -169,7 +169,7 @@ func addConfigExtensions(repoRoot string, testnet bool) error { TrustedPeer: "", } ws = schema.WalletsConfig{ - BTC: schema.CoinConfig{ + BTC: &schema.CoinConfig{ Type: "API", API: "https://btc.bloqapi.net/insight-api", APITestnet: "https://test-insight.bitpay.com/api", @@ -179,7 +179,7 @@ func addConfigExtensions(repoRoot string, testnet bool) error { MediumFeeDefault: 10, LowFeeDefault: 1, }, - BCH: schema.CoinConfig{ + BCH: &schema.CoinConfig{ Type: "API", API: "https://bch-insight.bitpay.com/api", APITestnet: "https://test-bch-insight.bitpay.com/api", @@ -188,7 +188,7 @@ func addConfigExtensions(repoRoot string, testnet bool) error { MediumFeeDefault: 5, LowFeeDefault: 1, }, - LTC: schema.CoinConfig{ + LTC: &schema.CoinConfig{ Type: "API", API: "https://insight.litecore.io/api", APITestnet: "https://testnet.litecore.io/api", @@ -197,7 +197,7 @@ func addConfigExtensions(repoRoot string, testnet bool) error { MediumFeeDefault: 10, LowFeeDefault: 5, }, - ZEC: schema.CoinConfig{ + ZEC: &schema.CoinConfig{ Type: "API", API: "https://zcashnetwork.info/api", APITestnet: "https://explorer.testnet.z.cash/api", diff --git a/schema/configuration.go b/schema/configuration.go index 7dc4d9f806..7857bf4a8d 100644 --- a/schema/configuration.go +++ b/schema/configuration.go @@ -42,10 +42,10 @@ type WalletConfig struct { } type WalletsConfig struct { - BTC CoinConfig `json:"BTC"` - BCH CoinConfig `json:"BCH"` - LTC CoinConfig `json:"LTC"` - ZEC CoinConfig `json:"ZEC"` + BTC *CoinConfig `json:"BTC"` + BCH *CoinConfig `json:"BCH"` + LTC *CoinConfig `json:"LTC"` + ZEC *CoinConfig `json:"ZEC"` } type CoinConfig struct { @@ -57,6 +57,7 @@ type CoinConfig struct { HighFeeDefault int MediumFeeDefault int LowFeeDefault int + TrustedPeer string } type DataSharing struct { @@ -64,7 +65,7 @@ type DataSharing struct { PushTo []string } -var MalformedConfigError error = errors.New("Config file is malformed") +var MalformedConfigError error = errors.New("config file is malformed") func GetAPIConfig(cfgBytes []byte) (*APIConfig, error) { var cfgIface interface{} diff --git a/test_compile.sh b/test_compile.sh index d63de3a848..892899ad1d 100755 --- a/test_compile.sh +++ b/test_compile.sh @@ -3,8 +3,8 @@ set -e pwd go test -coverprofile=api.cover.out ./api -go test -coverprofile=bitcoin.cover.out ./bitcoin -go test -coverprofile=bitcoin.listeners.cover.out ./bitcoin/listeners +go test -coverprofile=bitcoin.cover.out ./wallet +go test -coverprofile=bitcoin.listeners.cover.out ./wallet/listeners go test -coverprofile=core.cover.out ./core go test -coverprofile=ipfs.cover.out ./ipfs go test -coverprofile=mobile.cover.out ./mobile diff --git a/vendor/github.com/OpenBazaar/golang-socketio/.gitignore b/vendor/github.com/OpenBazaar/golang-socketio/.gitignore new file mode 100644 index 0000000000..eb21d98409 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/.gitignore @@ -0,0 +1,42 @@ +# Test artifacts + +repo/testdata/ + +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so +*.pyc + +# Folders +_obj +_test +__pycache__ + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +.idea/ +*.iml +.gx/ +dist + +# Development environment files +.ackrc +.tags* +*.sw? + +# macOS +.DS_Store \ No newline at end of file diff --git a/vendor/github.com/OpenBazaar/golang-socketio/LICENSE b/vendor/github.com/OpenBazaar/golang-socketio/LICENSE new file mode 100644 index 0000000000..9cecc1d466 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + {one line to give the program's name and a brief idea of what it does.} + Copyright (C) {year} {name of author} + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + {project} Copyright (C) {year} {fullname} + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/vendor/github.com/OpenBazaar/golang-socketio/LICENSE.MIT b/vendor/github.com/OpenBazaar/golang-socketio/LICENSE.MIT new file mode 100644 index 0000000000..7e17e00e68 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/LICENSE.MIT @@ -0,0 +1,22 @@ +(The MIT License) + +Copyright (c) 2016 Gennadiy Kovalev + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/OpenBazaar/golang-socketio/README.md b/vendor/github.com/OpenBazaar/golang-socketio/README.md new file mode 100644 index 0000000000..21fa4ccf80 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/README.md @@ -0,0 +1,168 @@ +golang socket.io +================ + +golang implementation of [socket.io](http://socket.io) library, client and server + +You can check working chat server, based on caller library, at http://funstream.tv + +Examples directory contains simple client and server. + +### Installation + + go get github.com/OpenBazaar/golang-socketio + +### Simple server usage + +```go + //create + server := gosocketio.NewServer(transport.GetDefaultWebsocketTransport()) + + //handle connected + server.On(gosocketio.OnConnection, func(c *gosocketio.Channel) { + log.Println("New client connected") + //join them to room + c.Join("chat") + }) + + type Message struct { + Name string `json:"name"` + Message string `json:"message"` + } + + //handle custom event + server.On("send", func(c *gosocketio.Channel, msg Message) string { + //send event to all in room + c.BroadcastTo("chat", "message", msg) + return "OK" + }) + + //setup http server + serveMux := http.NewServeMux() + serveMux.Handle("/socket.io/", server) + log.Panic(http.ListenAndServe(":80", serveMux)) +``` + +### Javascript client for caller server + +```javascript +var socket = io('ws://yourdomain.com', {transports: ['websocket']}); + + // listen for messages + socket.on('message', function(message) { + + console.log('new message'); + console.log(message); + }); + + socket.on('connect', function () { + + console.log('socket connected'); + + //send something + socket.emit('send', {name: "my name", message: "hello"}, function(result) { + + console.log('sended successfully'); + console.log(result); + }); + }); +``` + +### Server, detailed usage + +```go + //create server instance, you can setup transport parameters or get the default one + //look at websocket.go for parameters description + server := gosocketio.NewServer(transport.GetDefaultWebsocketTransport()) + + // --- caller is default handlers + + //on connection handler, occurs once for each connected client + server.On(gosocketio.OnConnection, func(c *gosocketio.Channel, args interface{}) { + //client id is unique + log.Println("New client connected, client id is ", c.Id()) + + //you can join clients to rooms + c.Join("room name") + + //of course, you can list the clients in the room, or account them + channels := c.List(data.Channel) + //or check the amount of clients in room + amount := c.Amount(data.Channel) + log.Println(amount, "clients in room") + }) + //on disconnection handler, if client hangs connection unexpectedly, it will still occurs + //you can omit function args if you do not need them + //you can return string value for ack, or return nothing for emit + server.On(gosocketio.OnDisconnection, func(c *gosocketio.Channel) { + //caller is not necessary, client will be removed from rooms + //automatically on disconnect + //but you can remove client from room whenever you need to + c.Leave("room name") + + log.Println("Disconnected") + }) + //error catching handler + server.On(gosocketio.OnError, func(c *gosocketio.Channel) { + log.Println("Error occurs") + }) + + // --- caller is custom handler + + //custom event handler + server.On("handle something", func(c *gosocketio.Channel, channel Channel) string { + log.Println("Something successfully handled") + + //you can return result of handler, in caller case + //handler will be converted from "emit" to "ack" + return "result" + }) + + //you can get client connection by it's id + channel, _ := server.GetChannel("client id here") + //and send the event to the client + type MyEventData struct { + Data: string + } + channel.Emit("my event", MyEventData{"my data"}) + + //or you can send ack to client and get result back + result, err := channel.Ack("my custom ack", MyEventData{"ack data"}, time.Second * 5) + + //you can broadcast to all clients + server.BroadcastToAll("my event", MyEventData{"broadcast"}) + + //or for clients joined to room + server.BroadcastTo("my room", "my event", MyEventData{"room broadcast"}) + + //setup http server like caller for handling connections + serveMux := http.NewServeMux() + serveMux.Handle("/socket.io/", server) + log.Panic(http.ListenAndServe(":80", serveMux)) +``` + +### Client + +```go + //connect to server, you can use your own transport settings + c, err := gosocketio.Dial( + gosocketio.GetUrl("localhost", 80, false), + transport.GetDefaultWebsocketTransport(), + ) + + //do something, handlers and functions are same as server ones + + //close connection + c.Close() +``` + +### Roadmap + +1. Tests +2. Travis CI +3. http longpoll transport +4. pure http (short-timed queries) transport +5. binary format + +### Licence + +Double licensed under GPL3 and MIT so you can use whichever you please. diff --git a/vendor/github.com/OpenBazaar/golang-socketio/ack.go b/vendor/github.com/OpenBazaar/golang-socketio/ack.go new file mode 100644 index 0000000000..e889dfcae0 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/ack.go @@ -0,0 +1,64 @@ +package gosocketio + +import ( + "errors" + "sync" +) + +var ( + ErrorWaiterNotFound = errors.New("Waiter not found") +) + +/** +Processes functions that require answers, also known as acknowledge or ack +*/ +type ackProcessor struct { + counter int + counterLock sync.Mutex + + resultWaiters map[int](chan string) + resultWaitersLock sync.RWMutex +} + +/** +get next id of ack call +*/ +func (a *ackProcessor) getNextId() int { + a.counterLock.Lock() + defer a.counterLock.Unlock() + + a.counter++ + return a.counter +} + +/** +Just before the ack function called, the waiter should be added +to wait and receive response to ack call +*/ +func (a *ackProcessor) addWaiter(id int, w chan string) { + a.resultWaitersLock.Lock() + a.resultWaiters[id] = w + a.resultWaitersLock.Unlock() +} + +/** +removes waiter that is unnecessary anymore +*/ +func (a *ackProcessor) removeWaiter(id int) { + a.resultWaitersLock.Lock() + delete(a.resultWaiters, id) + a.resultWaitersLock.Unlock() +} + +/** +check if waiter with given ack id is exists, and returns it +*/ +func (a *ackProcessor) getWaiter(id int) (chan string, error) { + a.resultWaitersLock.RLock() + defer a.resultWaitersLock.RUnlock() + + if waiter, ok := a.resultWaiters[id]; ok { + return waiter, nil + } + return nil, ErrorWaiterNotFound +} diff --git a/vendor/github.com/OpenBazaar/golang-socketio/caller.go b/vendor/github.com/OpenBazaar/golang-socketio/caller.go new file mode 100644 index 0000000000..bb423ddef9 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/caller.go @@ -0,0 +1,75 @@ +package gosocketio + +import ( + "errors" + "reflect" +) + +type caller struct { + Func reflect.Value + Args reflect.Type + ArgsPresent bool + Out bool +} + +var ( + ErrorCallerNotFunc = errors.New("f is not function") + ErrorCallerNot2Args = errors.New("f should have 1 or 2 args") + ErrorCallerMaxOneValue = errors.New("f should return not more than one value") +) + +/** +Parses function passed by using reflection, and stores its representation +for further call on message or ack +*/ +func newCaller(f interface{}) (*caller, error) { + fVal := reflect.ValueOf(f) + if fVal.Kind() != reflect.Func { + return nil, ErrorCallerNotFunc + } + + fType := fVal.Type() + if fType.NumOut() > 1 { + return nil, ErrorCallerMaxOneValue + } + + curCaller := &caller{ + Func: fVal, + Out: fType.NumOut() == 1, + } + if fType.NumIn() == 1 { + curCaller.Args = nil + curCaller.ArgsPresent = false + } else if fType.NumIn() == 2 { + curCaller.Args = fType.In(1) + curCaller.ArgsPresent = true + } else { + return nil, ErrorCallerNot2Args + } + + return curCaller, nil +} + +/** +returns function parameter as it is present in it using reflection +*/ +func (c *caller) getArgs() interface{} { + return reflect.New(c.Args).Interface() +} + +/** +calls function with given arguments from its representation using reflection +*/ +func (c *caller) callFunc(h *Channel, args interface{}) []reflect.Value { + //nil is untyped, so use the default empty value of correct type + if args == nil { + args = c.getArgs() + } + + a := []reflect.Value{reflect.ValueOf(h), reflect.ValueOf(args).Elem()} + if !c.ArgsPresent { + a = a[0:1] + } + + return c.Func.Call(a) +} diff --git a/vendor/github.com/OpenBazaar/golang-socketio/client.go b/vendor/github.com/OpenBazaar/golang-socketio/client.go new file mode 100644 index 0000000000..d20992b6a9 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/client.go @@ -0,0 +1,66 @@ +package gosocketio + +import ( + "github.com/OpenBazaar/golang-socketio/transport" + "strconv" +) + +const ( + webSocketProtocol = "ws://" + webSocketSecureProtocol = "wss://" + socketioUrl = "/socket.io/?EIO=3&transport=websocket" +) + +/** +Socket.io client representation +*/ +type Client struct { + methods + Channel +} + +/** +Get ws/wss url by host and port + */ +func GetUrl(host string, port int, secure bool) string { + var prefix string + if secure { + prefix = webSocketSecureProtocol + } else { + prefix = webSocketProtocol + } + return prefix + host + ":" + strconv.Itoa(port) + socketioUrl +} + +/** +connect to host and initialise socket.io protocol + +The correct ws protocol url example: +ws://myserver.com/socket.io/?EIO=3&transport=websocket + +You can use GetUrlByHost for generating correct url +*/ +func Dial(url string, tr transport.Transport) (*Client, error) { + c := &Client{} + c.initChannel() + c.initMethods() + + var err error + c.conn, err = tr.Connect(url) + if err != nil { + return nil, err + } + + go inLoop(&c.Channel, &c.methods) + go outLoop(&c.Channel, &c.methods) + go pinger(&c.Channel) + + return c, nil +} + +/** +Close client connection +*/ +func (c *Client) Close() { + closeChannel(&c.Channel, &c.methods) +} diff --git a/vendor/github.com/OpenBazaar/golang-socketio/handler.go b/vendor/github.com/OpenBazaar/golang-socketio/handler.go new file mode 100644 index 0000000000..856d008ca1 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/handler.go @@ -0,0 +1,141 @@ +package gosocketio + +import ( + "encoding/json" + "github.com/OpenBazaar/golang-socketio/protocol" + "sync" + "reflect" +) + +const ( + OnConnection = "connection" + OnDisconnection = "disconnection" + OnError = "error" +) + +/** +System handler function for internal event processing +*/ +type systemHandler func(c *Channel) + +/** +Contains maps of message processing functions +*/ +type methods struct { + messageHandlers map[string]*caller + messageHandlersLock sync.RWMutex + + onConnection systemHandler + onDisconnection systemHandler +} + +/** +create messageHandlers map +*/ +func (m *methods) initMethods() { + m.messageHandlers = make(map[string]*caller) +} + +/** +Add message processing function, and bind it to given method +*/ +func (m *methods) On(method string, f interface{}) error { + c, err := newCaller(f) + if err != nil { + return err + } + + m.messageHandlersLock.Lock() + defer m.messageHandlersLock.Unlock() + m.messageHandlers[method] = c + + return nil +} + +/** +Find message processing function associated with given method +*/ +func (m *methods) findMethod(method string) (*caller, bool) { + m.messageHandlersLock.RLock() + defer m.messageHandlersLock.RUnlock() + + f, ok := m.messageHandlers[method] + return f, ok +} + +func (m *methods) callLoopEvent(c *Channel, event string) { + if m.onConnection != nil && event == OnConnection { + m.onConnection(c) + } + if m.onDisconnection != nil && event == OnDisconnection { + m.onDisconnection(c) + } + + f, ok := m.findMethod(event) + if !ok { + return + } + + f.callFunc(c, &struct{}{}) +} + +/** +Check incoming message +On ack_resp - look for waiter +On ack_req - look for processing function and send ack_resp +On emit - look for processing function +*/ +func (m *methods) processIncomingMessage(c *Channel, msg *protocol.Message) { + switch msg.Type { + case protocol.MessageTypeEmit: + f, ok := m.findMethod(msg.Method) + if !ok { + return + } + + if !f.ArgsPresent { + f.callFunc(c, &struct{}{}) + return + } + + data := f.getArgs() + err := json.Unmarshal([]byte(msg.Args), &data) + if err != nil { + return + } + + f.callFunc(c, data) + + case protocol.MessageTypeAckRequest: + f, ok := m.findMethod(msg.Method) + if !ok || !f.Out { + return + } + + var result []reflect.Value + if f.ArgsPresent { + //data type should be defined for unmarshall + data := f.getArgs() + err := json.Unmarshal([]byte(msg.Args), &data) + if err != nil { + return + } + + result = f.callFunc(c, data) + } else { + result = f.callFunc(c, &struct{}{}) + } + + ack := &protocol.Message{ + Type: protocol.MessageTypeAckResponse, + AckId: msg.AckId, + } + send(ack, c, protocol.ToArgArray(result[0].Interface())) + + case protocol.MessageTypeAckResponse: + waiter, err := c.ack.getWaiter(msg.AckId) + if err == nil { + waiter <- msg.Args + } + } +} diff --git a/vendor/github.com/OpenBazaar/golang-socketio/loop.go b/vendor/github.com/OpenBazaar/golang-socketio/loop.go new file mode 100644 index 0000000000..95de3715f6 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/loop.go @@ -0,0 +1,196 @@ +package gosocketio + +import ( + "encoding/json" + "errors" + "github.com/OpenBazaar/golang-socketio/protocol" + "github.com/OpenBazaar/golang-socketio/transport" + "net/http" + "sync" + "time" +) + +const ( + queueBufferSize = 500 +) + +var ( + ErrorWrongHeader = errors.New("Wrong header") +) + +/** +engine.io header to send or receive +*/ +type Header struct { + Sid string `json:"sid"` + Upgrades []string `json:"upgrades"` + PingInterval int `json:"pingInterval"` + PingTimeout int `json:"pingTimeout"` +} + +/** +socket.io connection handler + +use IsAlive to check that handler is still working +use Dial to connect to websocket +use In and Out channels for message exchange +Close message means channel is closed +ping is automatic +*/ +type Channel struct { + conn transport.Connection + + out chan string + header Header + + alive bool + aliveLock sync.Mutex + + ack ackProcessor + + server *Server + ip string + requestHeader http.Header +} + +/** +create channel, map, and set active +*/ +func (c *Channel) initChannel() { + //TODO: queueBufferSize from constant to server or client variable + c.out = make(chan string, queueBufferSize) + c.ack.resultWaiters = make(map[int](chan string)) + c.alive = true +} + +/** +Get id of current socket connection +*/ +func (c *Channel) Id() string { + return c.header.Sid +} + +/** +Checks that Channel is still alive +*/ +func (c *Channel) IsAlive() bool { + c.aliveLock.Lock() + defer c.aliveLock.Unlock() + + return c.alive +} + +/** +Close channel +*/ +func closeChannel(c *Channel, m *methods, args ...interface{}) error { + c.aliveLock.Lock() + defer c.aliveLock.Unlock() + + if !c.alive { + //already closed + return nil + } + + c.conn.Close() + c.alive = false + + //clean outloop + for len(c.out) > 0 { + <-c.out + } + c.out <- protocol.CloseMessage + + m.callLoopEvent(c, OnDisconnection) + + overfloodedLock.Lock() + delete(overflooded, c) + overfloodedLock.Unlock() + + return nil +} + +//incoming messages loop, puts incoming messages to In channel +func inLoop(c *Channel, m *methods) error { + for { + pkg, err := c.conn.GetMessage() + if err != nil { + return closeChannel(c, m, err) + } + msg, err := protocol.Decode(pkg) + if err != nil { + closeChannel(c, m, protocol.ErrorWrongPacket) + return err + } + + switch msg.Type { + case protocol.MessageTypeOpen: + if err := json.Unmarshal([]byte(msg.Source[1:]), &c.header); err != nil { + closeChannel(c, m, ErrorWrongHeader) + } + m.callLoopEvent(c, OnConnection) + case protocol.MessageTypePing: + c.out <- protocol.PongMessage + case protocol.MessageTypePong: + default: + go m.processIncomingMessage(c, msg) + } + } + return nil +} + +var overflooded map[*Channel]struct{} = make(map[*Channel]struct{}) +var overfloodedLock sync.Mutex + +func AmountOfOverflooded() int64 { + overfloodedLock.Lock() + defer overfloodedLock.Unlock() + + return int64(len(overflooded)) +} + +/** +outgoing messages loop, sends messages from channel to socket +*/ +func outLoop(c *Channel, m *methods) error { + for { + outBufferLen := len(c.out) + if outBufferLen >= queueBufferSize-1 { + return closeChannel(c, m, ErrorSocketOverflood) + } else if outBufferLen > int(queueBufferSize/2) { + overfloodedLock.Lock() + overflooded[c] = struct{}{} + overfloodedLock.Unlock() + } else { + overfloodedLock.Lock() + delete(overflooded, c) + overfloodedLock.Unlock() + } + + msg := <-c.out + if msg == protocol.CloseMessage { + return nil + } + + err := c.conn.WriteMessage(msg) + if err != nil { + return closeChannel(c, m, err) + } + } + return nil +} + +/** +Pinger sends ping messages for keeping connection alive +*/ +func pinger(c *Channel) { + for { + interval, _ := c.conn.PingParams() + time.Sleep(interval) + if !c.IsAlive() { + return + } + + c.out <- protocol.PingMessage + } +} diff --git a/vendor/github.com/OpenBazaar/golang-socketio/protocol/message.go b/vendor/github.com/OpenBazaar/golang-socketio/protocol/message.go new file mode 100644 index 0000000000..3ba9d813cf --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/protocol/message.go @@ -0,0 +1,45 @@ +package protocol + +const ( + /** + Message with connection options + */ + MessageTypeOpen = iota + /** + Close connection and destroy all handle routines + */ + MessageTypeClose = iota + /** + Ping request message + */ + MessageTypePing = iota + /** + Pong response message + */ + MessageTypePong = iota + /** + Empty message + */ + MessageTypeEmpty = iota + /** + Emit request, no response + */ + MessageTypeEmit = iota + /** + Emit request, wait for response (ack) + */ + MessageTypeAckRequest = iota + /** + ack response + */ + MessageTypeAckResponse = iota +) + +type Message struct { + Type int + AckId int + Method string + Args string + Source string +} + diff --git a/vendor/github.com/OpenBazaar/golang-socketio/protocol/socketio.go b/vendor/github.com/OpenBazaar/golang-socketio/protocol/socketio.go new file mode 100644 index 0000000000..f15a1550bd --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/protocol/socketio.go @@ -0,0 +1,219 @@ +package protocol + +import ( + "encoding/json" + "errors" + "strconv" + "strings" +) + +const ( + open = "0" + msg = "4" + emptyMessage = "40" + commonMessage = "42" + ackMessage = "43" + + CloseMessage = "1" + PingMessage = "2" + PongMessage = "3" +) + +var ( + ErrorWrongMessageType = errors.New("Wrong message type") + ErrorWrongPacket = errors.New("Wrong packet") +) + +func typeToText(msgType int) (string, error) { + switch msgType { + case MessageTypeOpen: + return open, nil + case MessageTypeClose: + return CloseMessage, nil + case MessageTypePing: + return PingMessage, nil + case MessageTypePong: + return PongMessage, nil + case MessageTypeEmpty: + return emptyMessage, nil + case MessageTypeEmit, MessageTypeAckRequest: + return commonMessage, nil + case MessageTypeAckResponse: + return ackMessage, nil + } + return "", ErrorWrongMessageType +} + +func Encode(msg *Message) (string, error) { + result, err := typeToText(msg.Type) + if err != nil { + return "", err + } + + if msg.Type == MessageTypeEmpty || msg.Type == MessageTypePing || + msg.Type == MessageTypePong { + return result, nil + } + + if msg.Type == MessageTypeAckRequest || msg.Type == MessageTypeAckResponse { + result += strconv.Itoa(msg.AckId) + } + + if msg.Type == MessageTypeOpen || msg.Type == MessageTypeClose { + return result + msg.Args, nil + } + + if msg.Type == MessageTypeAckResponse { + return result + "[" + msg.Args + "]", nil + } + + jsonMethod, err := json.Marshal(&msg.Method) + if err != nil { + return "", err + } + + return result + "[" + string(jsonMethod) + "," + msg.Args + "]", nil +} + +func MustEncode(msg *Message) string { + result, err := Encode(msg) + if err != nil { + panic(err) + } + + return result +} + +func getMessageType(data string) (int, error) { + if len(data) == 0 { + return 0, ErrorWrongMessageType + } + switch data[0:1] { + case open: + return MessageTypeOpen, nil + case CloseMessage: + return MessageTypeClose, nil + case PingMessage: + return MessageTypePing, nil + case PongMessage: + return MessageTypePong, nil + case msg: + if len(data) == 1 { + return 0, ErrorWrongMessageType + } + switch data[0:2] { + case emptyMessage: + return MessageTypeEmpty, nil + case commonMessage: + return MessageTypeAckRequest, nil + case ackMessage: + return MessageTypeAckResponse, nil + } + } + return 0, ErrorWrongMessageType +} + +/** +Get ack id of current packet, if present +*/ +func getAck(text string) (ackId int, restText string, err error) { + if len(text) < 4 { + return 0, "", ErrorWrongPacket + } + text = text[2:] + + pos := strings.IndexByte(text, '[') + if pos == -1 { + return 0, "", ErrorWrongPacket + } + + ack, err := strconv.Atoi(text[0:pos]) + if err != nil { + return 0, "", err + } + + return ack, text[pos:], nil +} + +/** +Get message method of current packet, if present +*/ +func getMethod(text string) (method, restText string, err error) { + var start, end, rest, countQuote int + + for i, c := range text { + if c == '"' { + switch countQuote { + case 0: + start = i + 1 + case 1: + end = i + rest = i + 1 + default: + return "", "", ErrorWrongPacket + } + countQuote++ + } + if c == ',' { + if countQuote < 2 { + continue + } + rest = i + 1 + break + } + } + + if (end < start) || (rest >= len(text)) { + return "", "", ErrorWrongPacket + } + + return text[start:end], text[rest : len(text)-1], nil +} + +func Decode(data string) (*Message, error) { + var err error + msg := &Message{} + msg.Source = data + + msg.Type, err = getMessageType(data) + if err != nil { + return nil, err + } + + if msg.Type == MessageTypeOpen { + msg.Args = data[1:] + return msg, nil + } + + if msg.Type == MessageTypeClose || msg.Type == MessageTypePing || + msg.Type == MessageTypePong || msg.Type == MessageTypeEmpty { + return msg, nil + } + + ack, rest, err := getAck(data) + msg.AckId = ack + if msg.Type == MessageTypeAckResponse { + if err != nil { + return nil, err + } + msg.Args = rest[1 : len(rest)-1] + return msg, nil + } + + if err != nil { + msg.Type = MessageTypeEmit + rest = data[2:] + } + + msg.Method, msg.Args, err = getMethod(rest) + if err != nil { + return nil, err + } + + return msg, nil +} + +func ToArgArray(arg interface{})(i []interface{}) { + i = append(i, arg) + return i +} diff --git a/vendor/github.com/OpenBazaar/golang-socketio/send.go b/vendor/github.com/OpenBazaar/golang-socketio/send.go new file mode 100644 index 0000000000..c5571bff25 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/send.go @@ -0,0 +1,93 @@ +package gosocketio + +import ( + "encoding/json" + "errors" + "github.com/OpenBazaar/golang-socketio/protocol" + "log" + "time" +) + +var ( + ErrorSendTimeout = errors.New("Timeout") + ErrorSocketOverflood = errors.New("Socket overflood") +) + +/** +Send message packet to socket +*/ +func send(msg *protocol.Message, c *Channel, args []interface{}) error { + //preventing json/encoding "index out of range" panic + defer func() { + if r := recover(); r != nil { + log.Println("socket.io send panic: ", r) + } + }() + + if len(args) > 0 { + for i, arg := range args { + json, err := json.Marshal(&arg) + if err != nil { + return err + } + + msg.Args += string(json) + if i < len(args)-1 { + msg.Args += ", " + } + } + } + + command, err := protocol.Encode(msg) + if err != nil { + return err + } + + if len(c.out) == queueBufferSize { + return ErrorSocketOverflood + } + + c.out <- command + + return nil +} + +/** +Create packet based on given data and send it +*/ +func (c *Channel) Emit(method string, args []interface{}) error { + msg := &protocol.Message{ + Type: protocol.MessageTypeEmit, + Method: method, + } + + return send(msg, c, args) +} + +/** +Create ack packet based on given data and send it and receive response +*/ +func (c *Channel) Ack(method string, args interface{}, timeout time.Duration) (string, error) { + msg := &protocol.Message{ + Type: protocol.MessageTypeAckRequest, + AckId: c.ack.getNextId(), + Method: method, + } + + waiter := make(chan string) + c.ack.addWaiter(msg.AckId, waiter) + + err := send(msg, c, protocol.ToArgArray(args)) + if err != nil { + c.ack.removeWaiter(msg.AckId) + } + + select { + case result := <-waiter: + return result, nil + case <-time.After(timeout): + c.ack.removeWaiter(msg.AckId) + return "", ErrorSendTimeout + } +} + diff --git a/vendor/github.com/OpenBazaar/golang-socketio/server.go b/vendor/github.com/OpenBazaar/golang-socketio/server.go new file mode 100644 index 0000000000..f242729382 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/server.go @@ -0,0 +1,381 @@ +package gosocketio + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "github.com/OpenBazaar/golang-socketio/protocol" + "github.com/OpenBazaar/golang-socketio/transport" + "math/rand" + "net/http" + "sync" + "time" +) + +const ( + HeaderForward = "X-Forwarded-For" +) + +var ( + ErrorServerNotSet = errors.New("Server not set") + ErrorConnectionNotFound = errors.New("Connection not found") +) + +/** +socket.io server instance +*/ +type Server struct { + methods + http.Handler + + channels map[string]map[*Channel]struct{} + rooms map[*Channel]map[string]struct{} + channelsLock sync.RWMutex + + sids map[string]*Channel + sidsLock sync.RWMutex + + tr transport.Transport +} + +/** +Close current channel + */ +func (c *Channel) Close() { + if c.server != nil { + closeChannel(c, &c.server.methods) + } +} + +/** +Get ip of socket client +*/ +func (c *Channel) Ip() string { + forward := c.RequestHeader().Get(HeaderForward) + if forward != "" { + return forward + } + return c.ip +} + +/** +Get request header of this connection +*/ +func (c *Channel) RequestHeader() http.Header { + return c.requestHeader +} + +/** +Get channel by it's sid +*/ +func (s *Server) GetChannel(sid string) (*Channel, error) { + s.sidsLock.RLock() + defer s.sidsLock.RUnlock() + + c, ok := s.sids[sid] + if !ok { + return nil, ErrorConnectionNotFound + } + + return c, nil +} + +/** +Join this channel to given room +*/ +func (c *Channel) Join(room string) error { + if c.server == nil { + return ErrorServerNotSet + } + + c.server.channelsLock.Lock() + defer c.server.channelsLock.Unlock() + + cn := c.server.channels + if _, ok := cn[room]; !ok { + cn[room] = make(map[*Channel]struct{}) + } + + byRoom := c.server.rooms + if _, ok := byRoom[c]; !ok { + byRoom[c] = make(map[string]struct{}) + } + + cn[room][c] = struct{}{} + byRoom[c][room] = struct{}{} + + return nil +} + +/** +Remove this channel from given room +*/ +func (c *Channel) Leave(room string) error { + if c.server == nil { + return ErrorServerNotSet + } + + c.server.channelsLock.Lock() + defer c.server.channelsLock.Unlock() + + cn := c.server.channels + if _, ok := cn[room]; ok { + delete(cn[room], c) + if len(cn[room]) == 0 { + delete(cn, room) + } + } + + byRoom := c.server.rooms + if _, ok := byRoom[c]; ok { + delete(byRoom[c], room) + } + + return nil +} + +/** +Get amount of channels, joined to given room, using channel +*/ +func (c *Channel) Amount(room string) int { + if c.server == nil { + return 0 + } + + return c.server.Amount(room) +} + +/** +Get amount of channels, joined to given room, using server +*/ +func (s *Server) Amount(room string) int { + s.channelsLock.RLock() + defer s.channelsLock.RUnlock() + + roomChannels, _ := s.channels[room] + return len(roomChannels) +} + +/** +Get list of channels, joined to given room, using channel +*/ +func (c *Channel) List(room string) []*Channel { + if c.server == nil { + return []*Channel{} + } + + return c.server.List(room) +} + +/** +Get list of channels, joined to given room, using server +*/ +func (s *Server) List(room string) []*Channel { + s.channelsLock.RLock() + defer s.channelsLock.RUnlock() + + roomChannels, ok := s.channels[room] + if !ok { + return []*Channel{} + } + + i := 0 + roomChannelsCopy := make([]*Channel, len(roomChannels)) + for channel := range roomChannels { + roomChannelsCopy[i] = channel + i++ + } + + return roomChannelsCopy + +} + +func (c *Channel) BroadcastTo(room, method string, args interface{}) { + if c.server == nil { + return + } + c.server.BroadcastTo(room, method, args) +} + +/** +Broadcast message to all room channels +*/ +func (s *Server) BroadcastTo(room, method string, args interface{}) { + s.channelsLock.RLock() + defer s.channelsLock.RUnlock() + + roomChannels, ok := s.channels[room] + if !ok { + return + } + + for cn := range roomChannels { + if cn.IsAlive() { + go cn.Emit(method, protocol.ToArgArray(args)) + } + } +} + +/** +Broadcast to all clients +*/ +func (s *Server) BroadcastToAll(method string, args interface{}) { + s.sidsLock.RLock() + defer s.sidsLock.RUnlock() + + for _, cn := range s.sids { + if cn.IsAlive() { + go cn.Emit(method, protocol.ToArgArray(args)) + } + } +} + +/** +Generate new id for socket.io connection +*/ +func generateNewId(custom string) string { + hash := fmt.Sprintf("%s %s %n %n", custom, time.Now(), rand.Uint32(), rand.Uint32()) + buf := bytes.NewBuffer(nil) + sum := md5.Sum([]byte(hash)) + encoder := base64.NewEncoder(base64.URLEncoding, buf) + encoder.Write(sum[:]) + encoder.Close() + return buf.String()[:20] +} + +/** +On connection system handler, store sid +*/ +func onConnectStore(c *Channel) { + c.server.sidsLock.Lock() + defer c.server.sidsLock.Unlock() + + c.server.sids[c.Id()] = c +} + +/** +On disconnection system handler, clean joins and sid +*/ +func onDisconnectCleanup(c *Channel) { + c.server.channelsLock.Lock() + defer c.server.channelsLock.Unlock() + + cn := c.server.channels + byRoom, ok := c.server.rooms[c] + if ok { + for room := range byRoom { + if curRoom, ok := cn[room]; ok { + delete(curRoom, c) + if len(curRoom) == 0 { + delete(cn, room) + } + } + } + + delete(c.server.rooms, c) + } + + c.server.sidsLock.Lock() + defer c.server.sidsLock.Unlock() + + delete(c.server.sids, c.Id()) +} + +func (s *Server) SendOpenSequence(c *Channel) { + jsonHdr, err := json.Marshal(&c.header) + if err != nil { + panic(err) + } + + c.out <- protocol.MustEncode( + &protocol.Message{ + Type: protocol.MessageTypeOpen, + Args: string(jsonHdr), + }, + ) + + c.out <- protocol.MustEncode(&protocol.Message{Type: protocol.MessageTypeEmpty}) +} + +/** +Setup event loop for given connection +*/ +func (s *Server) SetupEventLoop(conn transport.Connection, remoteAddr string, + requestHeader http.Header) { + + interval, timeout := conn.PingParams() + hdr := Header{ + Sid: generateNewId(remoteAddr), + Upgrades: []string{}, + PingInterval: int(interval / time.Millisecond), + PingTimeout: int(timeout / time.Millisecond), + } + + c := &Channel{} + c.conn = conn + c.ip = remoteAddr + c.requestHeader = requestHeader + c.initChannel() + + c.server = s + c.header = hdr + + s.SendOpenSequence(c) + + go inLoop(c, &s.methods) + go outLoop(c, &s.methods) + + s.callLoopEvent(c, OnConnection) +} + +/** +implements ServeHTTP function from http.Handler +*/ +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + conn, err := s.tr.HandleConnection(w, r) + if err != nil { + return + } + + s.SetupEventLoop(conn, r.RemoteAddr, r.Header) + s.tr.Serve(w, r) +} + +/** +Get amount of current connected sids +*/ +func (s *Server) AmountOfSids() int64 { + s.sidsLock.RLock() + defer s.sidsLock.RUnlock() + + return int64(len(s.sids)) +} + +/** +Get amount of rooms with at least one channel(or sid) joined +*/ +func (s *Server) AmountOfRooms() int64 { + s.channelsLock.RLock() + defer s.channelsLock.RUnlock() + + return int64(len(s.channels)) +} + +/** +Create new socket.io server +*/ +func NewServer(tr transport.Transport) *Server { + s := Server{} + s.initMethods() + s.tr = tr + s.channels = make(map[string]map[*Channel]struct{}) + s.rooms = make(map[*Channel]map[string]struct{}) + s.sids = make(map[string]*Channel) + s.onConnection = onConnectStore + s.onDisconnection = onDisconnectCleanup + + return &s +} diff --git a/vendor/github.com/OpenBazaar/golang-socketio/transport/transport.go b/vendor/github.com/OpenBazaar/golang-socketio/transport/transport.go new file mode 100644 index 0000000000..d64dbf0ca3 --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/transport/transport.go @@ -0,0 +1,51 @@ +package transport + +import ( + "net/http" + "time" +) + +/** +End-point connection for given transport +*/ +type Connection interface { + /** + Receive one more message, block until received + */ + GetMessage() (message string, err error) + + /** + Send given message, block until sent + */ + WriteMessage(message string) error + + /** + Close current connection + */ + Close() + + /** + Get ping time interval and ping request timeout + */ + PingParams() (interval, timeout time.Duration) +} + +/** +Connection factory for given transport +*/ +type Transport interface { + /** + Get client connection + */ + Connect(url string) (conn Connection, err error) + + /** + Handle one server connection + */ + HandleConnection(w http.ResponseWriter, r *http.Request) (conn Connection, err error) + + /** + Serve HTTP request after making connection and events setup + */ + Serve(w http.ResponseWriter, r *http.Request) +} diff --git a/vendor/github.com/OpenBazaar/golang-socketio/transport/websocket.go b/vendor/github.com/OpenBazaar/golang-socketio/transport/websocket.go new file mode 100644 index 0000000000..58da6739ca --- /dev/null +++ b/vendor/github.com/OpenBazaar/golang-socketio/transport/websocket.go @@ -0,0 +1,138 @@ +package transport + +import ( + "errors" + "github.com/gorilla/websocket" + "io/ioutil" + "net/http" + "time" +) + +const ( + upgradeFailed = "Upgrade failed: " + + WsDefaultPingInterval = 30 * time.Second + WsDefaultPingTimeout = 60 * time.Second + WsDefaultReceiveTimeout = 60 * time.Second + WsDefaultSendTimeout = 60 * time.Second + WsDefaultBufferSize = 1024 * 32 +) + +var ( + ErrorBinaryMessage = errors.New("Binary messages are not supported") + ErrorBadBuffer = errors.New("Buffer error") + ErrorPacketWrong = errors.New("Wrong packet type error") + ErrorMethodNotAllowed = errors.New("Method not allowed") + ErrorHttpUpgradeFailed = errors.New("Http upgrade failed") +) + +type WebsocketConnection struct { + socket *websocket.Conn + transport *WebsocketTransport +} + +func (wsc *WebsocketConnection) GetMessage() (message string, err error) { + wsc.socket.SetReadDeadline(time.Now().Add(wsc.transport.ReceiveTimeout)) + msgType, reader, err := wsc.socket.NextReader() + if err != nil { + return "", err + } + + //support only text messages exchange + if msgType != websocket.TextMessage { + return "", ErrorBinaryMessage + } + + data, err := ioutil.ReadAll(reader) + if err != nil { + return "", ErrorBadBuffer + } + text := string(data) + + //empty messages are not allowed + if len(text) == 0 { + return "", ErrorPacketWrong + } + + return text, nil +} + +func (wsc *WebsocketConnection) WriteMessage(message string) error { + wsc.socket.SetWriteDeadline(time.Now().Add(wsc.transport.SendTimeout)) + writer, err := wsc.socket.NextWriter(websocket.TextMessage) + if err != nil { + return err + } + + if _, err := writer.Write([]byte(message)); err != nil { + return err + } + if err := writer.Close(); err != nil { + return err + } + return nil +} + +func (wsc *WebsocketConnection) Close() { + wsc.socket.Close() +} + +func (wsc *WebsocketConnection) PingParams() (interval, timeout time.Duration) { + return wsc.transport.PingInterval, wsc.transport.PingTimeout +} + +type WebsocketTransport struct { + PingInterval time.Duration + PingTimeout time.Duration + ReceiveTimeout time.Duration + SendTimeout time.Duration + + BufferSize int + + RequestHeader http.Header +} + +func (wst *WebsocketTransport) Connect(url string) (conn Connection, err error) { + dialer := websocket.Dialer{} + socket, _, err := dialer.Dial(url, wst.RequestHeader) + if err != nil { + return nil, err + } + + return &WebsocketConnection{socket, wst}, nil +} + +func (wst *WebsocketTransport) HandleConnection( + w http.ResponseWriter, r *http.Request) (conn Connection, err error) { + + if r.Method != "GET" { + http.Error(w, upgradeFailed+ErrorMethodNotAllowed.Error(), 503) + return nil, ErrorMethodNotAllowed + } + + socket, err := websocket.Upgrade(w, r, nil, wst.BufferSize, wst.BufferSize) + if err != nil { + http.Error(w, upgradeFailed+err.Error(), 503) + return nil, ErrorHttpUpgradeFailed + } + + return &WebsocketConnection{socket, wst}, nil +} + +/** +Websocket connection do not require any additional processing +*/ +func (wst *WebsocketTransport) Serve(w http.ResponseWriter, r *http.Request) {} + +/** +Returns websocket connection with default params +*/ +func GetDefaultWebsocketTransport() *WebsocketTransport { + return &WebsocketTransport{ + PingInterval: WsDefaultPingInterval, + PingTimeout: WsDefaultPingTimeout, + ReceiveTimeout: WsDefaultReceiveTimeout, + SendTimeout: WsDefaultSendTimeout, + BufferSize: WsDefaultBufferSize, + } +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/.gitignore b/vendor/github.com/OpenBazaar/multiwallet/.gitignore new file mode 100644 index 0000000000..131a3162af --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/.gitignore @@ -0,0 +1,37 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test +__pycache__ + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +.idea/ +*.iml +.gx/ +dist + +# Development environment files +.ackrc +.tags* +*.sw? + +# macOS +.DS_Store diff --git a/vendor/github.com/OpenBazaar/multiwallet/.travis.yml b/vendor/github.com/OpenBazaar/multiwallet/.travis.yml new file mode 100644 index 0000000000..d50cc26b01 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/.travis.yml @@ -0,0 +1,16 @@ +language: go +go: + - 1.9 +sudo: required +services: + - docker +env: + - "PATH=/home/travis/gopath/bin:$PATH" +before_install: + - go get github.com/tcnksm/ghr + - go get github.com/axw/gocov/gocov + - go get github.com/mattn/goveralls +script: + - diff -u <(echo -n) <(gofmt -d -s $(find . -type f -name '*.go' -not -path "./cmd/*" -not -path "./cli/*" -not -path "./api/*")) + - cd $TRAVIS_BUILD_DIR && chmod a+x test_compile.sh && ./test_compile.sh + - goveralls -coverprofile=coverage.out -service travis-ci \ No newline at end of file diff --git a/vendor/github.com/OpenBazaar/multiwallet/LICENSE b/vendor/github.com/OpenBazaar/multiwallet/LICENSE new file mode 100644 index 0000000000..54c962577b --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 OpenBazaar + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/OpenBazaar/multiwallet/README.md b/vendor/github.com/OpenBazaar/multiwallet/README.md new file mode 100644 index 0000000000..683c93e176 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/README.md @@ -0,0 +1,6 @@ +[![Build Status](https://travis-ci.org/OpenBazaar/multiwallet.svg?branch=master)](https://travis-ci.org/OpenBazaar/multiwallet) +[![Coverage Status](https://coveralls.io/repos/github/OpenBazaar/multiwallet/badge.svg?branch=master)](https://coveralls.io/github/OpenBazaar/multiwallet?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/OpenBazaar/multiwallet)](https://goreportcard.com/report/github.com/OpenBazaar/multiwallet) + +# multiwallet +Insight API based multi-cryptocurrency wallet diff --git a/vendor/github.com/OpenBazaar/multiwallet/bitcoin/sign.go b/vendor/github.com/OpenBazaar/multiwallet/bitcoin/sign.go new file mode 100644 index 0000000000..0e6dbb273e --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/bitcoin/sign.go @@ -0,0 +1,610 @@ +package bitcoin + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/OpenBazaar/spvwallet" + wi "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/blockchain" + "github.com/btcsuite/btcd/btcec" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + btc "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcutil/coinset" + hd "github.com/btcsuite/btcutil/hdkeychain" + "github.com/btcsuite/btcutil/txsort" + "github.com/btcsuite/btcwallet/wallet/txauthor" + "github.com/btcsuite/btcwallet/wallet/txrules" + + "github.com/OpenBazaar/multiwallet/util" +) + +func (w *BitcoinWallet) buildTx(amount int64, addr btc.Address, feeLevel wi.FeeLevel, optionalOutput *wire.TxOut) (*wire.MsgTx, error) { + // Check for dust + script, _ := txscript.PayToAddrScript(addr) + if txrules.IsDustAmount(btc.Amount(amount), len(script), txrules.DefaultRelayFeePerKb) { + return nil, wi.ErrorDustAmount + } + + var additionalPrevScripts map[wire.OutPoint][]byte + var additionalKeysByAddress map[string]*btc.WIF + + // Create input source + height, _ := w.ws.ChainTip() + utxos, err := w.db.Utxos().GetAll() + if err != nil { + return nil, err + } + coinMap := util.GatherCoins(height, utxos, w.ScriptToAddress, w.km.GetKeyForScript) + + coins := make([]coinset.Coin, 0, len(coinMap)) + for k := range coinMap { + coins = append(coins, k) + } + inputSource := func(target btc.Amount) (total btc.Amount, inputs []*wire.TxIn, inputValues []btc.Amount, scripts [][]byte, err error) { + coinSelector := coinset.MaxValueAgeCoinSelector{MaxInputs: 10000, MinChangeAmount: btc.Amount(0)} + coins, err := coinSelector.CoinSelect(target, coins) + if err != nil { + return total, inputs, inputValues, scripts, wi.ErrorInsuffientFunds + } + additionalPrevScripts = make(map[wire.OutPoint][]byte) + additionalKeysByAddress = make(map[string]*btc.WIF) + for _, c := range coins.Coins() { + total += c.Value() + outpoint := wire.NewOutPoint(c.Hash(), c.Index()) + in := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + in.Sequence = 0 // Opt-in RBF so we can bump fees + inputs = append(inputs, in) + additionalPrevScripts[*outpoint] = c.PkScript() + key := coinMap[c] + addr, err := key.Address(w.params) + if err != nil { + continue + } + privKey, err := key.ECPrivKey() + if err != nil { + continue + } + wif, _ := btc.NewWIF(privKey, w.params, true) + additionalKeysByAddress[addr.EncodeAddress()] = wif + } + return total, inputs, inputValues, scripts, nil + } + + // Get the fee per kilobyte + feePerKB := int64(w.GetFeePerByte(feeLevel)) * 1000 + + // outputs + out := wire.NewTxOut(amount, script) + + // Create change source + changeSource := func() ([]byte, error) { + addr := w.CurrentAddress(wi.INTERNAL) + script, err := txscript.PayToAddrScript(addr) + if err != nil { + return []byte{}, err + } + return script, nil + } + + outputs := []*wire.TxOut{out} + if optionalOutput != nil { + outputs = append(outputs, optionalOutput) + } + authoredTx, err := newUnsignedTransaction(outputs, btc.Amount(feePerKB), inputSource, changeSource) + if err != nil { + return nil, err + } + + // BIP 69 sorting + txsort.InPlaceSort(authoredTx.Tx) + + // Sign tx + getKey := txscript.KeyClosure(func(addr btc.Address) (*btcec.PrivateKey, bool, error) { + addrStr := addr.EncodeAddress() + wif := additionalKeysByAddress[addrStr] + return wif.PrivKey, wif.CompressPubKey, nil + }) + getScript := txscript.ScriptClosure(func( + addr btc.Address) ([]byte, error) { + return []byte{}, nil + }) + for i, txIn := range authoredTx.Tx.TxIn { + prevOutScript := additionalPrevScripts[txIn.PreviousOutPoint] + script, err := txscript.SignTxOutput(w.params, + authoredTx.Tx, i, prevOutScript, txscript.SigHashAll, getKey, + getScript, txIn.SignatureScript) + if err != nil { + return nil, errors.New("Failed to sign transaction") + } + txIn.SignatureScript = script + } + return authoredTx.Tx, nil +} + +func newUnsignedTransaction(outputs []*wire.TxOut, feePerKb btc.Amount, fetchInputs txauthor.InputSource, fetchChange txauthor.ChangeSource) (*txauthor.AuthoredTx, error) { + + var targetAmount btc.Amount + for _, txOut := range outputs { + targetAmount += btc.Amount(txOut.Value) + } + + estimatedSize := EstimateSerializeSize(1, outputs, true, P2PKH) + targetFee := txrules.FeeForSerializeSize(feePerKb, estimatedSize) + + for { + inputAmount, inputs, _, scripts, err := fetchInputs(targetAmount + targetFee) + if err != nil { + return nil, err + } + if inputAmount < targetAmount+targetFee { + return nil, errors.New("insufficient funds available to construct transaction") + } + + maxSignedSize := EstimateSerializeSize(len(inputs), outputs, true, P2PKH) + maxRequiredFee := txrules.FeeForSerializeSize(feePerKb, maxSignedSize) + remainingAmount := inputAmount - targetAmount + if remainingAmount < maxRequiredFee { + targetFee = maxRequiredFee + continue + } + + unsignedTransaction := &wire.MsgTx{ + Version: wire.TxVersion, + TxIn: inputs, + TxOut: outputs, + LockTime: 0, + } + changeIndex := -1 + changeAmount := inputAmount - targetAmount - maxRequiredFee + if changeAmount != 0 && !txrules.IsDustAmount(changeAmount, + P2PKHOutputSize, txrules.DefaultRelayFeePerKb) { + changeScript, err := fetchChange() + if err != nil { + return nil, err + } + if len(changeScript) > P2PKHPkScriptSize { + return nil, errors.New("fee estimation requires change " + + "scripts no larger than P2PKH output scripts") + } + change := wire.NewTxOut(int64(changeAmount), changeScript) + l := len(outputs) + unsignedTransaction.TxOut = append(outputs[:l:l], change) + changeIndex = l + } + + return &txauthor.AuthoredTx{ + Tx: unsignedTransaction, + PrevScripts: scripts, + TotalInput: inputAmount, + ChangeIndex: changeIndex, + }, nil + } +} + +func (w *BitcoinWallet) bumpFee(txid chainhash.Hash) (*chainhash.Hash, error) { + txn, err := w.db.Txns().Get(txid) + if err != nil { + return nil, err + } + if txn.Height > 0 { + return nil, spvwallet.BumpFeeAlreadyConfirmedError + } + if txn.Height < 0 { + return nil, spvwallet.BumpFeeTransactionDeadError + } + // Check utxos for CPFP + utxos, _ := w.db.Utxos().GetAll() + for _, u := range utxos { + if u.Op.Hash.IsEqual(&txid) && u.AtHeight == 0 { + addr, err := w.ScriptToAddress(u.ScriptPubkey) + if err != nil { + return nil, err + } + key, err := w.km.GetKeyForScript(addr.ScriptAddress()) + if err != nil { + return nil, err + } + h, err := hex.DecodeString(u.Op.Hash.String()) + if err != nil { + return nil, err + } + in := wi.TransactionInput{ + LinkedAddress: addr, + OutpointIndex: u.Op.Index, + OutpointHash: h, + Value: int64(u.Value), + } + transactionID, err := w.sweepAddress([]wi.TransactionInput{in}, nil, key, nil, wi.FEE_BUMP) + if err != nil { + return nil, err + } + return transactionID, nil + } + } + return nil, spvwallet.BumpFeeNotFoundError +} + +func (w *BitcoinWallet) sweepAddress(ins []wi.TransactionInput, address *btc.Address, key *hd.ExtendedKey, redeemScript *[]byte, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + var internalAddr btc.Address + if address != nil { + internalAddr = *address + } else { + internalAddr = w.CurrentAddress(wi.INTERNAL) + } + script, err := txscript.PayToAddrScript(internalAddr) + if err != nil { + return nil, err + } + + var val int64 + var inputs []*wire.TxIn + additionalPrevScripts := make(map[wire.OutPoint][]byte) + for _, in := range ins { + val += in.Value + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return nil, err + } + script, err := txscript.PayToAddrScript(in.LinkedAddress) + if err != nil { + return nil, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + inputs = append(inputs, input) + additionalPrevScripts[*outpoint] = script + } + out := wire.NewTxOut(val, script) + + txType := P2PKH + if redeemScript != nil { + txType = P2SH_1of2_Multisig + _, err := spvwallet.LockTimeFromRedeemScript(*redeemScript) + if err == nil { + txType = P2SH_Multisig_Timelock_1Sig + } + } + estimatedSize := EstimateSerializeSize(len(ins), []*wire.TxOut{out}, false, txType) + + // Calculate the fee + feePerByte := int(w.GetFeePerByte(feeLevel)) + fee := estimatedSize * feePerByte + + outVal := val - int64(fee) + if outVal < 0 { + outVal = 0 + } + out.Value = outVal + + tx := &wire.MsgTx{ + Version: wire.TxVersion, + TxIn: inputs, + TxOut: []*wire.TxOut{out}, + LockTime: 0, + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + // Sign tx + privKey, err := key.ECPrivKey() + if err != nil { + return nil, err + } + pk := privKey.PubKey().SerializeCompressed() + addressPub, err := btc.NewAddressPubKey(pk, w.params) + + getKey := txscript.KeyClosure(func(addr btc.Address) (*btcec.PrivateKey, bool, error) { + if addressPub.EncodeAddress() == addr.EncodeAddress() { + wif, err := btc.NewWIF(privKey, w.params, true) + if err != nil { + return nil, false, err + } + return wif.PrivKey, wif.CompressPubKey, nil + } + return nil, false, errors.New("Not found") + }) + getScript := txscript.ScriptClosure(func(addr btc.Address) ([]byte, error) { + if redeemScript == nil { + return []byte{}, nil + } + return *redeemScript, nil + }) + + // Check if time locked + var timeLocked bool + if redeemScript != nil { + rs := *redeemScript + if rs[0] == txscript.OP_IF { + timeLocked = true + tx.Version = 2 + for _, txIn := range tx.TxIn { + locktime, err := spvwallet.LockTimeFromRedeemScript(*redeemScript) + if err != nil { + return nil, err + } + txIn.Sequence = locktime + } + } + } + + hashes := txscript.NewTxSigHashes(tx) + for i, txIn := range tx.TxIn { + if redeemScript == nil { + prevOutScript := additionalPrevScripts[txIn.PreviousOutPoint] + script, err := txscript.SignTxOutput(w.params, + tx, i, prevOutScript, txscript.SigHashAll, getKey, + getScript, txIn.SignatureScript) + if err != nil { + return nil, errors.New("Failed to sign transaction") + } + txIn.SignatureScript = script + } else { + sig, err := txscript.RawTxInWitnessSignature(tx, hashes, i, ins[i].Value, *redeemScript, txscript.SigHashAll, privKey) + if err != nil { + return nil, err + } + var witness wire.TxWitness + if timeLocked { + witness = wire.TxWitness{sig, []byte{}} + } else { + witness = wire.TxWitness{[]byte{}, sig} + } + witness = append(witness, *redeemScript) + txIn.Witness = witness + } + } + + // broadcast + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + _, err = w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + txid := tx.TxHash() + return &txid, nil +} + +func (w *BitcoinWallet) createMultisigSignature(ins []wi.TransactionInput, outs []wi.TransactionOutput, key *hd.ExtendedKey, redeemScript []byte, feePerByte uint64) ([]wi.Signature, error) { + var sigs []wi.Signature + tx := wire.NewMsgTx(1) + for _, in := range ins { + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return sigs, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + tx.TxIn = append(tx.TxIn, input) + } + for _, out := range outs { + scriptPubKey, err := txscript.PayToAddrScript(out.Address) + if err != nil { + return sigs, err + } + output := wire.NewTxOut(out.Value, scriptPubKey) + tx.TxOut = append(tx.TxOut, output) + } + + // Subtract fee + txType := P2SH_2of3_Multisig + _, err := spvwallet.LockTimeFromRedeemScript(redeemScript) + if err == nil { + txType = P2SH_Multisig_Timelock_2Sigs + } + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, txType) + fee := estimatedSize * int(feePerByte) + if len(tx.TxOut) > 0 { + feePerOutput := fee / len(tx.TxOut) + for _, output := range tx.TxOut { + output.Value -= int64(feePerOutput) + } + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + signingKey, err := key.ECPrivKey() + if err != nil { + return sigs, err + } + + hashes := txscript.NewTxSigHashes(tx) + for i := range tx.TxIn { + sig, err := txscript.RawTxInWitnessSignature(tx, hashes, i, ins[i].Value, redeemScript, txscript.SigHashAll, signingKey) + if err != nil { + continue + } + bs := wi.Signature{InputIndex: uint32(i), Signature: sig} + sigs = append(sigs, bs) + } + return sigs, nil +} + +func (w *BitcoinWallet) multisign(ins []wi.TransactionInput, outs []wi.TransactionOutput, sigs1 []wi.Signature, sigs2 []wi.Signature, redeemScript []byte, feePerByte uint64, broadcast bool) ([]byte, error) { + tx := wire.NewMsgTx(1) + for _, in := range ins { + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return nil, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + tx.TxIn = append(tx.TxIn, input) + } + for _, out := range outs { + scriptPubKey, err := txscript.PayToAddrScript(out.Address) + if err != nil { + return nil, err + } + output := wire.NewTxOut(out.Value, scriptPubKey) + tx.TxOut = append(tx.TxOut, output) + } + + // Subtract fee + txType := P2SH_2of3_Multisig + _, err := spvwallet.LockTimeFromRedeemScript(redeemScript) + if err == nil { + txType = P2SH_Multisig_Timelock_2Sigs + } + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, txType) + fee := estimatedSize * int(feePerByte) + if len(tx.TxOut) > 0 { + feePerOutput := fee / len(tx.TxOut) + for _, output := range tx.TxOut { + output.Value -= int64(feePerOutput) + } + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + // Check if time locked + var timeLocked bool + if redeemScript[0] == txscript.OP_IF { + timeLocked = true + } + + for i, input := range tx.TxIn { + var sig1 []byte + var sig2 []byte + for _, sig := range sigs1 { + if int(sig.InputIndex) == i { + sig1 = sig.Signature + break + } + } + for _, sig := range sigs2 { + if int(sig.InputIndex) == i { + sig2 = sig.Signature + break + } + } + + witness := wire.TxWitness{[]byte{}, sig1, sig2} + + if timeLocked { + witness = append(witness, []byte{0x01}) + } + witness = append(witness, redeemScript) + input.Witness = witness + } + // broadcast + if broadcast { + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + _, err = w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + } + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + return buf.Bytes(), nil +} + +func (w *BitcoinWallet) generateMultisigScript(keys []hd.ExtendedKey, threshold int, timeout time.Duration, timeoutKey *hd.ExtendedKey) (addr btc.Address, redeemScript []byte, err error) { + if uint32(timeout.Hours()) > 0 && timeoutKey == nil { + return nil, nil, errors.New("Timeout key must be non nil when using an escrow timeout") + } + + if len(keys) < threshold { + return nil, nil, fmt.Errorf("unable to generate multisig script with "+ + "%d required signatures when there are only %d public "+ + "keys available", threshold, len(keys)) + } + + var ecKeys []*btcec.PublicKey + for _, key := range keys { + ecKey, err := key.ECPubKey() + if err != nil { + return nil, nil, err + } + ecKeys = append(ecKeys, ecKey) + } + + builder := txscript.NewScriptBuilder() + if uint32(timeout.Hours()) == 0 { + + builder.AddInt64(int64(threshold)) + for _, key := range ecKeys { + builder.AddData(key.SerializeCompressed()) + } + builder.AddInt64(int64(len(ecKeys))) + builder.AddOp(txscript.OP_CHECKMULTISIG) + + } else { + ecKey, err := timeoutKey.ECPubKey() + if err != nil { + return nil, nil, err + } + sequenceLock := blockchain.LockTimeToSequence(false, uint32(timeout.Hours()*6)) + builder.AddOp(txscript.OP_IF) + builder.AddInt64(int64(threshold)) + for _, key := range ecKeys { + builder.AddData(key.SerializeCompressed()) + } + builder.AddInt64(int64(len(ecKeys))) + builder.AddOp(txscript.OP_CHECKMULTISIG) + builder.AddOp(txscript.OP_ELSE). + AddInt64(int64(sequenceLock)). + AddOp(txscript.OP_CHECKSEQUENCEVERIFY). + AddOp(txscript.OP_DROP). + AddData(ecKey.SerializeCompressed()). + AddOp(txscript.OP_CHECKSIG). + AddOp(txscript.OP_ENDIF) + } + redeemScript, err = builder.Script() + if err != nil { + return nil, nil, err + } + + witnessProgram := sha256.Sum256(redeemScript) + + addr, err = btc.NewAddressWitnessScriptHash(witnessProgram[:], w.params) + if err != nil { + return nil, nil, err + } + return addr, redeemScript, nil +} + +func (w *BitcoinWallet) estimateSpendFee(amount int64, feeLevel wi.FeeLevel) (uint64, error) { + // Since this is an estimate we can use a dummy output address. Let's use a long one so we don't under estimate. + addr, err := btc.DecodeAddress("bc1qxtq7ha2l5qg70atpwp3fus84fx3w0v2w4r2my7gt89ll3w0vnlgspu349h", w.params) + if err != nil { + return 0, err + } + tx, err := w.buildTx(amount, addr, feeLevel, nil) + if err != nil { + return 0, err + } + var outval int64 + for _, output := range tx.TxOut { + outval += output.Value + } + var inval int64 + utxos, err := w.db.Utxos().GetAll() + if err != nil { + return 0, err + } + for _, input := range tx.TxIn { + for _, utxo := range utxos { + if utxo.Op.Hash.IsEqual(&input.PreviousOutPoint.Hash) && utxo.Op.Index == input.PreviousOutPoint.Index { + inval += utxo.Value + break + } + } + } + if inval < outval { + return 0, errors.New("Error building transaction: inputs less than outputs") + } + return uint64(inval - outval), err +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/bitcoin/txsizes.go b/vendor/github.com/OpenBazaar/multiwallet/bitcoin/txsizes.go new file mode 100644 index 0000000000..8d53b33892 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/bitcoin/txsizes.go @@ -0,0 +1,249 @@ +package bitcoin + +// Copyright (c) 2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* Copied here from a btcd internal package*/ + +import ( + "github.com/btcsuite/btcd/wire" +) + +// Worst case script and input/output size estimates. +const ( + // RedeemP2PKHSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2PKH output. + // It is calculated as: + // + // - OP_DATA_73 + // - 72 bytes DER signature + 1 byte sighash + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + RedeemP2PKHSigScriptSize = 1 + 73 + 1 + 33 + + // RedeemP2SHMultisigSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a 2 of 3 P2SH multisig output with compressed keys. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_PUSHDATA + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + RedeemP2SH2of3MultisigSigScriptSize = 1 + 1 + 72 + 1 + 72 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + + // RedeemP2SH1of2MultisigSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a 1 of 2 P2SH multisig output with compressed keys. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_PUSHDATA + // - OP_1 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP2 + // - OP_CHECKMULTISIG + RedeemP2SH1of2MultisigSigScriptSize = 1 + 1 + 72 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 1 + + // RedeemP2SHMultisigTimelock1SigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2SH timelocked multisig using the timeout. + // It is calculated as: + // + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_0 + // - OP_PUSHDATA + // - OP_IF + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + // - OP_ELSE + // - OP_PUSHDATA + // - 2 byte block height + // - OP_CHECKSEQUENCEVERIFY + // - OP_DROP + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_CHECKSIG + // - OP_ENDIF + RedeemP2SHMultisigTimelock1SigScriptSize = 1 + 72 + 1 + 1 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + 1 + 1 + 2 + 1 + 1 + 1 + 33 + 1 + 1 + + // RedeemP2SHMultisigTimelock2SigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2SH timelocked multisig without using the timeout. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_1 + // - OP_PUSHDATA + // - OP_IF + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + // - OP_ELSE + // - OP_PUSHDATA + // - 2 byte block height + // - OP_CHECKSEQUENCEVERIFY + // - OP_DROP + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_CHECKSIG + // - OP_ENDIF + RedeemP2SHMultisigTimelock2SigScriptSize = 1 + 1 + 72 + +1 + 72 + 1 + 1 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + 1 + 1 + 2 + 1 + 1 + 1 + 33 + 1 + 1 + + // P2PKHPkScriptSize is the size of a transaction output script that + // pays to a compressed pubkey hash. It is calculated as: + // + // - OP_DUP + // - OP_HASH160 + // - OP_DATA_20 + // - 20 bytes pubkey hash + // - OP_EQUALVERIFY + // - OP_CHECKSIG + P2PKHPkScriptSize = 1 + 1 + 1 + 20 + 1 + 1 + + // RedeemP2PKHInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2PKH output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - signature script + // - 4 bytes sequence + RedeemP2PKHInputSize = 32 + 4 + 1 + RedeemP2PKHSigScriptSize + 4 + + // RedeemP2SH2of3MultisigInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH 2 of 3 multisig output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SH2of3MultisigInputSize = 32 + 4 + 1 + 4 + (RedeemP2SH2of3MultisigSigScriptSize / 4) + + // RedeemP2SH1of2MultisigInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH 2 of 3 multisig output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SH1of2MultisigInputSize = 32 + 4 + 1 + 4 + (RedeemP2SH1of2MultisigSigScriptSize / 4) + + // RedeemP2SHMultisigTimelock1InputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed p2sh timelocked multig output with using the timeout. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SHMultisigTimelock1InputSize = 32 + 4 + 1 + 4 + (RedeemP2SHMultisigTimelock1SigScriptSize / 4) + + // RedeemP2SHMultisigTimelock2InputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH timelocked multisig output without using the timeout. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SHMultisigTimelock2InputSize = 32 + 4 + 1 + 4 + (RedeemP2SHMultisigTimelock2SigScriptSize / 4) + + // P2PKHOutputSize is the serialize size of a transaction output with a + // P2PKH output script. It is calculated as: + // + // - 8 bytes output value + // - 1 byte compact int encoding value 25 + // - 25 bytes P2PKH output script + P2PKHOutputSize = 8 + 1 + P2PKHPkScriptSize +) + +type InputType int + +const ( + P2PKH InputType = iota + P2SH_1of2_Multisig + P2SH_2of3_Multisig + P2SH_Multisig_Timelock_1Sig + P2SH_Multisig_Timelock_2Sigs +) + +// EstimateSerializeSize returns a worst case serialize size estimate for a +// signed transaction that spends inputCount number of compressed P2PKH outputs +// and contains each transaction output from txOuts. The estimated size is +// incremented for an additional P2PKH change output if addChangeOutput is true. +func EstimateSerializeSize(inputCount int, txOuts []*wire.TxOut, addChangeOutput bool, inputType InputType) int { + changeSize := 0 + outputCount := len(txOuts) + if addChangeOutput { + changeSize = P2PKHOutputSize + outputCount++ + } + + var redeemScriptSize int + switch inputType { + case P2PKH: + redeemScriptSize = RedeemP2PKHInputSize + case P2SH_1of2_Multisig: + redeemScriptSize = RedeemP2SH1of2MultisigInputSize + case P2SH_2of3_Multisig: + redeemScriptSize = RedeemP2SH2of3MultisigInputSize + case P2SH_Multisig_Timelock_1Sig: + redeemScriptSize = RedeemP2SHMultisigTimelock1InputSize + case P2SH_Multisig_Timelock_2Sigs: + redeemScriptSize = RedeemP2SHMultisigTimelock2InputSize + } + + // 10 additional bytes are for version, locktime, and segwit flags + return 10 + wire.VarIntSerializeSize(uint64(inputCount)) + + wire.VarIntSerializeSize(uint64(outputCount)) + + inputCount*redeemScriptSize + + SumOutputSerializeSizes(txOuts) + + changeSize +} + +// SumOutputSerializeSizes sums up the serialized size of the supplied outputs. +func SumOutputSerializeSizes(outputs []*wire.TxOut) (serializeSize int) { + for _, txOut := range outputs { + serializeSize += txOut.SerializeSize() + } + return serializeSize +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/bitcoin/wallet.go b/vendor/github.com/OpenBazaar/multiwallet/bitcoin/wallet.go new file mode 100644 index 0000000000..8482e053c3 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/bitcoin/wallet.go @@ -0,0 +1,287 @@ +package bitcoin + +import ( + "bytes" + "errors" + "fmt" + "io" + "time" + + "github.com/OpenBazaar/spvwallet" + wi "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + btc "github.com/btcsuite/btcutil" + hd "github.com/btcsuite/btcutil/hdkeychain" + "github.com/btcsuite/btcwallet/wallet/txrules" + "github.com/tyler-smith/go-bip39" + "golang.org/x/net/proxy" + + "github.com/OpenBazaar/multiwallet/client" + "github.com/OpenBazaar/multiwallet/config" + "github.com/OpenBazaar/multiwallet/keys" + "github.com/OpenBazaar/multiwallet/service" + "github.com/OpenBazaar/multiwallet/util" +) + +type BitcoinWallet struct { + db wi.Datastore + km *keys.KeyManager + params *chaincfg.Params + client client.APIClient + ws *service.WalletService + fp *spvwallet.FeeProvider + + mPrivKey *hd.ExtendedKey + mPubKey *hd.ExtendedKey +} + +func NewBitcoinWallet(cfg config.CoinConfig, mnemonic string, params *chaincfg.Params, proxy proxy.Dialer) (*BitcoinWallet, error) { + seed := bip39.NewSeed(mnemonic, "") + + mPrivKey, err := hd.NewMaster(seed, params) + if err != nil { + return nil, err + } + mPubKey, err := mPrivKey.Neuter() + if err != nil { + return nil, err + } + km, err := keys.NewKeyManager(cfg.DB.Keys(), params, mPrivKey, wi.Bitcoin, keyToAddress) + if err != nil { + return nil, err + } + + c, err := client.NewInsightClient(cfg.ClientAPI.String(), proxy) + if err != nil { + return nil, err + } + + wm := service.NewWalletService(cfg.DB, km, c, params, wi.Bitcoin) + + fp := spvwallet.NewFeeProvider(cfg.MaxFee, cfg.HighFee, cfg.MediumFee, cfg.LowFee, cfg.FeeAPI.String(), proxy) + + return &BitcoinWallet{cfg.DB, km, params, c, wm, fp, mPrivKey, mPubKey}, nil +} + +func keyToAddress(key *hd.ExtendedKey, params *chaincfg.Params) (btc.Address, error) { + return key.Address(params) +} + +func (w *BitcoinWallet) Start() { + w.ws.Start() +} + +func (w *BitcoinWallet) Params() *chaincfg.Params { + return w.params +} + +func (w *BitcoinWallet) CurrencyCode() string { + if w.params.Name == chaincfg.MainNetParams.Name { + return "btc" + } else { + return "tbtc" + } +} + +func (w *BitcoinWallet) IsDust(amount int64) bool { + return txrules.IsDustAmount(btc.Amount(amount), 25, txrules.DefaultRelayFeePerKb) +} + +func (w *BitcoinWallet) MasterPrivateKey() *hd.ExtendedKey { + return w.mPrivKey +} + +func (w *BitcoinWallet) MasterPublicKey() *hd.ExtendedKey { + return w.mPubKey +} + +func (w *BitcoinWallet) ChildKey(keyBytes []byte, chaincode []byte, isPrivateKey bool) (*hd.ExtendedKey, error) { + parentFP := []byte{0x00, 0x00, 0x00, 0x00} + var id []byte + if isPrivateKey { + id = w.params.HDPrivateKeyID[:] + } else { + id = w.params.HDPublicKeyID[:] + } + hdKey := hd.NewExtendedKey( + id, + keyBytes, + chaincode, + parentFP, + 0, + 0, + isPrivateKey) + return hdKey.Child(0) +} + +func (w *BitcoinWallet) CurrentAddress(purpose wi.KeyPurpose) btc.Address { + key, _ := w.km.GetCurrentKey(purpose) + addr, _ := key.Address(w.params) + return btc.Address(addr) +} + +func (w *BitcoinWallet) NewAddress(purpose wi.KeyPurpose) btc.Address { + i, _ := w.db.Keys().GetUnused(purpose) + key, _ := w.km.GenerateChildKey(purpose, uint32(i[1])) + addr, _ := key.Address(w.params) + w.db.Keys().MarkKeyAsUsed(addr.ScriptAddress()) + return btc.Address(addr) +} + +func (w *BitcoinWallet) DecodeAddress(addr string) (btc.Address, error) { + return btc.DecodeAddress(addr, w.params) +} + +func (w *BitcoinWallet) ScriptToAddress(script []byte) (btc.Address, error) { + _, addrs, _, err := txscript.ExtractPkScriptAddrs(script, w.params) + if err != nil { + return nil, err + } + if len(addrs) == 0 { + return nil, errors.New("unknown script") + } + return addrs[0], nil +} + +func (w *BitcoinWallet) AddressToScript(addr btc.Address) ([]byte, error) { + return txscript.PayToAddrScript(addr) +} + +func (w *BitcoinWallet) HasKey(addr btc.Address) bool { + _, err := w.km.GetKeyForScript(addr.ScriptAddress()) + if err != nil { + return false + } + return true +} + +func (w *BitcoinWallet) Balance() (confirmed, unconfirmed int64) { + utxos, _ := w.db.Utxos().GetAll() + txns, _ := w.db.Txns().GetAll(false) + return util.CalcBalance(utxos, txns) +} + +func (w *BitcoinWallet) Transactions() ([]wi.Txn, error) { + return w.db.Txns().GetAll(false) +} + +func (w *BitcoinWallet) GetTransaction(txid chainhash.Hash) (wi.Txn, error) { + txn, err := w.db.Txns().Get(txid) + return txn, err +} + +func (w *BitcoinWallet) ChainTip() (uint32, chainhash.Hash) { + return w.ws.ChainTip() +} + +func (w *BitcoinWallet) GetFeePerByte(feeLevel wi.FeeLevel) uint64 { + return w.fp.GetFeePerByte(feeLevel) +} + +func (w *BitcoinWallet) Spend(amount int64, addr btc.Address, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + tx, err := w.buildTx(amount, addr, feeLevel, nil) + if err != nil { + return nil, err + } + // Broadcast + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + + _, err = w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + + ch := tx.TxHash() + return &ch, nil +} + +func (w *BitcoinWallet) BumpFee(txid chainhash.Hash) (*chainhash.Hash, error) { + return w.bumpFee(txid) +} + +func (w *BitcoinWallet) EstimateFee(ins []wi.TransactionInput, outs []wi.TransactionOutput, feePerByte uint64) uint64 { + tx := new(wire.MsgTx) + for _, out := range outs { + scriptPubKey, _ := txscript.PayToAddrScript(out.Address) + output := wire.NewTxOut(out.Value, scriptPubKey) + tx.TxOut = append(tx.TxOut, output) + } + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, P2PKH) + fee := estimatedSize * int(feePerByte) + return uint64(fee) +} + +func (w *BitcoinWallet) EstimateSpendFee(amount int64, feeLevel wi.FeeLevel) (uint64, error) { + return w.estimateSpendFee(amount, feeLevel) +} + +func (w *BitcoinWallet) SweepAddress(ins []wi.TransactionInput, address *btc.Address, key *hd.ExtendedKey, redeemScript *[]byte, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + return w.sweepAddress(ins, address, key, redeemScript, feeLevel) +} + +func (w *BitcoinWallet) CreateMultisigSignature(ins []wi.TransactionInput, outs []wi.TransactionOutput, key *hd.ExtendedKey, redeemScript []byte, feePerByte uint64) ([]wi.Signature, error) { + return w.createMultisigSignature(ins, outs, key, redeemScript, feePerByte) +} + +func (w *BitcoinWallet) Multisign(ins []wi.TransactionInput, outs []wi.TransactionOutput, sigs1 []wi.Signature, sigs2 []wi.Signature, redeemScript []byte, feePerByte uint64, broadcast bool) ([]byte, error) { + return w.multisign(ins, outs, sigs1, sigs2, redeemScript, feePerByte, broadcast) +} + +func (w *BitcoinWallet) GenerateMultisigScript(keys []hd.ExtendedKey, threshold int, timeout time.Duration, timeoutKey *hd.ExtendedKey) (addr btc.Address, redeemScript []byte, err error) { + return w.generateMultisigScript(keys, threshold, timeout, timeoutKey) +} + +func (w *BitcoinWallet) AddWatchedAddress(addr btc.Address) error { + script, err := w.AddressToScript(addr) + if err != nil { + return err + } + err = w.db.WatchedScripts().Put(script) + if err != nil { + return err + } + w.client.ListenAddress(addr) + return nil +} + +func (w *BitcoinWallet) AddTransactionListener(callback func(wi.TransactionCallback)) { + w.ws.AddTransactionListener(callback) +} + +func (w *BitcoinWallet) ReSyncBlockchain(fromTime time.Time) { + go w.ws.UpdateState() +} + +func (w *BitcoinWallet) GetConfirmations(txid chainhash.Hash) (uint32, uint32, error) { + txn, err := w.db.Txns().Get(txid) + if err != nil { + return 0, 0, err + } + if txn.Height == 0 { + return 0, 0, nil + } + chainTip, _ := w.ChainTip() + return chainTip - uint32(txn.Height) + 1, uint32(txn.Height), nil +} + +func (w *BitcoinWallet) Close() { + w.ws.Stop() + w.client.Close() +} + +func (w *BitcoinWallet) DumpTables(wr io.Writer) { + fmt.Fprintln(wr, "Transactions-----") + txns, _ := w.db.Txns().GetAll(true) + for _, tx := range txns { + fmt.Fprintf(wr, "Hash: %s, Height: %d, Value: %d, WatchOnly: %t\n", tx.Txid, int(tx.Height), int(tx.Value), tx.WatchOnly) + } + fmt.Fprintln(wr, "\nUtxos-----") + utxos, _ := w.db.Utxos().GetAll() + for _, u := range utxos { + fmt.Fprintf(wr, "Hash: %s, Index: %d, Height: %d, Value: %d, WatchOnly: %t\n", u.Op.Hash.String(), int(u.Op.Index), int(u.AtHeight), int(u.Value), u.WatchOnly) + } +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/bitcoincash/sign.go b/vendor/github.com/OpenBazaar/multiwallet/bitcoincash/sign.go new file mode 100644 index 0000000000..224df7a2c0 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/bitcoincash/sign.go @@ -0,0 +1,616 @@ +package bitcoincash + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/OpenBazaar/spvwallet" + wi "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/blockchain" + "github.com/btcsuite/btcd/btcec" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + btc "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcutil/coinset" + hd "github.com/btcsuite/btcutil/hdkeychain" + "github.com/btcsuite/btcutil/txsort" + "github.com/btcsuite/btcwallet/wallet/txauthor" + "github.com/btcsuite/btcwallet/wallet/txrules" + "github.com/cpacia/bchutil" + + "github.com/OpenBazaar/multiwallet/util" +) + +func (w *BitcoinCashWallet) buildTx(amount int64, addr btc.Address, feeLevel wi.FeeLevel, optionalOutput *wire.TxOut) (*wire.MsgTx, error) { + // Check for dust + script, _ := bchutil.PayToAddrScript(addr) + if txrules.IsDustAmount(btc.Amount(amount), len(script), txrules.DefaultRelayFeePerKb) { + return nil, wi.ErrorDustAmount + } + + var additionalPrevScripts map[wire.OutPoint][]byte + var additionalKeysByAddress map[string]*btc.WIF + var inVals map[wire.OutPoint]int64 + + // Create input source + height, _ := w.ws.ChainTip() + utxos, err := w.db.Utxos().GetAll() + if err != nil { + return nil, err + } + coinMap := util.GatherCoins(height, utxos, w.ScriptToAddress, w.km.GetKeyForScript) + + coins := make([]coinset.Coin, 0, len(coinMap)) + for k := range coinMap { + coins = append(coins, k) + } + inputSource := func(target btc.Amount) (total btc.Amount, inputs []*wire.TxIn, inputValues []btcutil.Amount, scripts [][]byte, err error) { + coinSelector := coinset.MaxValueAgeCoinSelector{MaxInputs: 10000, MinChangeAmount: btc.Amount(0)} + coins, err := coinSelector.CoinSelect(target, coins) + if err != nil { + return total, inputs, inputValues, scripts, wi.ErrorInsuffientFunds + } + additionalPrevScripts = make(map[wire.OutPoint][]byte) + additionalKeysByAddress = make(map[string]*btc.WIF) + inVals = make(map[wire.OutPoint]int64) + for _, c := range coins.Coins() { + total += c.Value() + outpoint := wire.NewOutPoint(c.Hash(), c.Index()) + in := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + in.Sequence = 0 // Opt-in RBF so we can bump fees + inputs = append(inputs, in) + additionalPrevScripts[*outpoint] = c.PkScript() + key := coinMap[c] + addr, err := key.Address(w.params) + if err != nil { + continue + } + privKey, err := key.ECPrivKey() + if err != nil { + continue + } + wif, _ := btc.NewWIF(privKey, w.params, true) + additionalKeysByAddress[addr.EncodeAddress()] = wif + val := c.Value() + sat := val.ToUnit(btc.AmountSatoshi) + inVals[*outpoint] = int64(sat) + } + return total, inputs, inputValues, scripts, nil + } + + // Get the fee per kilobyte + feePerKB := int64(w.GetFeePerByte(feeLevel)) * 1000 + + // outputs + out := wire.NewTxOut(amount, script) + + // Create change source + changeSource := func() ([]byte, error) { + addr := w.CurrentAddress(wi.INTERNAL) + script, err := bchutil.PayToAddrScript(addr) + if err != nil { + return []byte{}, err + } + return script, nil + } + + outputs := []*wire.TxOut{out} + if optionalOutput != nil { + outputs = append(outputs, optionalOutput) + } + authoredTx, err := newUnsignedTransaction(outputs, btc.Amount(feePerKB), inputSource, changeSource) + if err != nil { + return nil, err + } + + // BIP 69 sorting + txsort.InPlaceSort(authoredTx.Tx) + + // Sign tx + getKey := txscript.KeyClosure(func(addr btc.Address) (*btcec.PrivateKey, bool, error) { + addrStr := addr.EncodeAddress() + wif := additionalKeysByAddress[addrStr] + return wif.PrivKey, wif.CompressPubKey, nil + }) + getScript := txscript.ScriptClosure(func( + addr btc.Address) ([]byte, error) { + return []byte{}, nil + }) + for i, txIn := range authoredTx.Tx.TxIn { + prevOutScript := additionalPrevScripts[txIn.PreviousOutPoint] + script, err := bchutil.SignTxOutput(w.params, + authoredTx.Tx, i, prevOutScript, txscript.SigHashAll, getKey, + getScript, txIn.SignatureScript, inVals[txIn.PreviousOutPoint]) + if err != nil { + return nil, errors.New("Failed to sign transaction") + } + txIn.SignatureScript = script + } + return authoredTx.Tx, nil +} + +func newUnsignedTransaction(outputs []*wire.TxOut, feePerKb btc.Amount, fetchInputs txauthor.InputSource, fetchChange txauthor.ChangeSource) (*txauthor.AuthoredTx, error) { + + var targetAmount btc.Amount + for _, txOut := range outputs { + targetAmount += btc.Amount(txOut.Value) + } + + estimatedSize := EstimateSerializeSize(1, outputs, true, P2PKH) + targetFee := txrules.FeeForSerializeSize(feePerKb, estimatedSize) + + for { + inputAmount, inputs, _, scripts, err := fetchInputs(targetAmount + targetFee) + if err != nil { + return nil, err + } + if inputAmount < targetAmount+targetFee { + return nil, errors.New("insufficient funds available to construct transaction") + } + + maxSignedSize := EstimateSerializeSize(len(inputs), outputs, true, P2PKH) + maxRequiredFee := txrules.FeeForSerializeSize(feePerKb, maxSignedSize) + remainingAmount := inputAmount - targetAmount + if remainingAmount < maxRequiredFee { + targetFee = maxRequiredFee + continue + } + + unsignedTransaction := &wire.MsgTx{ + Version: wire.TxVersion, + TxIn: inputs, + TxOut: outputs, + LockTime: 0, + } + changeIndex := -1 + changeAmount := inputAmount - targetAmount - maxRequiredFee + if changeAmount != 0 && !txrules.IsDustAmount(changeAmount, + P2PKHOutputSize, txrules.DefaultRelayFeePerKb) { + changeScript, err := fetchChange() + if err != nil { + return nil, err + } + if len(changeScript) > P2PKHPkScriptSize { + return nil, errors.New("fee estimation requires change " + + "scripts no larger than P2PKH output scripts") + } + change := wire.NewTxOut(int64(changeAmount), changeScript) + l := len(outputs) + unsignedTransaction.TxOut = append(outputs[:l:l], change) + changeIndex = l + } + + return &txauthor.AuthoredTx{ + Tx: unsignedTransaction, + PrevScripts: scripts, + TotalInput: inputAmount, + ChangeIndex: changeIndex, + }, nil + } +} + +func (w *BitcoinCashWallet) bumpFee(txid chainhash.Hash) (*chainhash.Hash, error) { + txn, err := w.db.Txns().Get(txid) + if err != nil { + return nil, err + } + if txn.Height > 0 { + return nil, spvwallet.BumpFeeAlreadyConfirmedError + } + if txn.Height < 0 { + return nil, spvwallet.BumpFeeTransactionDeadError + } + // Check utxos for CPFP + utxos, _ := w.db.Utxos().GetAll() + for _, u := range utxos { + if u.Op.Hash.IsEqual(&txid) && u.AtHeight == 0 { + addr, err := w.ScriptToAddress(u.ScriptPubkey) + if err != nil { + return nil, err + } + key, err := w.km.GetKeyForScript(addr.ScriptAddress()) + if err != nil { + return nil, err + } + h, err := hex.DecodeString(u.Op.Hash.String()) + if err != nil { + return nil, err + } + in := wi.TransactionInput{ + LinkedAddress: addr, + OutpointIndex: u.Op.Index, + OutpointHash: h, + Value: int64(u.Value), + } + transactionID, err := w.sweepAddress([]wi.TransactionInput{in}, nil, key, nil, wi.FEE_BUMP) + if err != nil { + return nil, err + } + return transactionID, nil + } + } + return nil, spvwallet.BumpFeeNotFoundError +} + +func (w *BitcoinCashWallet) sweepAddress(ins []wi.TransactionInput, address *btc.Address, key *hd.ExtendedKey, redeemScript *[]byte, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + var internalAddr btc.Address + if address != nil { + internalAddr = *address + } else { + internalAddr = w.CurrentAddress(wi.INTERNAL) + } + script, err := bchutil.PayToAddrScript(internalAddr) + if err != nil { + return nil, err + } + + var val int64 + var inputs []*wire.TxIn + additionalPrevScripts := make(map[wire.OutPoint][]byte) + for _, in := range ins { + val += in.Value + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return nil, err + } + script, err := bchutil.PayToAddrScript(in.LinkedAddress) + if err != nil { + return nil, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + inputs = append(inputs, input) + additionalPrevScripts[*outpoint] = script + } + out := wire.NewTxOut(val, script) + + txType := P2PKH + if redeemScript != nil { + txType = P2SH_1of2_Multisig + _, err := spvwallet.LockTimeFromRedeemScript(*redeemScript) + if err == nil { + txType = P2SH_Multisig_Timelock_1Sig + } + } + estimatedSize := EstimateSerializeSize(len(ins), []*wire.TxOut{out}, false, txType) + + // Calculate the fee + feePerByte := int(w.GetFeePerByte(feeLevel)) + fee := estimatedSize * feePerByte + + outVal := val - int64(fee) + if outVal < 0 { + outVal = 0 + } + out.Value = outVal + + tx := &wire.MsgTx{ + Version: wire.TxVersion, + TxIn: inputs, + TxOut: []*wire.TxOut{out}, + LockTime: 0, + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + // Sign tx + privKey, err := key.ECPrivKey() + if err != nil { + return nil, err + } + pk := privKey.PubKey().SerializeCompressed() + addressPub, err := btc.NewAddressPubKey(pk, w.params) + + getKey := txscript.KeyClosure(func(addr btc.Address) (*btcec.PrivateKey, bool, error) { + if addressPub.EncodeAddress() == addr.EncodeAddress() { + wif, err := btc.NewWIF(privKey, w.params, true) + if err != nil { + return nil, false, err + } + return wif.PrivKey, wif.CompressPubKey, nil + } + return nil, false, errors.New("Not found") + }) + getScript := txscript.ScriptClosure(func(addr btc.Address) ([]byte, error) { + if redeemScript == nil { + return []byte{}, nil + } + return *redeemScript, nil + }) + + // Check if time locked + var timeLocked bool + if redeemScript != nil { + rs := *redeemScript + if rs[0] == txscript.OP_IF { + timeLocked = true + tx.Version = 2 + for _, txIn := range tx.TxIn { + locktime, err := spvwallet.LockTimeFromRedeemScript(*redeemScript) + if err != nil { + return nil, err + } + txIn.Sequence = locktime + } + } + } + + for i, txIn := range tx.TxIn { + if !timeLocked { + prevOutScript := additionalPrevScripts[txIn.PreviousOutPoint] + script, err := bchutil.SignTxOutput(w.params, + tx, i, prevOutScript, txscript.SigHashAll, getKey, + getScript, txIn.SignatureScript, ins[i].Value) + if err != nil { + return nil, errors.New("Failed to sign transaction") + } + txIn.SignatureScript = script + } else { + priv, err := key.ECPrivKey() + if err != nil { + return nil, err + } + script, err := bchutil.RawTxInSignature(tx, i, *redeemScript, txscript.SigHashAll, priv, ins[i].Value) + if err != nil { + return nil, err + } + builder := txscript.NewScriptBuilder(). + AddData(script). + AddOp(txscript.OP_0). + AddData(*redeemScript) + scriptSig, _ := builder.Script() + txIn.SignatureScript = scriptSig + } + } + + // broadcast + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + _, err = w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + txid := tx.TxHash() + return &txid, nil +} + +func (w *BitcoinCashWallet) createMultisigSignature(ins []wi.TransactionInput, outs []wi.TransactionOutput, key *hd.ExtendedKey, redeemScript []byte, feePerByte uint64) ([]wi.Signature, error) { + var sigs []wi.Signature + tx := wire.NewMsgTx(1) + for _, in := range ins { + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return sigs, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + tx.TxIn = append(tx.TxIn, input) + } + for _, out := range outs { + scriptPubkey, err := bchutil.PayToAddrScript(out.Address) + if err != nil { + return nil, err + } + output := wire.NewTxOut(out.Value, scriptPubkey) + tx.TxOut = append(tx.TxOut, output) + } + + // Subtract fee + txType := P2SH_2of3_Multisig + _, err := spvwallet.LockTimeFromRedeemScript(redeemScript) + if err == nil { + txType = P2SH_Multisig_Timelock_2Sigs + } + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, txType) + fee := estimatedSize * int(feePerByte) + if len(tx.TxOut) > 0 { + feePerOutput := fee / len(tx.TxOut) + for _, output := range tx.TxOut { + output.Value -= int64(feePerOutput) + } + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + signingKey, err := key.ECPrivKey() + if err != nil { + return sigs, err + } + + for i := range tx.TxIn { + sig, err := bchutil.RawTxInSignature(tx, i, redeemScript, txscript.SigHashAll, signingKey, ins[i].Value) + if err != nil { + continue + } + bs := wi.Signature{InputIndex: uint32(i), Signature: sig} + sigs = append(sigs, bs) + } + return sigs, nil +} + +func (w *BitcoinCashWallet) multisign(ins []wi.TransactionInput, outs []wi.TransactionOutput, sigs1 []wi.Signature, sigs2 []wi.Signature, redeemScript []byte, feePerByte uint64, broadcast bool) ([]byte, error) { + tx := wire.NewMsgTx(1) + for _, in := range ins { + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return nil, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + tx.TxIn = append(tx.TxIn, input) + } + for _, out := range outs { + scriptPubkey, err := bchutil.PayToAddrScript(out.Address) + if err != nil { + return nil, err + } + output := wire.NewTxOut(out.Value, scriptPubkey) + tx.TxOut = append(tx.TxOut, output) + } + + // Subtract fee + txType := P2SH_2of3_Multisig + _, err := spvwallet.LockTimeFromRedeemScript(redeemScript) + if err == nil { + txType = P2SH_Multisig_Timelock_2Sigs + } + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, txType) + fee := estimatedSize * int(feePerByte) + if len(tx.TxOut) > 0 { + feePerOutput := fee / len(tx.TxOut) + for _, output := range tx.TxOut { + output.Value -= int64(feePerOutput) + } + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + // Check if time locked + var timeLocked bool + if redeemScript[0] == txscript.OP_IF { + timeLocked = true + } + + for i, input := range tx.TxIn { + var sig1 []byte + var sig2 []byte + for _, sig := range sigs1 { + if int(sig.InputIndex) == i { + sig1 = sig.Signature + } + } + for _, sig := range sigs2 { + if int(sig.InputIndex) == i { + sig2 = sig.Signature + } + } + builder := txscript.NewScriptBuilder() + builder.AddOp(txscript.OP_0) + builder.AddData(sig1) + builder.AddData(sig2) + + if timeLocked { + builder.AddOp(txscript.OP_1) + } + + builder.AddData(redeemScript) + scriptSig, err := builder.Script() + if err != nil { + return nil, err + } + input.SignatureScript = scriptSig + } + // broadcast + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.BaseEncoding) + if broadcast { + _, err = w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} + +func (w *BitcoinCashWallet) generateMultisigScript(keys []hd.ExtendedKey, threshold int, timeout time.Duration, timeoutKey *hd.ExtendedKey) (addr btc.Address, redeemScript []byte, err error) { + if uint32(timeout.Hours()) > 0 && timeoutKey == nil { + return nil, nil, errors.New("Timeout key must be non nil when using an escrow timeout") + } + + if len(keys) < threshold { + return nil, nil, fmt.Errorf("unable to generate multisig script with "+ + "%d required signatures when there are only %d public "+ + "keys available", threshold, len(keys)) + } + + var ecKeys []*btcec.PublicKey + for _, key := range keys { + ecKey, err := key.ECPubKey() + if err != nil { + return nil, nil, err + } + ecKeys = append(ecKeys, ecKey) + } + + builder := txscript.NewScriptBuilder() + if uint32(timeout.Hours()) == 0 { + + builder.AddInt64(int64(threshold)) + for _, key := range ecKeys { + builder.AddData(key.SerializeCompressed()) + } + builder.AddInt64(int64(len(ecKeys))) + builder.AddOp(txscript.OP_CHECKMULTISIG) + + } else { + ecKey, err := timeoutKey.ECPubKey() + if err != nil { + return nil, nil, err + } + sequenceLock := blockchain.LockTimeToSequence(false, uint32(timeout.Hours()*6)) + builder.AddOp(txscript.OP_IF) + builder.AddInt64(int64(threshold)) + for _, key := range ecKeys { + builder.AddData(key.SerializeCompressed()) + } + builder.AddInt64(int64(len(ecKeys))) + builder.AddOp(txscript.OP_CHECKMULTISIG) + builder.AddOp(txscript.OP_ELSE). + AddInt64(int64(sequenceLock)). + AddOp(txscript.OP_CHECKSEQUENCEVERIFY). + AddOp(txscript.OP_DROP). + AddData(ecKey.SerializeCompressed()). + AddOp(txscript.OP_CHECKSIG). + AddOp(txscript.OP_ENDIF) + } + redeemScript, err = builder.Script() + if err != nil { + return nil, nil, err + } + addr, err = bchutil.NewCashAddressScriptHash(redeemScript, w.params) + if err != nil { + return nil, nil, err + } + return addr, redeemScript, nil +} + +func (w *BitcoinCashWallet) estimateSpendFee(amount int64, feeLevel wi.FeeLevel) (uint64, error) { + // Since this is an estimate we can use a dummy output address. Let's use a long one so we don't under estimate. + addr, err := bchutil.DecodeAddress("qpf464w2g36kyklq9shvyjk9lvuf6ph7jv3k8qpq0m", w.params) + if err != nil { + return 0, err + } + tx, err := w.buildTx(amount, addr, feeLevel, nil) + if err != nil { + return 0, err + } + var outval int64 + for _, output := range tx.TxOut { + outval += output.Value + } + var inval int64 + utxos, err := w.db.Utxos().GetAll() + if err != nil { + return 0, err + } + for _, input := range tx.TxIn { + for _, utxo := range utxos { + if utxo.Op.Hash.IsEqual(&input.PreviousOutPoint.Hash) && utxo.Op.Index == input.PreviousOutPoint.Index { + inval += utxo.Value + break + } + } + } + if inval < outval { + return 0, errors.New("Error building transaction: inputs less than outputs") + } + return uint64(inval - outval), err +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/bitcoincash/txsizes.go b/vendor/github.com/OpenBazaar/multiwallet/bitcoincash/txsizes.go new file mode 100644 index 0000000000..0db2af0eb2 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/bitcoincash/txsizes.go @@ -0,0 +1,249 @@ +package bitcoincash + +// Copyright (c) 2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* Copied here from a btcd internal package*/ + +import ( + "github.com/btcsuite/btcd/wire" +) + +// Worst case script and input/output size estimates. +const ( + // RedeemP2PKHSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2PKH output. + // It is calculated as: + // + // - OP_DATA_73 + // - 72 bytes DER signature + 1 byte sighash + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + RedeemP2PKHSigScriptSize = 1 + 73 + 1 + 33 + + // RedeemP2SHMultisigSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a 2 of 3 P2SH multisig output with compressed keys. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_PUSHDATA + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + RedeemP2SH2of3MultisigSigScriptSize = 1 + 1 + 72 + 1 + 72 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + + // RedeemP2SH1of2MultisigSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a 1 of 2 P2SH multisig output with compressed keys. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_PUSHDATA + // - OP_1 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP2 + // - OP_CHECKMULTISIG + RedeemP2SH1of2MultisigSigScriptSize = 1 + 1 + 72 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 1 + + // RedeemP2SHMultisigTimelock1SigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2SH timelocked multisig using the timeout. + // It is calculated as: + // + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_0 + // - OP_PUSHDATA + // - OP_IF + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + // - OP_ELSE + // - OP_PUSHDATA + // - 2 byte block height + // - OP_CHECKSEQUENCEVERIFY + // - OP_DROP + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_CHECKSIG + // - OP_ENDIF + RedeemP2SHMultisigTimelock1SigScriptSize = 1 + 72 + 1 + 1 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + 1 + 1 + 2 + 1 + 1 + 1 + 33 + 1 + 1 + + // RedeemP2SHMultisigTimelock2SigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2SH timelocked multisig without using the timeout. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_1 + // - OP_PUSHDATA + // - OP_IF + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + // - OP_ELSE + // - OP_PUSHDATA + // - 2 byte block height + // - OP_CHECKSEQUENCEVERIFY + // - OP_DROP + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_CHECKSIG + // - OP_ENDIF + RedeemP2SHMultisigTimelock2SigScriptSize = 1 + 1 + 72 + +1 + 72 + 1 + 1 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + 1 + 1 + 2 + 1 + 1 + 1 + 33 + 1 + 1 + + // P2PKHPkScriptSize is the size of a transaction output script that + // pays to a compressed pubkey hash. It is calculated as: + // + // - OP_DUP + // - OP_HASH160 + // - OP_DATA_20 + // - 20 bytes pubkey hash + // - OP_EQUALVERIFY + // - OP_CHECKSIG + P2PKHPkScriptSize = 1 + 1 + 1 + 20 + 1 + 1 + + // RedeemP2PKHInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2PKH output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - signature script + // - 4 bytes sequence + RedeemP2PKHInputSize = 32 + 4 + 1 + RedeemP2PKHSigScriptSize + 4 + + // RedeemP2SH2of3MultisigInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH 2 of 3 multisig output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SH2of3MultisigInputSize = 32 + 4 + 1 + 4 + (RedeemP2SH2of3MultisigSigScriptSize / 4) + + // RedeemP2SH1of2MultisigInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH 2 of 3 multisig output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SH1of2MultisigInputSize = 32 + 4 + 1 + 4 + (RedeemP2SH1of2MultisigSigScriptSize / 4) + + // RedeemP2SHMultisigTimelock1InputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed p2sh timelocked multig output with using the timeout. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SHMultisigTimelock1InputSize = 32 + 4 + 1 + 4 + (RedeemP2SHMultisigTimelock1SigScriptSize / 4) + + // RedeemP2SHMultisigTimelock2InputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH timelocked multisig output without using the timeout. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SHMultisigTimelock2InputSize = 32 + 4 + 1 + 4 + (RedeemP2SHMultisigTimelock2SigScriptSize / 4) + + // P2PKHOutputSize is the serialize size of a transaction output with a + // P2PKH output script. It is calculated as: + // + // - 8 bytes output value + // - 1 byte compact int encoding value 25 + // - 25 bytes P2PKH output script + P2PKHOutputSize = 8 + 1 + P2PKHPkScriptSize +) + +type InputType int + +const ( + P2PKH InputType = iota + P2SH_1of2_Multisig + P2SH_2of3_Multisig + P2SH_Multisig_Timelock_1Sig + P2SH_Multisig_Timelock_2Sigs +) + +// EstimateSerializeSize returns a worst case serialize size estimate for a +// signed transaction that spends inputCount number of compressed P2PKH outputs +// and contains each transaction output from txOuts. The estimated size is +// incremented for an additional P2PKH change output if addChangeOutput is true. +func EstimateSerializeSize(inputCount int, txOuts []*wire.TxOut, addChangeOutput bool, inputType InputType) int { + changeSize := 0 + outputCount := len(txOuts) + if addChangeOutput { + changeSize = P2PKHOutputSize + outputCount++ + } + + var redeemScriptSize int + switch inputType { + case P2PKH: + redeemScriptSize = RedeemP2PKHInputSize + case P2SH_1of2_Multisig: + redeemScriptSize = RedeemP2SH1of2MultisigInputSize + case P2SH_2of3_Multisig: + redeemScriptSize = RedeemP2SH2of3MultisigInputSize + case P2SH_Multisig_Timelock_1Sig: + redeemScriptSize = RedeemP2SHMultisigTimelock1InputSize + case P2SH_Multisig_Timelock_2Sigs: + redeemScriptSize = RedeemP2SHMultisigTimelock2InputSize + } + + // 10 additional bytes are for version, locktime, and segwit flags + return 10 + wire.VarIntSerializeSize(uint64(inputCount)) + + wire.VarIntSerializeSize(uint64(outputCount)) + + inputCount*redeemScriptSize + + SumOutputSerializeSizes(txOuts) + + changeSize +} + +// SumOutputSerializeSizes sums up the serialized size of the supplied outputs. +func SumOutputSerializeSizes(outputs []*wire.TxOut) (serializeSize int) { + for _, txOut := range outputs { + serializeSize += txOut.SerializeSize() + } + return serializeSize +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/bitcoincash/wallet.go b/vendor/github.com/OpenBazaar/multiwallet/bitcoincash/wallet.go new file mode 100644 index 0000000000..40d0d14496 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/bitcoincash/wallet.go @@ -0,0 +1,297 @@ +package bitcoincash + +import ( + "bytes" + "fmt" + "io" + "time" + + wi "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + hd "github.com/btcsuite/btcutil/hdkeychain" + "github.com/btcsuite/btcwallet/wallet/txrules" + bcw "github.com/cpacia/BitcoinCash-Wallet" + er "github.com/cpacia/BitcoinCash-Wallet/exchangerates" + "github.com/cpacia/bchutil" + "github.com/tyler-smith/go-bip39" + "golang.org/x/net/proxy" + + "github.com/OpenBazaar/multiwallet/client" + "github.com/OpenBazaar/multiwallet/config" + "github.com/OpenBazaar/multiwallet/keys" + "github.com/OpenBazaar/multiwallet/service" + "github.com/OpenBazaar/multiwallet/util" +) + +type BitcoinCashWallet struct { + db wi.Datastore + km *keys.KeyManager + params *chaincfg.Params + client client.APIClient + ws *service.WalletService + fp *bcw.FeeProvider + + mPrivKey *hd.ExtendedKey + mPubKey *hd.ExtendedKey +} + +func NewBitcoinCashWallet(cfg config.CoinConfig, mnemonic string, params *chaincfg.Params, proxy proxy.Dialer) (*BitcoinCashWallet, error) { + seed := bip39.NewSeed(mnemonic, "") + + mPrivKey, err := hd.NewMaster(seed, params) + if err != nil { + return nil, err + } + mPubKey, err := mPrivKey.Neuter() + if err != nil { + return nil, err + } + km, err := keys.NewKeyManager(cfg.DB.Keys(), params, mPrivKey, wi.BitcoinCash, bitcoinCashAddress) + if err != nil { + return nil, err + } + + c, err := client.NewInsightClient(cfg.ClientAPI.String(), proxy) + if err != nil { + return nil, err + } + + wm := service.NewWalletService(cfg.DB, km, c, params, wi.BitcoinCash) + + fp := bcw.NewFeeProvider(cfg.MaxFee, cfg.HighFee, cfg.MediumFee, cfg.LowFee, er.NewBitcoinCashPriceFetcher(proxy)) + + return &BitcoinCashWallet{cfg.DB, km, params, c, wm, fp, mPrivKey, mPubKey}, nil +} + +func bitcoinCashAddress(key *hd.ExtendedKey, params *chaincfg.Params) (btcutil.Address, error) { + addr, err := key.Address(params) + if err != nil { + return nil, err + } + return bchutil.NewCashAddressPubKeyHash(addr.ScriptAddress(), params) +} + +func (w *BitcoinCashWallet) Start() { + w.ws.Start() +} + +func (w *BitcoinCashWallet) Params() *chaincfg.Params { + return w.params +} + +func (w *BitcoinCashWallet) CurrencyCode() string { + if w.params.Name == chaincfg.MainNetParams.Name { + return "bch" + } else { + return "tbch" + } +} + +func (w *BitcoinCashWallet) IsDust(amount int64) bool { + return txrules.IsDustAmount(btcutil.Amount(amount), 25, txrules.DefaultRelayFeePerKb) +} + +func (w *BitcoinCashWallet) MasterPrivateKey() *hd.ExtendedKey { + return w.mPrivKey +} + +func (w *BitcoinCashWallet) MasterPublicKey() *hd.ExtendedKey { + return w.mPubKey +} + +func (w *BitcoinCashWallet) ChildKey(keyBytes []byte, chaincode []byte, isPrivateKey bool) (*hd.ExtendedKey, error) { + parentFP := []byte{0x00, 0x00, 0x00, 0x00} + var id []byte + if isPrivateKey { + id = w.params.HDPrivateKeyID[:] + } else { + id = w.params.HDPublicKeyID[:] + } + hdKey := hd.NewExtendedKey( + id, + keyBytes, + chaincode, + parentFP, + 0, + 0, + isPrivateKey) + return hdKey.Child(0) +} + +func (w *BitcoinCashWallet) CurrentAddress(purpose wi.KeyPurpose) btcutil.Address { + key, _ := w.km.GetCurrentKey(purpose) + addr, _ := w.km.KeyToAddress(key) + return btcutil.Address(addr) +} + +func (w *BitcoinCashWallet) NewAddress(purpose wi.KeyPurpose) btcutil.Address { + i, _ := w.db.Keys().GetUnused(purpose) + key, _ := w.km.GenerateChildKey(purpose, uint32(i[1])) + addr, _ := w.km.KeyToAddress(key) + w.db.Keys().MarkKeyAsUsed(addr.ScriptAddress()) + return btcutil.Address(addr) +} + +func (w *BitcoinCashWallet) DecodeAddress(addr string) (btcutil.Address, error) { + return bchutil.DecodeAddress(addr, w.params) +} + +func (w *BitcoinCashWallet) ScriptToAddress(script []byte) (btcutil.Address, error) { + return bchutil.ExtractPkScriptAddrs(script, w.params) +} + +func (w *BitcoinCashWallet) AddressToScript(addr btcutil.Address) ([]byte, error) { + return bchutil.PayToAddrScript(addr) +} + +func (w *BitcoinCashWallet) HasKey(addr btcutil.Address) bool { + _, err := w.km.GetKeyForScript(addr.ScriptAddress()) + if err != nil { + return false + } + return true +} + +func (w *BitcoinCashWallet) Balance() (confirmed, unconfirmed int64) { + utxos, _ := w.db.Utxos().GetAll() + txns, _ := w.db.Txns().GetAll(false) + return util.CalcBalance(utxos, txns) +} + +func (w *BitcoinCashWallet) Transactions() ([]wi.Txn, error) { + return w.db.Txns().GetAll(false) +} + +func (w *BitcoinCashWallet) GetTransaction(txid chainhash.Hash) (wi.Txn, error) { + txn, err := w.db.Txns().Get(txid) + return txn, err +} + +func (w *BitcoinCashWallet) ChainTip() (uint32, chainhash.Hash) { + return w.ws.ChainTip() +} + +func (w *BitcoinCashWallet) GetFeePerByte(feeLevel wi.FeeLevel) uint64 { + return w.fp.GetFeePerByte(feeLevel) +} + +func (w *BitcoinCashWallet) Spend(amount int64, addr btcutil.Address, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + tx, err := w.buildTx(amount, addr, feeLevel, nil) + if err != nil { + return nil, err + } + // Broadcast + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.BaseEncoding) + + _, err = w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + + ch := tx.TxHash() + return &ch, nil +} + +func (w *BitcoinCashWallet) BumpFee(txid chainhash.Hash) (*chainhash.Hash, error) { + return w.bumpFee(txid) +} + +func (w *BitcoinCashWallet) EstimateFee(ins []wi.TransactionInput, outs []wi.TransactionOutput, feePerByte uint64) uint64 { + tx := new(wire.MsgTx) + for _, out := range outs { + scriptPubKey, _ := bchutil.PayToAddrScript(out.Address) + output := wire.NewTxOut(out.Value, scriptPubKey) + tx.TxOut = append(tx.TxOut, output) + } + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, P2PKH) + fee := estimatedSize * int(feePerByte) + return uint64(fee) +} + +func (w *BitcoinCashWallet) EstimateSpendFee(amount int64, feeLevel wi.FeeLevel) (uint64, error) { + return w.estimateSpendFee(amount, feeLevel) +} + +func (w *BitcoinCashWallet) SweepAddress(ins []wi.TransactionInput, address *btcutil.Address, key *hd.ExtendedKey, redeemScript *[]byte, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + return w.sweepAddress(ins, address, key, redeemScript, feeLevel) +} + +func (w *BitcoinCashWallet) CreateMultisigSignature(ins []wi.TransactionInput, outs []wi.TransactionOutput, key *hd.ExtendedKey, redeemScript []byte, feePerByte uint64) ([]wi.Signature, error) { + return w.createMultisigSignature(ins, outs, key, redeemScript, feePerByte) +} + +func (w *BitcoinCashWallet) Multisign(ins []wi.TransactionInput, outs []wi.TransactionOutput, sigs1 []wi.Signature, sigs2 []wi.Signature, redeemScript []byte, feePerByte uint64, broadcast bool) ([]byte, error) { + return w.multisign(ins, outs, sigs1, sigs2, redeemScript, feePerByte, broadcast) +} + +func (w *BitcoinCashWallet) GenerateMultisigScript(keys []hd.ExtendedKey, threshold int, timeout time.Duration, timeoutKey *hd.ExtendedKey) (addr btcutil.Address, redeemScript []byte, err error) { + return w.generateMultisigScript(keys, threshold, timeout, timeoutKey) +} + +func (w *BitcoinCashWallet) AddWatchedAddress(addr btcutil.Address) error { + script, err := w.AddressToScript(addr) + if err != nil { + return err + } + err = w.db.WatchedScripts().Put(script) + if err != nil { + return err + } + w.client.ListenAddress(addr) + return nil +} + +func (w *BitcoinCashWallet) AddWatchedScript(script []byte) error { + err := w.db.WatchedScripts().Put(script) + if err != nil { + return err + } + addr, err := w.ScriptToAddress(script) + if err != nil { + return err + } + w.client.ListenAddress(addr) + return nil +} + +func (w *BitcoinCashWallet) AddTransactionListener(callback func(wi.TransactionCallback)) { + w.ws.AddTransactionListener(callback) +} + +func (w *BitcoinCashWallet) ReSyncBlockchain(fromTime time.Time) { + go w.ws.UpdateState() +} + +func (w *BitcoinCashWallet) GetConfirmations(txid chainhash.Hash) (uint32, uint32, error) { + txn, err := w.db.Txns().Get(txid) + if err != nil { + return 0, 0, err + } + if txn.Height == 0 { + return 0, 0, nil + } + chainTip, _ := w.ChainTip() + return chainTip - uint32(txn.Height) + 1, uint32(txn.Height), nil +} + +func (w *BitcoinCashWallet) Close() { + w.ws.Stop() + w.client.Close() +} + +func (w *BitcoinCashWallet) DumpTables(wr io.Writer) { + fmt.Fprintln(wr, "Transactions-----") + txns, _ := w.db.Txns().GetAll(true) + for _, tx := range txns { + fmt.Fprintf(wr, "Hash: %s, Height: %d, Value: %d, WatchOnly: %t\n", tx.Txid, int(tx.Height), int(tx.Value), tx.WatchOnly) + } + fmt.Fprintln(wr, "\nUtxos-----") + utxos, _ := w.db.Utxos().GetAll() + for _, u := range utxos { + fmt.Fprintf(wr, "Hash: %s, Index: %d, Height: %d, Value: %d, WatchOnly: %t\n", u.Op.Hash.String(), int(u.Op.Index), int(u.AtHeight), int(u.Value), u.WatchOnly) + } +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/client/client.go b/vendor/github.com/OpenBazaar/multiwallet/client/client.go new file mode 100644 index 0000000000..a6e0ebccce --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/client/client.go @@ -0,0 +1,451 @@ +package client + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + + "io/ioutil" + "net" + "net/http" + "net/url" + "path" + "strconv" + "time" + + "github.com/OpenBazaar/golang-socketio" + "github.com/OpenBazaar/golang-socketio/protocol" + "github.com/OpenBazaar/multiwallet/client/transport" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcutil" + "github.com/op/go-logging" + "golang.org/x/net/proxy" +) + +var log = logging.MustGetLogger("client") + +type InsightClient struct { + httpClient http.Client + apiUrl url.URL + blockNotifyChan chan Block + txNotifyChan chan Transaction + socketClient SocketClient +} + +func NewInsightClient(apiUrl string, proxyDialer proxy.Dialer) (*InsightClient, error) { + u, err := url.Parse(apiUrl) + if err != nil { + return nil, err + } + var port int + var secure bool + if u.Scheme == "https" { + port = 443 + secure = true + } else if u.Scheme == "http" { + port = 80 + secure = false + } else { + return nil, errors.New("Unknown url scheme") + } + dial := net.Dial + if proxyDialer != nil { + dial = proxyDialer.Dial + } + socketClient, err := gosocketio.Dial( + gosocketio.GetUrl(u.Host, port, secure), + transport.GetDefaultWebsocketTransport(proxyDialer), + ) + if err != nil { + return nil, err + } + socketReady := make(chan struct{}) + socketClient.On(gosocketio.OnConnection, func(h *gosocketio.Channel, args interface{}) { + close(socketReady) + }) + select { + case <-time.After(10 * time.Second): + return nil, errors.New("Timed out waiting for websocket connection") + case <-socketReady: + break + } + + bch := make(chan Block) + tch := make(chan Transaction) + tbTransport := &http.Transport{Dial: dial} + ic := &InsightClient{ + http.Client{Timeout: time.Second * 30, Transport: tbTransport}, + *u, + bch, + tch, + socketClient, + } + ic.setupListeners() + return ic, nil +} + +func (i *InsightClient) Close() { + i.socketClient.Close() +} + +func (i *InsightClient) doRequest(endpoint, method string, body []byte, query url.Values) (*http.Response, error) { + requestUrl := i.apiUrl + requestUrl.Path = path.Join(i.apiUrl.Path, endpoint) + req, err := http.NewRequest(method, requestUrl.String(), bytes.NewReader(body)) + if query != nil { + req.URL.RawQuery = query.Encode() + } + if err != nil { + return nil, fmt.Errorf("creating request: %s", err) + } + req.Header.Add("Content-Type", "application/json") + + resp, err := i.httpClient.Do(req) + if err != nil { + return nil, err + } + // Try again if for some reason it returned a bad request + if resp.StatusCode == http.StatusBadRequest { + // Reset the body so we can read it again. + req.Body = ioutil.NopCloser(bytes.NewReader(body)) + resp, err = i.httpClient.Do(req) + if err != nil { + return nil, err + } + } + if resp.StatusCode != http.StatusOK { + respBody, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("status not ok: %s, body: %s", resp.Status, string(respBody)) + } + return resp, nil +} + +func (i *InsightClient) GetInfo() (*Info, error) { + q, err := url.ParseQuery("?q=values") + if err != nil { + return nil, err + } + resp, err := i.doRequest("status", http.MethodGet, nil, q) + if err != nil { + return nil, err + } + decoder := json.NewDecoder(resp.Body) + stat := new(Status) + defer resp.Body.Close() + if err = decoder.Decode(stat); err != nil { + return nil, fmt.Errorf("error decoding status: %s", err) + } + info := stat.Info + f, err := toFloat(stat.Info.RelayFeeIface) + if err != nil { + return nil, err + } + info.RelayFee = f + f, err = toFloat(stat.Info.DifficultyIface) + if err != nil { + return nil, err + } + info.Difficulty = f + return &info, nil +} + +func (i *InsightClient) GetTransaction(txid string) (*Transaction, error) { + resp, err := i.doRequest("tx/"+txid, http.MethodGet, nil, nil) + if err != nil { + return nil, err + } + tx := new(Transaction) + decoder := json.NewDecoder(resp.Body) + defer resp.Body.Close() + if err = decoder.Decode(tx); err != nil { + return nil, fmt.Errorf("error decoding transactions: %s", err) + } + for n, in := range tx.Inputs { + f, err := toFloat(in.ValueIface) + if err != nil { + return nil, err + } + tx.Inputs[n].Value = f + } + for n, out := range tx.Outputs { + f, err := toFloat(out.ValueIface) + if err != nil { + return nil, err + } + tx.Outputs[n].Value = f + } + return tx, nil +} + +func (i *InsightClient) GetRawTransaction(txid string) ([]byte, error) { + resp, err := i.doRequest("rawtx/"+txid, http.MethodGet, nil, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + tx := new(RawTxResponse) + if err = json.NewDecoder(resp.Body).Decode(tx); err != nil { + return nil, fmt.Errorf("error decoding transactions: %s", err) + } + return hex.DecodeString(tx.RawTx) +} + +func (i *InsightClient) GetTransactions(addrs []btcutil.Address) ([]Transaction, error) { + var txs []Transaction + from := 0 + for { + tl, err := i.getTransactions(addrs, from, from+50) + if err != nil { + return txs, err + } + txs = append(txs, tl.Items...) + if len(txs) >= tl.TotalItems { + break + } + from += 50 + } + return txs, nil +} + +func (i *InsightClient) getTransactions(addrs []btcutil.Address, from, to int) (*TransactionList, error) { + type req struct { + Addrs string `json:"addrs"` + From int `json:"from"` + To int `json:"to"` + } + s := `` + for n, addr := range addrs { + s += addr.String() + if n < len(addrs)-1 { + s += "," + } + } + r := &req{ + Addrs: s, + From: from, + To: to, + } + b, err := json.Marshal(r) + if err != nil { + return nil, err + } + resp, err := i.doRequest("addrs/txs", http.MethodPost, b, nil) + if err != nil { + return nil, err + } + tl := new(TransactionList) + decoder := json.NewDecoder(resp.Body) + defer resp.Body.Close() + if err = decoder.Decode(tl); err != nil { + return nil, fmt.Errorf("error decoding transaction list: %s", err) + } + for z, tx := range tl.Items { + for n, in := range tx.Inputs { + f, err := toFloat(in.ValueIface) + if err != nil { + return nil, err + } + tl.Items[z].Inputs[n].Value = f + } + for n, out := range tx.Outputs { + f, err := toFloat(out.ValueIface) + if err != nil { + return nil, err + } + tl.Items[z].Outputs[n].Value = f + } + } + return tl, nil +} + +func (i *InsightClient) GetUtxos(addrs []btcutil.Address) ([]Utxo, error) { + type req struct { + Addrs string `json:"addrs"` + } + s := `` + for n, addr := range addrs { + s += addr.String() + if n < len(addrs)-1 { + s += "," + } + } + r := &req{ + Addrs: s, + } + b, err := json.Marshal(r) + if err != nil { + return nil, err + } + resp, err := i.doRequest("addrs/utxo", http.MethodPost, b, nil) + if err != nil { + return nil, err + } + utxos := []Utxo{} + decoder := json.NewDecoder(resp.Body) + defer resp.Body.Close() + if err = decoder.Decode(&utxos); err != nil { + return nil, fmt.Errorf("error decoding utxo list: %s", err) + } + for z, u := range utxos { + f, err := toFloat(u.AmountIface) + if err != nil { + return nil, err + } + utxos[z].Amount = f + } + return utxos, nil +} + +func (i *InsightClient) BlockNotify() <-chan Block { + return i.blockNotifyChan +} + +func (i *InsightClient) TransactionNotify() <-chan Transaction { + return i.txNotifyChan +} + +func (i *InsightClient) ListenAddress(addr btcutil.Address) { + var args []interface{} + args = append(args, "bitcoind/addresstxid") + args = append(args, []string{addr.String()}) + i.socketClient.Emit("subscribe", args) +} + +func (i *InsightClient) setupListeners() { + i.socketClient.On("bitcoind/hashblock", func(h *gosocketio.Channel, arg interface{}) { + best, err := i.GetBestBlock() + if err != nil { + log.Errorf("Error downloading best block: %s", err.Error()) + return + } + i.blockNotifyChan <- *best + }) + i.socketClient.Emit("subscribe", protocol.ToArgArray("bitcoind/hashblock")) + + i.socketClient.On("bitcoind/addresstxid", func(h *gosocketio.Channel, arg interface{}) { + m, ok := arg.(map[string]interface{}) + if !ok { + log.Errorf("Error checking type after socket notification: %T", arg) + return + } + for _, v := range m { + txid, ok := v.(string) + if !ok { + log.Errorf("Error checking type after socket notification: %T", arg) + return + } + _, err := chainhash.NewHashFromStr(txid) // Check is 256 bit hash. Might also be address + if err == nil { + tx, err := i.GetTransaction(txid) + if err != nil { + log.Errorf("Error downloading tx after socket notification: %s", err.Error()) + return + } + i.txNotifyChan <- *tx + } + } + }) +} + +func (i *InsightClient) Broadcast(tx []byte) (string, error) { + txHex := hex.EncodeToString(tx) + type RawTx struct { + Raw string `json:"rawtx"` + } + t := RawTx{txHex} + txJson, err := json.Marshal(&t) + if err != nil { + return "", fmt.Errorf("error encoding tx: %s", err) + } + resp, err := i.doRequest("tx/send", http.MethodPost, txJson, nil) + if err != nil { + return "", fmt.Errorf("error broadcasting tx: %s", err) + } + defer resp.Body.Close() + + type Txid struct { + Result string `json:"result"` + } + type Response struct { + Txid Txid `json:"txid"` + } + rs := new(Response) + if err = json.NewDecoder(resp.Body).Decode(rs); err != nil { + return "", fmt.Errorf("error decoding txid: %s", err) + } + return rs.Txid.Result, nil +} + +func (i *InsightClient) GetBestBlock() (*Block, error) { + q, err := url.ParseQuery("limit=2") + if err != nil { + return nil, err + } + resp, err := i.doRequest("blocks", http.MethodGet, nil, q) + if err != nil { + return nil, err + } + decoder := json.NewDecoder(resp.Body) + sl := new(BlockList) + defer resp.Body.Close() + if err = decoder.Decode(sl); err != nil { + return nil, fmt.Errorf("error decoding block list: %s", err) + } + if len(sl.Blocks) < 2 { + return nil, fmt.Errorf("API returned incorrect number of block summaries: n=%d", len(sl.Blocks)) + } + sum := sl.Blocks[0] + sum.PreviousBlockhash = sl.Blocks[1].Hash + return &sum, nil +} + +func (i *InsightClient) GetBlocksBefore(to time.Time, limit int) (*BlockList, error) { + resp, err := i.doRequest("blocks", http.MethodGet, nil, url.Values{ + "blockDate": {to.Format("2006-01-02")}, + "startTimestamp": {fmt.Sprint(to.Unix())}, + "limit": {fmt.Sprint(limit)}, + }) + if err != nil { + return nil, err + } + list := new(BlockList) + decoder := json.NewDecoder(resp.Body) + defer resp.Body.Close() + if err = decoder.Decode(list); err != nil { + return nil, fmt.Errorf("error decoding block list: %s", err) + } + return list, nil +} + +// API sometimees returns a float64 or a string so we'll always convert it into a float64 +func toFloat(i interface{}) (float64, error) { + _, fok := i.(float64) + _, sok := i.(string) + if fok { + return i.(float64), nil + } else if sok { + s := i.(string) + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, fmt.Errorf("error parsing value float: %s", err) + } + return f, nil + } else { + return 0, errors.New("Unknown value type in response") + } +} + +func (i *InsightClient) EstimateFee(nbBlocks int) (int, error) { + resp, err := i.doRequest("utils/estimatefee", http.MethodGet, nil, url.Values{"nbBlocks": {fmt.Sprint(nbBlocks)}}) + if err != nil { + return 0, err + } + data := map[int]float64{} + defer resp.Body.Close() + if err = json.NewDecoder(resp.Body).Decode(&data); err != nil { + return 0, fmt.Errorf("error decoding fee estimate: %s", err) + } + return int(data[nbBlocks] * 1e8), nil +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/client/interface.go b/vendor/github.com/OpenBazaar/multiwallet/client/interface.go new file mode 100644 index 0000000000..4764c0643e --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/client/interface.go @@ -0,0 +1,56 @@ +package client + +import ( + "github.com/btcsuite/btcutil" +) + +type APIClient interface { + // Get info about the server + GetInfo() (*Info, error) + + // For a given txid get back the transaction metadata + GetTransaction(txid string) (*Transaction, error) + + // For a given txid get back the full transaction bytes + GetRawTransaction(txid string) ([]byte, error) + + // Get back all the transactions for the given list of addresses + GetTransactions(addrs []btcutil.Address) ([]Transaction, error) + + // Get back all spendable UTXOs for the given list of addresses + GetUtxos(addrs []btcutil.Address) ([]Utxo, error) + + // Returns a chan which fires on each new block + BlockNotify() <-chan Block + + // Returns a chan which fires whenever a new transaction is received or + // when an existing transaction confirms for all addresses the API is listening on. + TransactionNotify() <-chan Transaction + + // Listen for events on this addresses. Results are returned to TransactionNotify() + ListenAddress(addr btcutil.Address) + + // Broadcast a transaction to the network + Broadcast(tx []byte) (string, error) + + // Get info on the current chain tip + GetBestBlock() (*Block, error) + + // Estimate the fee required for a transaction + EstimateFee(nBlocks int) (int, error) + + // Close all connections and shutdown + Close() +} + +type SocketClient interface { + + // Set callback for method + On(method string, callback interface{}) error + + // Listen on method + Emit(method string, args []interface{}) error + + // Close the socket connection + Close() +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/client/mock.go b/vendor/github.com/OpenBazaar/multiwallet/client/mock.go new file mode 100644 index 0000000000..080c322c45 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/client/mock.go @@ -0,0 +1,416 @@ +package client + +import ( + "encoding/hex" + "errors" + "github.com/OpenBazaar/golang-socketio" + "github.com/btcsuite/btcutil" +) + +var MockInfo = Info{ + Version: 1, + ProtocolVersion: 9005, + Blocks: 1289596, + TimeOffset: 0, + Connections: 1024, + DifficultyIface: "1.23", + Difficulty: 1.23, + Testnet: true, + RelayFeeIface: "1.00", + RelayFee: 1.00, + Errors: "", + Network: "testnet", +} + +var MockBlocks = []Block{ + { + Hash: "000000000000004c68a477283a8db18c1d1c2155b03d9bc23d587ac5e1c4d1af", + Height: 1289594, + PreviousBlockhash: "00000000000003df72ec254d787b216ae913cb82c6ab601c4b3f19fd5d1cf9aa", + Tx: make([]string, 21), + Size: 4705, + Time: 1522349145, + }, + { + Hash: "0000000000000142ffae87224cb67206e93bf934f9fdeba75d02a7050acc6136", + Height: 1289595, + PreviousBlockhash: "000000000000004c68a477283a8db18c1d1c2155b03d9bc23d587ac5e1c4d1af", + Tx: make([]string, 30), + Size: 6623, + Time: 1522349136, + }, + { + Hash: "000000000000033ef24180d5d282d0e6d03b1185e29421fda97e1ba0ffd7c918", + Height: 1289596, + PreviousBlockhash: "0000000000000142ffae87224cb67206e93bf934f9fdeba75d02a7050acc6136", + Tx: make([]string, 5), + Size: 1186, + Time: 1522349156, + }, +} + +var MockTransactions = []Transaction{ + { + Txid: "54ebaa07c42216393b9d5816e40dd608593b92c42e2d6525f45bdd36bce8fe4d", + Version: 2, + Locktime: 512378, + Inputs: []Input{ + { + Txid: "6d892f04fc097f430d58ab06229c9b6344a130fc1842da5b990e857daed42194", + Vout: 1, + Sequence: 1, + ValueIface: "0.04294455", + Value: 0.04294455, + N: 0, + ScriptSig: Script{ + Hex: "4830450221008665481674067564ef562cfd8d1ca8f1506133fb26a2319e4b8dfba3cedfd5de022038f27121c44e6c64b93b94d72620e11b9de35fd864730175db9176ca98f1ec610121022023e49335a0dddb864ff673468a6cc04e282571b1227933fcf3ff9babbcc662", + }, + Addr: "1C74Gbij8Q5h61W58aSKGvXK4rk82T2A3y", + Satoshis: 4294455, + }, + }, + Outputs: []Output{ + { + ScriptPubKey: OutScript{ + Script: Script{ + Hex: "76a914ff3f7d402fbd6d116ba4a02af9784f3ae9b7108a88ac", + }, + Type: "pay-to-pubkey-hash", + Addresses: []string{"1QGdNEDjWnghrjfTBCTDAPZZ3ffoKvGc9B"}, + }, + ValueIface: "0.01398175", + Value: 0.01398175, + N: 0, + }, + { + ScriptPubKey: OutScript{ + Script: Script{ + Hex: "76a914f99b84270843bdab59a71ce9af15b89bef5087a388ac", + }, + Type: "pay-to-pubkey-hash", + Addresses: []string{"1PkoZDtXT63BnYGd429Vy4DoyGhdDcjQiN"}, // var + }, + ValueIface: "0.02717080", + Value: 0.02717080, + N: 1, + }, + }, + Time: 1520449061, + BlockHash: "0000000000000000003f1fb88ac3dab0e607e87def0e9031f7bea02cb464a04f", + BlockHeight: 1289475, + Confirmations: 15, + }, + { + Txid: "ff2b865c3b73439912eebf4cce9a15b12c7d7bcdd14ae1110a90541426c4e7c5", + Version: 2, + Locktime: 0, + Inputs: []Input{ + { + Txid: "54ebaa07c42216393b9d5816e40dd608593b92c42e2d6525f45bdd36bce8fe4d", + Vout: 1, + Sequence: 1, + ValueIface: "0.02717080", + Value: 0.02717080, + N: 0, + ScriptSig: Script{ + Hex: "4830450221008665481674067564ef562cfd8d1ca8f1506133fb26a2319e4b8dfba3cedfd5de022038f27121c44e6c64b93b94d72620e11b9de35fd864730175db9176ca98f1ec610121022023e49335a0dddb864ff673468a6cc04e282571b1227933fcf3ff9babbcc662", + }, + Addr: "1PkoZDtXT63BnYGd429Vy4DoyGhdDcjQiN", // var tx0:1 + Satoshis: 2717080, + }, + }, + Outputs: []Output{ + { + ScriptPubKey: OutScript{ + Script: Script{ + Hex: "a9144b18dadba74ad5ef4dbbfea47f9d5aaefe766c6387", + }, + Type: "pay-to-script-hash", + Addresses: []string{"38Y6Nt35hQcEDxyCfCEi62QLGPnr4mhANc"}, + }, + ValueIface: "0.01398175", + Value: 0.01617080, + N: 0, + }, + { + ScriptPubKey: OutScript{ + Script: Script{ + Hex: "76a914f821d6db9376dc60124de46a8683110877e1f13188ac", + }, + Type: "pay-to-pubkey-hash", + Addresses: []string{"1Pd17mbYsVPcCKLtNdPkngtizTj7zjzqeK"}, // var change + }, + ValueIface: "0.01", + Value: 0.01, + N: 1, + }, + }, + Time: 1520449061, + BlockHash: "0000000000000000003f1fb88ac3dab0e607e87def0e9031f7bea02cb464a04f", + BlockHeight: 1289475, + Confirmations: 10, + }, + { + Txid: "1d4288fa682fa376fbae73dbd74ea04b9ea33011d63315ca9d2d50d081e671d5", + Version: 2, + Locktime: 0, + Inputs: []Input{ + { + Txid: "bffb894c27dac82525c1f00a085150be94c70834e8d05ea5e7bb3bd1278d3138", + Vout: 1, + Sequence: 1, + ValueIface: "0.3", + Value: 0.3, + N: 0, + ScriptSig: Script{ + Hex: "4830450221008665481674067564ef562cfd8d1ca8f1506133fb26a2319e4b8dfba3cedfd5de022038f27121c44e6c64b93b94d72620e11b9de35fd864730175db9176ca98f1ec610121022023e49335a0dddb864ff673468a6cc04e282571b1227933fcf3ff9babbcc662", + }, + Addr: "1H2ZS69jUZz6CuCtiRCTWXr4AhAWfXc4YT", + Satoshis: 2717080, + }, + }, + Outputs: []Output{ + { + ScriptPubKey: OutScript{ + Script: Script{ + Hex: "76a914e20c0ca5875b1fb0d057e23d032ba88b9dda6f3888ac", + }, + Type: "pay-to-pubkey-hash", + Addresses: []string{"1McE9ZXFhWkFeAqR1hyAm1XaDK8zvyrFPr"}, + }, + ValueIface: "0.2", + Value: 0.2, + N: 0, + }, + { + ScriptPubKey: OutScript{ + Script: Script{ + Hex: "76a914594963287fe6684872340e9078a78d0accbec26288ac", + }, + Type: "pay-to-pubkey-hash", + Addresses: []string{"199747e2arXMBPiWfTqpBTXz3eFbeJPMqS"}, // var + }, + ValueIface: "0.1", + Value: 0.1, + N: 1, + }, + }, + Time: 1520449061, + BlockHash: "0000000000000000003f1fb88ac3dab0e607e87def0e9031f7bea02cb464a04f", + BlockHeight: 1289475, + Confirmations: 2, + }, + { + Txid: "830bf683ab8eec1a75d891689e2989f846508bc7d500cb026ef671c2d1dce20c", + Version: 2, + Locktime: 516299, + Inputs: []Input{ + { + Txid: "b466d034076ab53f4b019d573b6c68cf68c5b9a8cfbf07c8d46208d0fcf37762", + Vout: 0, + Sequence: 4294967294, + ValueIface: "0.01983741", + Value: 0.01983741, + N: 0, + ScriptSig: Script{ + Hex: "483045022100baa2b3653d48ccf2838caa549d96a40540c838c4f4a8e7048dbe158ec180b3f602206f1bb8c6d055103ce635db562c31ebd8c30565c5d415458affb9f99407ec06d10121039fea462cb64296e01384cffc16af4b86ab14b6027094399bf5a4b52e5c9ffef3", + }, + Addr: "1LUv9VNMZQR4VknWj1TBa1oDgPq53wP7BK", + Satoshis: 1983741, + }, + }, + Outputs: []Output{ + { + ScriptPubKey: OutScript{ + Script: Script{ + Hex: "76a91491a8a9e0375f10b721743782162a0b4f9fae69a888ac", + }, + Type: "pay-to-pubkey-hash", + Addresses: []string{"1EHB2mSaUXzkM6r6XgVHcutFDZoB9e2mZH"}, + }, + ValueIface: "0.01181823", + Value: 0.01181823, + N: 0, + }, + { + ScriptPubKey: OutScript{ + Script: Script{ + Hex: "a91457fc729da2a83dc8cd3c1835351c4a813c2ae8ba87", + }, + Type: "pay-to-script-hash", + Addresses: []string{"39iF8cDMhctrPVoPbi2Vb1NnErg6CEB7BZ"}, + }, + ValueIface: "0.00751918", + Value: 0.00751918, + N: 1, + }, + }, + Time: 1520449061, + BlockHash: "0000000000000000003f1fb88ac3dab0e607e87def0e9031f7bea02cb464a04f", + BlockHeight: 1289475, + Confirmations: 2, + }, +} + +var MockRawTransactions = map[string][]byte{} + +var MockUtxos = []Utxo{ + { + Address: "1Pd17mbYsVPcCKLtNdPkngtizTj7zjzqeK", // tx1:1 + ScriptPubKey: "76a914f821d6db9376dc60124de46a8683110877e1f13188ac", + Vout: 1, + Satoshis: 1000000, + Confirmations: 10, + Txid: "ff2b865c3b73439912eebf4cce9a15b12c7d7bcdd14ae1110a90541426c4e7c5", + AmountIface: "0.01", + Amount: 0.01, + }, + { + Address: "199747e2arXMBPiWfTqpBTXz3eFbeJPMqS", //tx2:1 + ScriptPubKey: "76a914594963287fe6684872340e9078a78d0accbec26288ac", + Vout: 1, + Satoshis: 10000000, + Confirmations: 2, + Txid: "1d4288fa682fa376fbae73dbd74ea04b9ea33011d63315ca9d2d50d081e671d5", + AmountIface: "0.1", + Amount: 0.1, + }, + { + Address: "39iF8cDMhctrPVoPbi2Vb1NnErg6CEB7BZ", + ScriptPubKey: "a91457fc729da2a83dc8cd3c1835351c4a813c2ae8ba87", + Vout: 1, + Satoshis: 751918, + Confirmations: 2, + Txid: "830bf683ab8eec1a75d891689e2989f846508bc7d500cb026ef671c2d1dce20c", + AmountIface: "0.00751918", + Amount: 0.00751918, + }, +} + +type MockAPIClient struct { + blockChan chan Block + txChan chan Transaction + + listeningAddrs []btcutil.Address + chainTip int + feePerBlock int + info *Info + addrToScript func(btcutil.Address) ([]byte, error) +} + +func NewMockApiClient(addrToScript func(btcutil.Address) ([]byte, error)) APIClient { + return &MockAPIClient{ + blockChan: make(chan Block), + txChan: make(chan Transaction), + chainTip: 0, + addrToScript: addrToScript, + feePerBlock: 1, + info: &MockInfo, + } +} + +func (m *MockAPIClient) GetInfo() (*Info, error) { + return m.info, nil +} + +func (m *MockAPIClient) GetTransaction(txid string) (*Transaction, error) { + for _, tx := range MockTransactions { + if tx.Txid == txid { + return &tx, nil + } + } + return nil, errors.New("Not found") +} + +func (m *MockAPIClient) GetRawTransaction(txid string) ([]byte, error) { + if raw, ok := MockRawTransactions[txid]; ok { + return raw, nil + } + return nil, errors.New("Not found") +} + +func (m *MockAPIClient) GetTransactions(addrs []btcutil.Address) ([]Transaction, error) { + txs := make([]Transaction, len(MockTransactions)) + copy(txs, MockTransactions) + txs[0].Outputs[1].ScriptPubKey.Addresses = []string{addrs[0].String()} + txs[1].Inputs[0].Addr = addrs[0].String() + txs[1].Outputs[1].ScriptPubKey.Addresses = []string{addrs[1].String()} + txs[2].Outputs[1].ScriptPubKey.Addresses = []string{addrs[2].String()} + return txs, nil +} + +func (m *MockAPIClient) GetUtxos(addrs []btcutil.Address) ([]Utxo, error) { + utxos := make([]Utxo, len(MockUtxos)) + copy(utxos, MockUtxos) + utxos[0].Address = addrs[1].String() + script, _ := m.addrToScript(addrs[1]) + utxos[0].ScriptPubKey = hex.EncodeToString(script) + utxos[1].Address = addrs[2].String() + script, _ = m.addrToScript(addrs[2]) + utxos[1].ScriptPubKey = hex.EncodeToString(script) + return utxos, nil +} + +func (m *MockAPIClient) BlockNotify() <-chan Block { + return m.blockChan +} + +func (m *MockAPIClient) TransactionNotify() <-chan Transaction { + return m.txChan +} + +func (m *MockAPIClient) ListenAddress(addr btcutil.Address) { + m.listeningAddrs = append(m.listeningAddrs, addr) +} + +func (m *MockAPIClient) Broadcast(tx []byte) (string, error) { + return "txid", nil +} + +func (m *MockAPIClient) GetBestBlock() (*Block, error) { + return &MockBlocks[m.chainTip], nil +} + +func (m *MockAPIClient) EstimateFee(nBlocks int) (int, error) { + return m.feePerBlock * nBlocks, nil +} + +func (m *MockAPIClient) Close() {} + +type MockSocketClient struct { + callbacks map[string]func(h *gosocketio.Channel, args interface{}) + listeningAddresses []string +} + +func (m *MockSocketClient) On(method string, callback interface{}) error { + c, ok := callback.(func(h *gosocketio.Channel, args interface{})) + if !ok { + return nil + } + + if method == "bitcoind/addresstxid" { + m.callbacks[method] = c + } else if method == "bitcoind/hashblock" { + m.callbacks[method] = c + } + return nil +} + +func (m *MockSocketClient) Emit(method string, args []interface{}) error { + if method == "subscribe" { + subscribeTo, ok := args[0].(string) + if !ok || subscribeTo != "bitcoind/addresstxid" { + return nil + } + addrs, ok := args[1].([]string) + if !ok { + return nil + } + for _, addr := range addrs { + m.listeningAddresses = append(m.listeningAddresses, addr) + } + } + return nil +} + +func (m *MockSocketClient) Close() {} diff --git a/vendor/github.com/OpenBazaar/multiwallet/client/models.go b/vendor/github.com/OpenBazaar/multiwallet/client/models.go new file mode 100644 index 0000000000..79f9292af9 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/client/models.go @@ -0,0 +1,170 @@ +package client + +type Status struct { + Info Info `json:"info"` +} + +type Info struct { + Version int `json:"version"` + ProtocolVersion int `json:"protocolversion"` + Blocks int `json:"blocks"` + TimeOffset int `json:"timeoffset"` + Connections int `json:"connections"` + DifficultyIface interface{} `json:"difficulty"` + Difficulty float64 `json:"-"` + Testnet bool `json:"testnet"` + RelayFeeIface interface{} `json:"relayfee"` + RelayFee float64 `json:"-"` + Errors string `json:"errors"` + Network string `json:"network"` +} + +func (i Info) IsEqual(other Info) bool { + if i.Version != other.Version { + return false + } + if i.ProtocolVersion != other.ProtocolVersion { + return false + } + if i.Blocks != other.Blocks { + return false + } + if i.TimeOffset != other.TimeOffset { + return false + } + if i.Connections != other.Connections { + return false + } + if i.Difficulty != other.Difficulty { + return false + } + if i.Testnet != other.Testnet { + return false + } + if i.RelayFee != other.RelayFee { + return false + } + if i.Errors != other.Errors { + return false + } + if i.Network != other.Network { + return false + } + return true +} + +type BlockList struct { + Blocks []Block `json:"blocks"` + Length int `json:"length"` + Pagination Pagination `json:"pagination"` +} + +type Pagination struct { + Next string `json:"next"` + Prev string `json:"prev"` + CurrentTs int `json:"currentTs"` + Current string `json:"current"` + IsToday bool `json:"isToday"` + More bool `json:"more"` + MoreTs int `json:"moreTs"` +} + +type Block struct { + Hash string `json:"hash"` + Size int `json:"size"` + Height int `json:"height"` + Version int `json:"version"` + MerkleRoot string `json:"merkleroot"` + Tx []string `json:"tx"` + Time int64 `json:"time"` + Nonce string `json:"nonce"` + Solution string `json:"solution"` + Bits string `json:"bits"` + Difficulty float64 `json:"difficulty"` + Chainwork string `json:"chainwork"` + Confirmations int `json:"confirmations"` + PreviousBlockhash string `json:"previousblockhash"` + NextBlockhash string `json:"nextblockhash"` + Reward float64 `json:"reward"` + IsMainChain bool `json:"isMainChain"` + PoolInfo *PoolInfo `json:"poolinfo"` +} + +type PoolInfo struct { + PoolName string `json:"poolName"` + URL string `json:"url"` +} + +type Utxo struct { + Address string `json:"address"` + Txid string `json:"txid"` + Vout int `json:"vout"` + ScriptPubKey string `json:"scriptPubKey"` + AmountIface interface{} `json:"amount"` + Amount float64 + Satoshis int64 `json:"satoshis"` + Confirmations int `json:"confirmations"` +} + +type TransactionList struct { + TotalItems int `json:"totalItems"` + From int `json:"from"` + To int `json:"to"` + Items []Transaction `json:"items"` +} + +type Transaction struct { + Txid string `json:"txid"` + Version int `json:"version"` + Locktime int `json:"locktime"` + Inputs []Input `json:"vin"` + Outputs []Output `json:"vout"` + BlockHash string `json:"blockhash"` + BlockHeight int `json:"blockheight"` + Confirmations int `json:"confirmations"` + Time int64 `json:"time"` + BlockTime int64 `json:"blocktime"` +} + +type RawTxResponse struct { + RawTx string `json:"rawtx"` +} + +type Input struct { + Txid string `json:"txid"` + Vout int `json:"vout"` + Sequence int `json:"sequence"` + N int `json:"n"` + ScriptSig Script `json:"scriptSig"` + Addr string `json:"addr"` + Satoshis int64 `json:"valueSat"` + ValueIface interface{} `json:"value"` + Value float64 + DoubleSpentTxid string `json:"doubleSpentTxID"` +} + +type Output struct { + ValueIface interface{} `json:"value"` + Value float64 + N int `json:"n"` + ScriptPubKey OutScript `json:"scriptPubKey"` + SpentTxid string `json:"spentTxId"` + SpentIndex int `json:"spentIndex"` + SpentHeight int `json:"spentHeight"` +} + +type Script struct { + Hex string `json:"hex"` + Asm string `json:"asm"` +} + +type OutScript struct { + Script + Addresses []string `json:"addresses"` + Type string `json:"type"` +} + +type AddressTxid struct { + Address string `json:"address"` + Txid string `json:"txid"` +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/client/transport/transport.go b/vendor/github.com/OpenBazaar/multiwallet/client/transport/transport.go new file mode 100644 index 0000000000..5a8940db3b --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/client/transport/transport.go @@ -0,0 +1,148 @@ +package transport + +import ( + "errors" + tp "github.com/OpenBazaar/golang-socketio/transport" + "github.com/gorilla/websocket" + "golang.org/x/net/proxy" + "io/ioutil" + "net" + "net/http" + "time" +) + +const ( + upgradeFailed = "Upgrade failed: " + + WsDefaultPingInterval = 30 * time.Second + WsDefaultPingTimeout = 60 * time.Second + WsDefaultReceiveTimeout = 60 * time.Second + WsDefaultSendTimeout = 60 * time.Second + WsDefaultBufferSize = 1024 * 32 +) + +var ( + ErrorBinaryMessage = errors.New("Binary messages are not supported") + ErrorBadBuffer = errors.New("Buffer error") + ErrorPacketWrong = errors.New("Wrong packet type error") + ErrorMethodNotAllowed = errors.New("Method not allowed") + ErrorHttpUpgradeFailed = errors.New("Http upgrade failed") +) + +type WebsocketConnection struct { + socket *websocket.Conn + transport *WebsocketTransport +} + +func (wsc *WebsocketConnection) GetMessage() (message string, err error) { + wsc.socket.SetReadDeadline(time.Now().Add(wsc.transport.ReceiveTimeout)) + msgType, reader, err := wsc.socket.NextReader() + if err != nil { + return "", err + } + + //support only text messages exchange + if msgType != websocket.TextMessage { + return "", ErrorBinaryMessage + } + + data, err := ioutil.ReadAll(reader) + if err != nil { + return "", ErrorBadBuffer + } + text := string(data) + + //empty messages are not allowed + if len(text) == 0 { + return "", ErrorPacketWrong + } + + return text, nil +} + +func (wsc *WebsocketConnection) WriteMessage(message string) error { + wsc.socket.SetWriteDeadline(time.Now().Add(wsc.transport.SendTimeout)) + writer, err := wsc.socket.NextWriter(websocket.TextMessage) + if err != nil { + return err + } + + if _, err := writer.Write([]byte(message)); err != nil { + return err + } + if err := writer.Close(); err != nil { + return err + } + return nil +} + +func (wsc *WebsocketConnection) Close() { + wsc.socket.Close() +} + +func (wsc *WebsocketConnection) PingParams() (interval, timeout time.Duration) { + return wsc.transport.PingInterval, wsc.transport.PingTimeout +} + +type WebsocketTransport struct { + PingInterval time.Duration + PingTimeout time.Duration + ReceiveTimeout time.Duration + SendTimeout time.Duration + + BufferSize int + + RequestHeader http.Header + + proxyDialer proxy.Dialer +} + +func (wst *WebsocketTransport) Connect(url string) (conn tp.Connection, err error) { + dial := net.Dial + if wst.proxyDialer != nil { + dial = wst.proxyDialer.Dial + } + dialer := websocket.Dialer{NetDial: dial} + socket, _, err := dialer.Dial(url, wst.RequestHeader) + if err != nil { + return nil, err + } + + return &WebsocketConnection{socket, wst}, nil +} + +func (wst *WebsocketTransport) HandleConnection( + w http.ResponseWriter, r *http.Request) (conn tp.Connection, err error) { + + if r.Method != "GET" { + http.Error(w, upgradeFailed+ErrorMethodNotAllowed.Error(), 503) + return nil, ErrorMethodNotAllowed + } + + socket, err := websocket.Upgrade(w, r, nil, wst.BufferSize, wst.BufferSize) + if err != nil { + http.Error(w, upgradeFailed+err.Error(), 503) + return nil, ErrorHttpUpgradeFailed + } + + return &WebsocketConnection{socket, wst}, nil +} + +/** +Websocket connection do not require any additional processing +*/ +func (wst *WebsocketTransport) Serve(w http.ResponseWriter, r *http.Request) {} + +/** +Returns websocket connection with default params +*/ +func GetDefaultWebsocketTransport(proxyDialer proxy.Dialer) *WebsocketTransport { + return &WebsocketTransport{ + PingInterval: WsDefaultPingInterval, + PingTimeout: WsDefaultPingTimeout, + ReceiveTimeout: WsDefaultReceiveTimeout, + SendTimeout: WsDefaultSendTimeout, + BufferSize: WsDefaultBufferSize, + proxyDialer: proxyDialer, + } +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/config/config.go b/vendor/github.com/OpenBazaar/multiwallet/config/config.go new file mode 100644 index 0000000000..b59c0b7dec --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/config/config.go @@ -0,0 +1,156 @@ +package config + +import ( + "github.com/OpenBazaar/multiwallet/datastore" + "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/chaincfg" + "github.com/op/go-logging" + "golang.org/x/net/proxy" + "net/url" + "os" + "time" +) + +type Config struct { + // Network parameters. Set mainnet, testnet, or regtest using this. + Params *chaincfg.Params + + // Bip39 mnemonic string. If empty a new mnemonic will be created. + Mnemonic string + + // The date the wallet was created. + // If before the earliest checkpoint the chain will be synced using the earliest checkpoint. + CreationDate time.Time + + // A Tor proxy can be set here causing the wallet will use Tor + Proxy proxy.Dialer + + // A logger. You can write the logs to file or stdout or however else you want. + Logger logging.Backend + + // A list of coin configs. One config should be included for each coin to be used. + Coins []CoinConfig +} + +type CoinConfig struct { + // The type of coin to configure + CoinType wallet.CoinType + + // The default fee-per-byte for each level + LowFee uint64 + MediumFee uint64 + HighFee uint64 + + // The highest allowable fee-per-byte + MaxFee uint64 + + // External API to query to look up fees. If this field is nil then the default fees will be used. + // If the API is unreachable then the default fees will likewise be used. If the API returns a fee + // greater than MaxFee then the MaxFee will be used in place. The API response must be formatted as + // { "fastestFee": 40, "halfHourFee": 20, "hourFee": 10 } + FeeAPI url.URL + + // The trusted API to use for querying for balances and listening to blockchain events. + ClientAPI url.URL + + // An implementation of the Datastore interface for each desired coin + DB wallet.Datastore +} + +func NewDefaultConfig(coinTypes map[wallet.CoinType]bool, params *chaincfg.Params) *Config { + cfg := &Config{ + Params: params, + Logger: logging.NewLogBackend(os.Stdout, "", 0), + } + var testnet bool + if params.Name == chaincfg.TestNet3Params.Name { + testnet = true + } + mockDB := datastore.NewMockMultiwalletDatastore() + if coinTypes[wallet.Bitcoin] { + var apiEndpoint string + if !testnet { + apiEndpoint = "https://btc.bloqapi.net/insight-api" + } else { + apiEndpoint = "https://test-insight.bitpay.com/api" + } + feeApi, _ := url.Parse("https://btc.fees.openbazaar.org") + clientApi, _ := url.Parse(apiEndpoint) + db, _ := mockDB.GetDatastoreForWallet(wallet.Bitcoin) + btcCfg := CoinConfig{ + CoinType: wallet.Bitcoin, + FeeAPI: *feeApi, + LowFee: 140, + MediumFee: 160, + HighFee: 180, + MaxFee: 2000, + ClientAPI: *clientApi, + DB: db, + } + cfg.Coins = append(cfg.Coins, btcCfg) + } + if coinTypes[wallet.BitcoinCash] { + var apiEndpoint string + if !testnet { + apiEndpoint = "https://bitcoincash.blockexplorer.com/api" + } else { + apiEndpoint = "https://test-bch-insight.bitpay.com/api" + } + clientApi, _ := url.Parse(apiEndpoint) + db, _ := mockDB.GetDatastoreForWallet(wallet.BitcoinCash) + bchCfg := CoinConfig{ + CoinType: wallet.BitcoinCash, + FeeAPI: url.URL{}, + LowFee: 140, + MediumFee: 160, + HighFee: 180, + MaxFee: 2000, + ClientAPI: *clientApi, + DB: db, + } + cfg.Coins = append(cfg.Coins, bchCfg) + } + if coinTypes[wallet.Zcash] { + var apiEndpoint string + if !testnet { + apiEndpoint = "https://zcashnetwork.info/api" + } else { + apiEndpoint = "https://explorer.testnet.z.cash/api" + } + clientApi, _ := url.Parse(apiEndpoint) + db, _ := mockDB.GetDatastoreForWallet(wallet.Zcash) + zecCfg := CoinConfig{ + CoinType: wallet.Zcash, + FeeAPI: url.URL{}, + LowFee: 140, + MediumFee: 160, + HighFee: 180, + MaxFee: 2000, + ClientAPI: *clientApi, + DB: db, + } + cfg.Coins = append(cfg.Coins, zecCfg) + } + if coinTypes[wallet.Litecoin] { + var apiEndpoint string + if !testnet { + apiEndpoint = "https://ltc.coin.space/api" + } else { + apiEndpoint = "https://testnet.litecore.io/api" + } + clientApi, _ := url.Parse(apiEndpoint) + db, _ := mockDB.GetDatastoreForWallet(wallet.Litecoin) + ltcCfg := CoinConfig{ + CoinType: wallet.Litecoin, + FeeAPI: url.URL{}, + LowFee: 140, + MediumFee: 160, + HighFee: 180, + MaxFee: 2000, + ClientAPI: *clientApi, + DB: db, + } + cfg.Coins = append(cfg.Coins, ltcCfg) + } + return cfg +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/datastore/mock.go b/vendor/github.com/OpenBazaar/multiwallet/datastore/mock.go new file mode 100644 index 0000000000..dff5cb7f89 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/datastore/mock.go @@ -0,0 +1,416 @@ +package datastore + +import ( + "bytes" + "encoding/hex" + "errors" + "sort" + "strconv" + "sync" + "time" + + "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/btcec" + "github.com/btcsuite/btcd/chaincfg/chainhash" +) + +type MockDatastore struct { + keys wallet.Keys + utxos wallet.Utxos + stxos wallet.Stxos + txns wallet.Txns + watchedScripts wallet.WatchedScripts +} + +type MockMultiwalletDatastore struct { + db map[wallet.CoinType]wallet.Datastore + sync.Mutex +} + +func (m *MockMultiwalletDatastore) GetDatastoreForWallet(coinType wallet.CoinType) (wallet.Datastore, error) { + m.Lock() + defer m.Unlock() + db, ok := m.db[coinType] + if !ok { + return nil, errors.New("Cointype not supported") + } + return db, nil +} + +func NewMockMultiwalletDatastore() *MockMultiwalletDatastore { + db := make(map[wallet.CoinType]wallet.Datastore) + db[wallet.Bitcoin] = wallet.Datastore(&MockDatastore{ + &MockKeyStore{Keys: make(map[string]*KeyStoreEntry)}, + &MockUtxoStore{utxos: make(map[string]*wallet.Utxo)}, + &MockStxoStore{stxos: make(map[string]*wallet.Stxo)}, + &MockTxnStore{txns: make(map[string]*txnStoreEntry)}, + &MockWatchedScriptsStore{scripts: make(map[string][]byte)}, + }) + db[wallet.BitcoinCash] = wallet.Datastore(&MockDatastore{ + &MockKeyStore{Keys: make(map[string]*KeyStoreEntry)}, + &MockUtxoStore{utxos: make(map[string]*wallet.Utxo)}, + &MockStxoStore{stxos: make(map[string]*wallet.Stxo)}, + &MockTxnStore{txns: make(map[string]*txnStoreEntry)}, + &MockWatchedScriptsStore{scripts: make(map[string][]byte)}, + }) + db[wallet.Zcash] = wallet.Datastore(&MockDatastore{ + &MockKeyStore{Keys: make(map[string]*KeyStoreEntry)}, + &MockUtxoStore{utxos: make(map[string]*wallet.Utxo)}, + &MockStxoStore{stxos: make(map[string]*wallet.Stxo)}, + &MockTxnStore{txns: make(map[string]*txnStoreEntry)}, + &MockWatchedScriptsStore{scripts: make(map[string][]byte)}, + }) + db[wallet.Litecoin] = wallet.Datastore(&MockDatastore{ + &MockKeyStore{Keys: make(map[string]*KeyStoreEntry)}, + &MockUtxoStore{utxos: make(map[string]*wallet.Utxo)}, + &MockStxoStore{stxos: make(map[string]*wallet.Stxo)}, + &MockTxnStore{txns: make(map[string]*txnStoreEntry)}, + &MockWatchedScriptsStore{scripts: make(map[string][]byte)}, + }) + return &MockMultiwalletDatastore{db: db} +} + +func (m *MockDatastore) Keys() wallet.Keys { + return m.keys +} + +func (m *MockDatastore) Utxos() wallet.Utxos { + return m.utxos +} + +func (m *MockDatastore) Stxos() wallet.Stxos { + return m.stxos +} + +func (m *MockDatastore) Txns() wallet.Txns { + return m.txns +} + +func (m *MockDatastore) WatchedScripts() wallet.WatchedScripts { + return m.watchedScripts +} + +type KeyStoreEntry struct { + ScriptAddress []byte + Path wallet.KeyPath + Used bool + Key *btcec.PrivateKey +} + +type MockKeyStore struct { + Keys map[string]*KeyStoreEntry + sync.Mutex +} + +func (m *MockKeyStore) Put(scriptAddress []byte, keyPath wallet.KeyPath) error { + m.Lock() + defer m.Unlock() + m.Keys[hex.EncodeToString(scriptAddress)] = &KeyStoreEntry{scriptAddress, keyPath, false, nil} + return nil +} + +func (m *MockKeyStore) ImportKey(scriptAddress []byte, key *btcec.PrivateKey) error { + m.Lock() + defer m.Unlock() + kp := wallet.KeyPath{Purpose: wallet.EXTERNAL, Index: -1} + m.Keys[hex.EncodeToString(scriptAddress)] = &KeyStoreEntry{scriptAddress, kp, false, key} + return nil +} + +func (m *MockKeyStore) MarkKeyAsUsed(scriptAddress []byte) error { + m.Lock() + defer m.Unlock() + key, ok := m.Keys[hex.EncodeToString(scriptAddress)] + if !ok { + return errors.New("key does not exist") + } + key.Used = true + return nil +} + +func (m *MockKeyStore) GetLastKeyIndex(purpose wallet.KeyPurpose) (int, bool, error) { + m.Lock() + defer m.Unlock() + i := -1 + used := false + for _, key := range m.Keys { + if key.Path.Purpose == purpose && key.Path.Index > i { + i = key.Path.Index + used = key.Used + } + } + if i == -1 { + return i, used, errors.New("No saved keys") + } + return i, used, nil +} + +func (m *MockKeyStore) GetPathForKey(scriptAddress []byte) (wallet.KeyPath, error) { + m.Lock() + defer m.Unlock() + key, ok := m.Keys[hex.EncodeToString(scriptAddress)] + if !ok || key.Path.Index == -1 { + return wallet.KeyPath{}, errors.New("key does not exist") + } + return key.Path, nil +} + +func (m *MockKeyStore) GetKey(scriptAddress []byte) (*btcec.PrivateKey, error) { + m.Lock() + defer m.Unlock() + for _, k := range m.Keys { + if k.Path.Index == -1 && bytes.Equal(scriptAddress, k.ScriptAddress) { + return k.Key, nil + } + } + return nil, errors.New("Not found") +} + +func (m *MockKeyStore) GetImported() ([]*btcec.PrivateKey, error) { + m.Lock() + defer m.Unlock() + var keys []*btcec.PrivateKey + for _, k := range m.Keys { + if k.Path.Index == -1 { + keys = append(keys, k.Key) + } + } + return keys, nil +} + +func (m *MockKeyStore) GetUnused(purpose wallet.KeyPurpose) ([]int, error) { + m.Lock() + defer m.Unlock() + var i []int + for _, key := range m.Keys { + if !key.Used && key.Path.Purpose == purpose { + i = append(i, key.Path.Index) + } + } + sort.Ints(i) + return i, nil +} + +func (m *MockKeyStore) GetAll() ([]wallet.KeyPath, error) { + m.Lock() + defer m.Unlock() + var kp []wallet.KeyPath + for _, key := range m.Keys { + kp = append(kp, key.Path) + } + return kp, nil +} + +func (m *MockKeyStore) GetLookaheadWindows() map[wallet.KeyPurpose]int { + m.Lock() + defer m.Unlock() + internalLastUsed := -1 + externalLastUsed := -1 + for _, key := range m.Keys { + if key.Path.Purpose == wallet.INTERNAL && key.Used && key.Path.Index > internalLastUsed { + internalLastUsed = key.Path.Index + } + if key.Path.Purpose == wallet.EXTERNAL && key.Used && key.Path.Index > externalLastUsed { + externalLastUsed = key.Path.Index + } + } + internalUnused := 0 + externalUnused := 0 + for _, key := range m.Keys { + if key.Path.Purpose == wallet.INTERNAL && !key.Used && key.Path.Index > internalLastUsed { + internalUnused++ + } + if key.Path.Purpose == wallet.EXTERNAL && !key.Used && key.Path.Index > externalLastUsed { + externalUnused++ + } + } + mp := make(map[wallet.KeyPurpose]int) + mp[wallet.INTERNAL] = internalUnused + mp[wallet.EXTERNAL] = externalUnused + return mp +} + +type MockUtxoStore struct { + utxos map[string]*wallet.Utxo + sync.Mutex +} + +func (m *MockUtxoStore) Put(utxo wallet.Utxo) error { + m.Lock() + defer m.Unlock() + key := utxo.Op.Hash.String() + ":" + strconv.Itoa(int(utxo.Op.Index)) + m.utxos[key] = &utxo + return nil +} + +func (m *MockUtxoStore) GetAll() ([]wallet.Utxo, error) { + m.Lock() + defer m.Unlock() + var utxos []wallet.Utxo + for _, v := range m.utxos { + utxos = append(utxos, *v) + } + return utxos, nil +} + +func (m *MockUtxoStore) SetWatchOnly(utxo wallet.Utxo) error { + m.Lock() + defer m.Unlock() + key := utxo.Op.Hash.String() + ":" + strconv.Itoa(int(utxo.Op.Index)) + u, ok := m.utxos[key] + if !ok { + return errors.New("Not found") + } + u.WatchOnly = true + return nil +} + +func (m *MockUtxoStore) Delete(utxo wallet.Utxo) error { + m.Lock() + defer m.Unlock() + key := utxo.Op.Hash.String() + ":" + strconv.Itoa(int(utxo.Op.Index)) + _, ok := m.utxos[key] + if !ok { + return errors.New("Not found") + } + delete(m.utxos, key) + return nil +} + +type MockStxoStore struct { + stxos map[string]*wallet.Stxo + sync.Mutex +} + +func (m *MockStxoStore) Put(stxo wallet.Stxo) error { + m.Lock() + defer m.Unlock() + m.stxos[stxo.SpendTxid.String()] = &stxo + return nil +} + +func (m *MockStxoStore) GetAll() ([]wallet.Stxo, error) { + m.Lock() + defer m.Unlock() + var stxos []wallet.Stxo + for _, v := range m.stxos { + stxos = append(stxos, *v) + } + return stxos, nil +} + +func (m *MockStxoStore) Delete(stxo wallet.Stxo) error { + m.Lock() + defer m.Unlock() + _, ok := m.stxos[stxo.SpendTxid.String()] + if !ok { + return errors.New("Not found") + } + delete(m.stxos, stxo.SpendTxid.String()) + return nil +} + +type txnStoreEntry struct { + txn []byte + value int + height int + timestamp time.Time + watchOnly bool +} + +type MockTxnStore struct { + txns map[string]*txnStoreEntry + sync.Mutex +} + +func (m *MockTxnStore) Put(tx []byte, txid string, value, height int, timestamp time.Time, watchOnly bool) error { + m.Lock() + defer m.Unlock() + m.txns[txid] = &txnStoreEntry{ + txn: tx, + value: value, + height: height, + timestamp: timestamp, + watchOnly: watchOnly, + } + return nil +} + +func (m *MockTxnStore) Get(txid chainhash.Hash) (wallet.Txn, error) { + m.Lock() + defer m.Unlock() + t, ok := m.txns[txid.String()] + if !ok { + return wallet.Txn{}, errors.New("Not found") + } + return wallet.Txn{txid.String(), int64(t.value), int32(t.height), t.timestamp, t.watchOnly, t.txn}, nil +} + +func (m *MockTxnStore) GetAll(includeWatchOnly bool) ([]wallet.Txn, error) { + m.Lock() + defer m.Unlock() + var txns []wallet.Txn + for txid, t := range m.txns { + txn := wallet.Txn{txid, int64(t.value), int32(t.height), t.timestamp, t.watchOnly, t.txn} + txns = append(txns, txn) + } + return txns, nil +} + +func (m *MockTxnStore) UpdateHeight(txid chainhash.Hash, height int, timestamp time.Time) error { + m.Lock() + defer m.Unlock() + txn, ok := m.txns[txid.String()] + if !ok { + return errors.New("Not found") + } + txn.height = height + txn.timestamp = timestamp + m.txns[txid.String()] = txn + return nil +} + +func (m *MockTxnStore) Delete(txid *chainhash.Hash) error { + m.Lock() + defer m.Unlock() + _, ok := m.txns[txid.String()] + if !ok { + return errors.New("Not found") + } + delete(m.txns, txid.String()) + return nil +} + +type MockWatchedScriptsStore struct { + scripts map[string][]byte + sync.Mutex +} + +func (m *MockWatchedScriptsStore) Put(scriptPubKey []byte) error { + m.Lock() + defer m.Unlock() + m.scripts[hex.EncodeToString(scriptPubKey)] = scriptPubKey + return nil +} + +func (m *MockWatchedScriptsStore) GetAll() ([][]byte, error) { + m.Lock() + defer m.Unlock() + var ret [][]byte + for _, b := range m.scripts { + ret = append(ret, b) + } + return ret, nil +} + +func (m *MockWatchedScriptsStore) Delete(scriptPubKey []byte) error { + m.Lock() + defer m.Unlock() + enc := hex.EncodeToString(scriptPubKey) + _, ok := m.scripts[enc] + if !ok { + return errors.New("Not found") + } + delete(m.scripts, enc) + return nil +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/keys/keys.go b/vendor/github.com/OpenBazaar/multiwallet/keys/keys.go new file mode 100644 index 0000000000..f902b643fa --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/keys/keys.go @@ -0,0 +1,186 @@ +package keys + +import ( + "errors" + "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcutil" + hd "github.com/btcsuite/btcutil/hdkeychain" +) + +const LOOKAHEADWINDOW = 20 + +type KeyManager struct { + datastore wallet.Keys + params *chaincfg.Params + + internalKey *hd.ExtendedKey + externalKey *hd.ExtendedKey + + coinType wallet.CoinType + getAddr AddrFunc +} + +type AddrFunc func(k *hd.ExtendedKey, net *chaincfg.Params) (btcutil.Address, error) + +func NewKeyManager(db wallet.Keys, params *chaincfg.Params, masterPrivKey *hd.ExtendedKey, coinType wallet.CoinType, getAddr AddrFunc) (*KeyManager, error) { + internal, external, err := Bip44Derivation(masterPrivKey, coinType) + if err != nil { + return nil, err + } + km := &KeyManager{ + datastore: db, + params: params, + internalKey: internal, + externalKey: external, + coinType: coinType, + getAddr: getAddr, + } + if err := km.lookahead(); err != nil { + return nil, err + } + return km, nil +} + +// m / purpose' / coin_type' / account' / change / address_index +func Bip44Derivation(masterPrivKey *hd.ExtendedKey, coinType wallet.CoinType) (internal, external *hd.ExtendedKey, err error) { + // Purpose = bip44 + fourtyFour, err := masterPrivKey.Child(hd.HardenedKeyStart + 44) + if err != nil { + return nil, nil, err + } + // Cointype + bitcoin, err := fourtyFour.Child(hd.HardenedKeyStart + uint32(coinType)) + if err != nil { + return nil, nil, err + } + // Account = 0 + account, err := bitcoin.Child(hd.HardenedKeyStart + 0) + if err != nil { + return nil, nil, err + } + // Change(0) = external + external, err = account.Child(0) + if err != nil { + return nil, nil, err + } + // Change(1) = internal + internal, err = account.Child(1) + if err != nil { + return nil, nil, err + } + return internal, external, nil +} + +func (km *KeyManager) GetCurrentKey(purpose wallet.KeyPurpose) (*hd.ExtendedKey, error) { + i, err := km.datastore.GetUnused(purpose) + if err != nil { + return nil, err + } + if len(i) == 0 { + return nil, errors.New("No unused keys in database") + } + return km.GenerateChildKey(purpose, uint32(i[0])) +} + +func (km *KeyManager) GetFreshKey(purpose wallet.KeyPurpose) (*hd.ExtendedKey, error) { + index, _, err := km.datastore.GetLastKeyIndex(purpose) + var childKey *hd.ExtendedKey + if err != nil { + index = 0 + } else { + index += 1 + } + for { + // There is a small possibility bip32 keys can be invalid. The procedure in such cases + // is to discard the key and derive the next one. This loop will continue until a valid key + // is derived. + childKey, err = km.GenerateChildKey(purpose, uint32(index)) + if err == nil { + break + } + index += 1 + } + addr, err := km.KeyToAddress(childKey) + if err != nil { + return nil, err + } + p := wallet.KeyPath{wallet.KeyPurpose(purpose), index} + err = km.datastore.Put(addr.ScriptAddress(), p) + if err != nil { + return nil, err + } + return childKey, nil +} + +func (km *KeyManager) GetKeys() []*hd.ExtendedKey { + var keys []*hd.ExtendedKey + keyPaths, err := km.datastore.GetAll() + if err != nil { + return keys + } + for _, path := range keyPaths { + k, err := km.GenerateChildKey(path.Purpose, uint32(path.Index)) + if err != nil { + continue + } + keys = append(keys, k) + } + return keys +} + +func (km *KeyManager) GetKeyForScript(scriptAddress []byte) (*hd.ExtendedKey, error) { + keyPath, err := km.datastore.GetPathForKey(scriptAddress) + if err != nil { + key, err := km.datastore.GetKey(scriptAddress) + if err != nil { + return nil, err + } + hdKey := hd.NewExtendedKey( + km.params.HDPrivateKeyID[:], + key.Serialize(), + make([]byte, 32), + []byte{0x00, 0x00, 0x00, 0x00}, + 0, + 0, + true) + return hdKey, nil + } + return km.GenerateChildKey(keyPath.Purpose, uint32(keyPath.Index)) +} + +// Mark the given key as used and extend the lookahead window +func (km *KeyManager) MarkKeyAsUsed(scriptAddress []byte) error { + if err := km.datastore.MarkKeyAsUsed(scriptAddress); err != nil { + return err + } + return km.lookahead() +} + +func (km *KeyManager) GenerateChildKey(purpose wallet.KeyPurpose, index uint32) (*hd.ExtendedKey, error) { + if purpose == wallet.EXTERNAL { + return km.externalKey.Child(index) + } else if purpose == wallet.INTERNAL { + return km.internalKey.Child(index) + } + return nil, errors.New("Unknown key purpose") +} + +func (km *KeyManager) lookahead() error { + lookaheadWindows := km.datastore.GetLookaheadWindows() + for purpose, size := range lookaheadWindows { + if size < LOOKAHEADWINDOW { + for i := 0; i < (LOOKAHEADWINDOW - size); i++ { + _, err := km.GetFreshKey(purpose) + if err != nil { + return err + } + } + } + } + return nil +} + +func (km *KeyManager) KeyToAddress(key *hd.ExtendedKey) (btcutil.Address, error) { + return km.getAddr(key, km.params) +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/litecoin/address/address.go b/vendor/github.com/OpenBazaar/multiwallet/litecoin/address/address.go new file mode 100644 index 0000000000..a982c3fa7f --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/litecoin/address/address.go @@ -0,0 +1,757 @@ +package address + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "strings" + + lparams "github.com/OpenBazaar/multiwallet/litecoin/params" + "github.com/btcsuite/btcd/btcec" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcutil/bech32" + "github.com/btcsuite/golangcrypto/ripemd160" + ltcparams "github.com/ltcsuite/ltcd/chaincfg" + "github.com/ltcsuite/ltcutil/base58" +) + +// UnsupportedWitnessVerError describes an error where a segwit address being +// decoded has an unsupported witness version. +type UnsupportedWitnessVerError byte + +func (e UnsupportedWitnessVerError) Error() string { + return "unsupported witness version: " + string(e) +} + +// UnsupportedWitnessProgLenError describes an error where a segwit address +// being decoded has an unsupported witness program length. +type UnsupportedWitnessProgLenError int + +func (e UnsupportedWitnessProgLenError) Error() string { + return "unsupported witness program length: " + string(e) +} + +var ( + // ErrChecksumMismatch describes an error where decoding failed due + // to a bad checksum. + ErrChecksumMismatch = errors.New("checksum mismatch") + + // ErrUnknownAddressType describes an error where an address can not + // decoded as a specific address type due to the string encoding + // begining with an identifier byte unknown to any standard or + // registered (via chaincfg.Register) network. + ErrUnknownAddressType = errors.New("unknown address type") + + // ErrAddressCollision describes an error where an address can not + // be uniquely determined as either a pay-to-pubkey-hash or + // pay-to-script-hash address since the leading identifier is used for + // describing both address kinds, but for different networks. Rather + // than assuming or defaulting to one or the other, this error is + // returned and the caller must decide how to decode the address. + ErrAddressCollision = errors.New("address collision") +) + +// encodeAddress returns a human-readable payment address given a ripemd160 hash +// and netID which encodes the litecoin network and address type. It is used +// in both pay-to-pubkey-hash (P2PKH) and pay-to-script-hash (P2SH) address +// encoding. +func encodeAddress(hash160 []byte, netID byte) string { + // Format is 1 byte for a network and address class (i.e. P2PKH vs + // P2SH), 20 bytes for a RIPEMD160 hash, and 4 bytes of checksum. + return base58.CheckEncode(hash160[:ripemd160.Size], netID) +} + +// encodeSegWitAddress creates a bech32 encoded address string representation +// from witness version and witness program. +func encodeSegWitAddress(hrp string, witnessVersion byte, witnessProgram []byte) (string, error) { + // Group the address bytes into 5 bit groups, as this is what is used to + // encode each character in the address string. + converted, err := bech32.ConvertBits(witnessProgram, 8, 5, true) + if err != nil { + return "", err + } + + // Concatenate the witness version and program, and encode the resulting + // bytes using bech32 encoding. + combined := make([]byte, len(converted)+1) + combined[0] = witnessVersion + copy(combined[1:], converted) + bech, err := bech32.Encode(hrp, combined) + if err != nil { + return "", err + } + + // Check validity by decoding the created address. + version, program, err := decodeSegWitAddress(bech) + if err != nil { + return "", fmt.Errorf("invalid segwit address: %v", err) + } + + if version != witnessVersion || !bytes.Equal(program, witnessProgram) { + return "", fmt.Errorf("invalid segwit address") + } + + return bech, nil +} + +// Address is an interface type for any type of destination a transaction +// output may spend to. This includes pay-to-pubkey (P2PK), pay-to-pubkey-hash +// (P2PKH), and pay-to-script-hash (P2SH). Address is designed to be generic +// enough that other kinds of addresses may be added in the future without +// changing the decoding and encoding API. +type Address interface { + // String returns the string encoding of the transaction output + // destination. + // + // Please note that String differs subtly from EncodeAddress: String + // will return the value as a string without any conversion, while + // EncodeAddress may convert destination types (for example, + // converting pubkeys to P2PKH addresses) before encoding as a + // payment address string. + String() string + + // EncodeAddress returns the string encoding of the payment address + // associated with the Address value. See the comment on String + // for how this method differs from String. + EncodeAddress() string + + // ScriptAddress returns the raw bytes of the address to be used + // when inserting the address into a txout's script. + ScriptAddress() []byte + + // IsForNet returns whether or not the address is associated with the + // passed litecoin network. + IsForNet(*chaincfg.Params) bool +} + +// DecodeAddress decodes the string encoding of an address and returns +// the Address if addr is a valid encoding for a known address type. +// +// The litecoin network the address is associated with is extracted if possible. +// When the address does not encode the network, such as in the case of a raw +// public key, the address will be associated with the passed defaultNet. +func DecodeAddress(addr string, defaultNet *chaincfg.Params) (Address, error) { + // Bech32 encoded segwit addresses start with a human-readable part + // (hrp) followed by '1'. For Bitcoin mainnet the hrp is "bc", and for + // testnet it is "tb". If the address string has a prefix that matches + // one of the prefixes for the known networks, we try to decode it as + // a segwit address. + oneIndex := strings.LastIndexByte(addr, '1') + if oneIndex > 1 { + prefix := addr[:oneIndex+1] + if IsBech32SegwitPrefix(prefix) { + witnessVer, witnessProg, err := decodeSegWitAddress(addr) + if err != nil { + return nil, err + } + + // We currently only support P2WPKH and P2WSH, which is + // witness version 0. + if witnessVer != 0 { + return nil, UnsupportedWitnessVerError(witnessVer) + } + + // The HRP is everything before the found '1'. + hrp := prefix[:len(prefix)-1] + + switch len(witnessProg) { + case 20: + return newAddressWitnessPubKeyHash(hrp, witnessProg) + case 32: + return newAddressWitnessScriptHash(hrp, witnessProg) + default: + return nil, UnsupportedWitnessProgLenError(len(witnessProg)) + } + } + } + + // Serialized public keys are either 65 bytes (130 hex chars) if + // uncompressed/hybrid or 33 bytes (66 hex chars) if compressed. + if len(addr) == 130 || len(addr) == 66 { + serializedPubKey, err := hex.DecodeString(addr) + if err != nil { + return nil, err + } + return NewAddressPubKey(serializedPubKey, defaultNet) + } + + // Switch on decoded length to determine the type. + decoded, netID, err := base58.CheckDecode(addr) + if err != nil { + if err == base58.ErrChecksum { + return nil, ErrChecksumMismatch + } + return nil, errors.New("decoded address is of unknown format") + } + switch len(decoded) { + case ripemd160.Size: // P2PKH or P2SH + isP2PKH := ltcparams.IsPubKeyHashAddrID(netID) + isP2SH := ltcparams.IsScriptHashAddrID(netID) + switch hash160 := decoded; { + case isP2PKH && isP2SH: + return nil, ErrAddressCollision + case isP2PKH: + return newAddressPubKeyHash(hash160, netID) + case isP2SH: + return newAddressScriptHashFromHash(hash160, netID) + default: + return nil, ErrUnknownAddressType + } + + default: + return nil, errors.New("decoded address is of unknown size") + } +} + +// IsBech32SegwitPrefix returns whether the prefix is a known prefix for segwit +// addresses on any default or registered network. This is used when decoding +// an address string into a specific address type. +func IsBech32SegwitPrefix(prefix string) bool { + prefix = strings.ToLower(prefix) + if prefix == "ltc1" { + return true + } + return false +} + +// decodeSegWitAddress parses a bech32 encoded segwit address string and +// returns the witness version and witness program byte representation. +func decodeSegWitAddress(address string) (byte, []byte, error) { + // Decode the bech32 encoded address. + _, data, err := bech32.Decode(address) + if err != nil { + return 0, nil, err + } + + // The first byte of the decoded address is the witness version, it must + // exist. + if len(data) < 1 { + return 0, nil, fmt.Errorf("no witness version") + } + + // ...and be <= 16. + version := data[0] + if version > 16 { + return 0, nil, fmt.Errorf("invalid witness version: %v", version) + } + + // The remaining characters of the address returned are grouped into + // words of 5 bits. In order to restore the original witness program + // bytes, we'll need to regroup into 8 bit words. + regrouped, err := bech32.ConvertBits(data[1:], 5, 8, false) + if err != nil { + return 0, nil, err + } + + // The regrouped data must be between 2 and 40 bytes. + if len(regrouped) < 2 || len(regrouped) > 40 { + return 0, nil, fmt.Errorf("invalid data length") + } + + // For witness version 0, address MUST be exactly 20 or 32 bytes. + if version == 0 && len(regrouped) != 20 && len(regrouped) != 32 { + return 0, nil, fmt.Errorf("invalid data length for witness "+ + "version 0: %v", len(regrouped)) + } + + return version, regrouped, nil +} + +// AddressPubKeyHash is an Address for a pay-to-pubkey-hash (P2PKH) +// transaction. +type AddressPubKeyHash struct { + hash [ripemd160.Size]byte + netID byte +} + +// NewAddressPubKeyHash returns a new AddressPubKeyHash. pkHash mustbe 20 +// bytes. +func NewAddressPubKeyHash(pkHash []byte, net *chaincfg.Params) (*AddressPubKeyHash, error) { + params := lparams.ConvertParams(net) + return newAddressPubKeyHash(pkHash, params.PubKeyHashAddrID) +} + +// newAddressPubKeyHash is the internal API to create a pubkey hash address +// with a known leading identifier byte for a network, rather than looking +// it up through its parameters. This is useful when creating a new address +// structure from a string encoding where the identifer byte is already +// known. +func newAddressPubKeyHash(pkHash []byte, netID byte) (*AddressPubKeyHash, error) { + // Check for a valid pubkey hash length. + if len(pkHash) != ripemd160.Size { + return nil, errors.New("pkHash must be 20 bytes") + } + + addr := &AddressPubKeyHash{netID: netID} + copy(addr.hash[:], pkHash) + return addr, nil +} + +// EncodeAddress returns the string encoding of a pay-to-pubkey-hash +// address. Part of the Address interface. +func (a *AddressPubKeyHash) EncodeAddress() string { + return encodeAddress(a.hash[:], a.netID) +} + +// ScriptAddress returns the bytes to be included in a txout script to pay +// to a pubkey hash. Part of the Address interface. +func (a *AddressPubKeyHash) ScriptAddress() []byte { + return a.hash[:] +} + +// IsForNet returns whether or not the pay-to-pubkey-hash address is associated +// with the passed litecoin network. +func (a *AddressPubKeyHash) IsForNet(net *chaincfg.Params) bool { + params := lparams.ConvertParams(net) + return a.netID == params.PubKeyHashAddrID +} + +// String returns a human-readable string for the pay-to-pubkey-hash address. +// This is equivalent to calling EncodeAddress, but is provided so the type can +// be used as a fmt.Stringer. +func (a *AddressPubKeyHash) String() string { + return a.EncodeAddress() +} + +// Hash160 returns the underlying array of the pubkey hash. This can be useful +// when an array is more appropiate than a slice (for example, when used as map +// keys). +func (a *AddressPubKeyHash) Hash160() *[ripemd160.Size]byte { + return &a.hash +} + +// AddressScriptHash is an Address for a pay-to-script-hash (P2SH) +// transaction. +type AddressScriptHash struct { + hash [ripemd160.Size]byte + netID byte +} + +// NewAddressScriptHash returns a new AddressScriptHash. +func NewAddressScriptHash(serializedScript []byte, net *chaincfg.Params) (*AddressScriptHash, error) { + scriptHash := btcutil.Hash160(serializedScript) + params := lparams.ConvertParams(net) + return newAddressScriptHashFromHash(scriptHash, params.ScriptHashAddrID) +} + +// NewAddressScriptHashFromHash returns a new AddressScriptHash. scriptHash +// must be 20 bytes. +func NewAddressScriptHashFromHash(scriptHash []byte, net *chaincfg.Params) (*AddressScriptHash, error) { + params := lparams.ConvertParams(net) + return newAddressScriptHashFromHash(scriptHash, params.ScriptHashAddrID) +} + +// newAddressScriptHashFromHash is the internal API to create a script hash +// address with a known leading identifier byte for a network, rather than +// looking it up through its parameters. This is useful when creating a new +// address structure from a string encoding where the identifer byte is already +// known. +func newAddressScriptHashFromHash(scriptHash []byte, netID byte) (*AddressScriptHash, error) { + // Check for a valid script hash length. + if len(scriptHash) != ripemd160.Size { + return nil, errors.New("scriptHash must be 20 bytes") + } + + addr := &AddressScriptHash{netID: netID} + copy(addr.hash[:], scriptHash) + return addr, nil +} + +// EncodeAddress returns the string encoding of a pay-to-script-hash +// address. Part of the Address interface. +func (a *AddressScriptHash) EncodeAddress() string { + return encodeAddress(a.hash[:], a.netID) +} + +// ScriptAddress returns the bytes to be included in a txout script to pay +// to a script hash. Part of the Address interface. +func (a *AddressScriptHash) ScriptAddress() []byte { + return a.hash[:] +} + +// IsForNet returns whether or not the pay-to-script-hash address is associated +// with the passed litecoin network. +func (a *AddressScriptHash) IsForNet(net *chaincfg.Params) bool { + params := lparams.ConvertParams(net) + return a.netID == params.ScriptHashAddrID +} + +// String returns a human-readable string for the pay-to-script-hash address. +// This is equivalent to calling EncodeAddress, but is provided so the type can +// be used as a fmt.Stringer. +func (a *AddressScriptHash) String() string { + return a.EncodeAddress() +} + +// Hash160 returns the underlying array of the script hash. This can be useful +// when an array is more appropiate than a slice (for example, when used as map +// keys). +func (a *AddressScriptHash) Hash160() *[ripemd160.Size]byte { + return &a.hash +} + +// PubKeyFormat describes what format to use for a pay-to-pubkey address. +type PubKeyFormat int + +const ( + // PKFUncompressed indicates the pay-to-pubkey address format is an + // uncompressed public key. + PKFUncompressed PubKeyFormat = iota + + // PKFCompressed indicates the pay-to-pubkey address format is a + // compressed public key. + PKFCompressed + + // PKFHybrid indicates the pay-to-pubkey address format is a hybrid + // public key. + PKFHybrid +) + +// AddressPubKey is an Address for a pay-to-pubkey transaction. +type AddressPubKey struct { + pubKeyFormat PubKeyFormat + pubKey *btcec.PublicKey + pubKeyHashID byte +} + +// NewAddressPubKey returns a new AddressPubKey which represents a pay-to-pubkey +// address. The serializedPubKey parameter must be a valid pubkey and can be +// uncompressed, compressed, or hybrid. +func NewAddressPubKey(serializedPubKey []byte, net *chaincfg.Params) (*AddressPubKey, error) { + pubKey, err := btcec.ParsePubKey(serializedPubKey, btcec.S256()) + if err != nil { + return nil, err + } + + // Set the format of the pubkey. This probably should be returned + // from btcec, but do it here to avoid API churn. We already know the + // pubkey is valid since it parsed above, so it's safe to simply examine + // the leading byte to get the format. + pkFormat := PKFUncompressed + switch serializedPubKey[0] { + case 0x02, 0x03: + pkFormat = PKFCompressed + case 0x06, 0x07: + pkFormat = PKFHybrid + } + params := lparams.ConvertParams(net) + + return &AddressPubKey{ + pubKeyFormat: pkFormat, + pubKey: pubKey, + pubKeyHashID: params.PubKeyHashAddrID, + }, nil +} + +// serialize returns the serialization of the public key according to the +// format associated with the address. +func (a *AddressPubKey) serialize() []byte { + switch a.pubKeyFormat { + default: + fallthrough + case PKFUncompressed: + return a.pubKey.SerializeUncompressed() + + case PKFCompressed: + return a.pubKey.SerializeCompressed() + + case PKFHybrid: + return a.pubKey.SerializeHybrid() + } +} + +// EncodeAddress returns the string encoding of the public key as a +// pay-to-pubkey-hash. Note that the public key format (uncompressed, +// compressed, etc) will change the resulting address. This is expected since +// pay-to-pubkey-hash is a hash of the serialized public key which obviously +// differs with the format. At the time of this writing, most Bitcoin addresses +// are pay-to-pubkey-hash constructed from the uncompressed public key. +// +// Part of the Address interface. +func (a *AddressPubKey) EncodeAddress() string { + return encodeAddress(btcutil.Hash160(a.serialize()), a.pubKeyHashID) +} + +// ScriptAddress returns the bytes to be included in a txout script to pay +// to a public key. Setting the public key format will affect the output of +// this function accordingly. Part of the Address interface. +func (a *AddressPubKey) ScriptAddress() []byte { + return a.serialize() +} + +// IsForNet returns whether or not the pay-to-pubkey address is associated +// with the passed litecoin network. +func (a *AddressPubKey) IsForNet(net *chaincfg.Params) bool { + params := lparams.ConvertParams(net) + return a.pubKeyHashID == params.PubKeyHashAddrID +} + +// String returns the hex-encoded human-readable string for the pay-to-pubkey +// address. This is not the same as calling EncodeAddress. +func (a *AddressPubKey) String() string { + return hex.EncodeToString(a.serialize()) +} + +// Format returns the format (uncompressed, compressed, etc) of the +// pay-to-pubkey address. +func (a *AddressPubKey) Format() PubKeyFormat { + return a.pubKeyFormat +} + +// SetFormat sets the format (uncompressed, compressed, etc) of the +// pay-to-pubkey address. +func (a *AddressPubKey) SetFormat(pkFormat PubKeyFormat) { + a.pubKeyFormat = pkFormat +} + +// AddressPubKeyHash returns the pay-to-pubkey address converted to a +// pay-to-pubkey-hash address. Note that the public key format (uncompressed, +// compressed, etc) will change the resulting address. This is expected since +// pay-to-pubkey-hash is a hash of the serialized public key which obviously +// differs with the format. At the time of this writing, most Bitcoin addresses +// are pay-to-pubkey-hash constructed from the uncompressed public key. +func (a *AddressPubKey) AddressPubKeyHash() *AddressPubKeyHash { + addr := &AddressPubKeyHash{netID: a.pubKeyHashID} + copy(addr.hash[:], btcutil.Hash160(a.serialize())) + return addr +} + +// PubKey returns the underlying public key for the address. +func (a *AddressPubKey) PubKey() *btcec.PublicKey { + return a.pubKey +} + +// AddressWitnessPubKeyHash is an Address for a pay-to-witness-pubkey-hash +// (P2WPKH) output. See BIP 173 for further details regarding native segregated +// witness address encoding: +// https://github.com/litecoin/bips/blob/master/bip-0173.mediawiki +type AddressWitnessPubKeyHash struct { + hrp string + witnessVersion byte + witnessProgram [20]byte +} + +// NewAddressWitnessPubKeyHash returns a new AddressWitnessPubKeyHash. +func NewAddressWitnessPubKeyHash(witnessProg []byte, net *chaincfg.Params) (*AddressWitnessPubKeyHash, error) { + params := lparams.ConvertParams(net) + return newAddressWitnessPubKeyHash(params.Bech32HRPSegwit, witnessProg) +} + +// newAddressWitnessPubKeyHash is an internal helper function to create an +// AddressWitnessPubKeyHash with a known human-readable part, rather than +// looking it up through its parameters. +func newAddressWitnessPubKeyHash(hrp string, witnessProg []byte) (*AddressWitnessPubKeyHash, error) { + // Check for valid program length for witness version 0, which is 20 + // for P2WPKH. + if len(witnessProg) != 20 { + return nil, errors.New("witness program must be 20 " + + "bytes for p2wpkh") + } + + addr := &AddressWitnessPubKeyHash{ + hrp: strings.ToLower(hrp), + witnessVersion: 0x00, + } + + copy(addr.witnessProgram[:], witnessProg) + + return addr, nil +} + +// EncodeAddress returns the bech32 string encoding of an +// AddressWitnessPubKeyHash. +// Part of the Address interface. +func (a *AddressWitnessPubKeyHash) EncodeAddress() string { + str, err := encodeSegWitAddress(a.hrp, a.witnessVersion, + a.witnessProgram[:]) + if err != nil { + return "" + } + return str +} + +// ScriptAddress returns the witness program for this address. +// Part of the Address interface. +func (a *AddressWitnessPubKeyHash) ScriptAddress() []byte { + return a.witnessProgram[:] +} + +// IsForNet returns whether or not the AddressWitnessPubKeyHash is associated +// with the passed litecoin network. +// Part of the Address interface. +func (a *AddressWitnessPubKeyHash) IsForNet(net *chaincfg.Params) bool { + params := lparams.ConvertParams(net) + return a.hrp == params.Bech32HRPSegwit +} + +// String returns a human-readable string for the AddressWitnessPubKeyHash. +// This is equivalent to calling EncodeAddress, but is provided so the type +// can be used as a fmt.Stringer. +// Part of the Address interface. +func (a *AddressWitnessPubKeyHash) String() string { + return a.EncodeAddress() +} + +// Hrp returns the human-readable part of the bech32 encoded +// AddressWitnessPubKeyHash. +func (a *AddressWitnessPubKeyHash) Hrp() string { + return a.hrp +} + +// WitnessVersion returns the witness version of the AddressWitnessPubKeyHash. +func (a *AddressWitnessPubKeyHash) WitnessVersion() byte { + return a.witnessVersion +} + +// WitnessProgram returns the witness program of the AddressWitnessPubKeyHash. +func (a *AddressWitnessPubKeyHash) WitnessProgram() []byte { + return a.witnessProgram[:] +} + +// Hash160 returns the witness program of the AddressWitnessPubKeyHash as a +// byte array. +func (a *AddressWitnessPubKeyHash) Hash160() *[20]byte { + return &a.witnessProgram +} + +// AddressWitnessScriptHash is an Address for a pay-to-witness-script-hash +// (P2WSH) output. See BIP 173 for further details regarding native segregated +// witness address encoding: +// https://github.com/litecoin/bips/blob/master/bip-0173.mediawiki +type AddressWitnessScriptHash struct { + hrp string + witnessVersion byte + witnessProgram [32]byte +} + +// NewAddressWitnessScriptHash returns a new AddressWitnessPubKeyHash. +func NewAddressWitnessScriptHash(witnessProg []byte, net *chaincfg.Params) (*AddressWitnessScriptHash, error) { + params := lparams.ConvertParams(net) + return newAddressWitnessScriptHash(params.Bech32HRPSegwit, witnessProg) +} + +// newAddressWitnessScriptHash is an internal helper function to create an +// AddressWitnessScriptHash with a known human-readable part, rather than +// looking it up through its parameters. +func newAddressWitnessScriptHash(hrp string, witnessProg []byte) (*AddressWitnessScriptHash, error) { + // Check for valid program length for witness version 0, which is 32 + // for P2WSH. + if len(witnessProg) != 32 { + return nil, errors.New("witness program must be 32 " + + "bytes for p2wsh") + } + + addr := &AddressWitnessScriptHash{ + hrp: strings.ToLower(hrp), + witnessVersion: 0x00, + } + + copy(addr.witnessProgram[:], witnessProg) + + return addr, nil +} + +// EncodeAddress returns the bech32 string encoding of an +// AddressWitnessScriptHash. +// Part of the Address interface. +func (a *AddressWitnessScriptHash) EncodeAddress() string { + str, err := encodeSegWitAddress(a.hrp, a.witnessVersion, + a.witnessProgram[:]) + if err != nil { + return "" + } + return str +} + +// ScriptAddress returns the witness program for this address. +// Part of the Address interface. +func (a *AddressWitnessScriptHash) ScriptAddress() []byte { + return a.witnessProgram[:] +} + +// IsForNet returns whether or not the AddressWitnessScriptHash is associated +// with the passed litecoin network. +// Part of the Address interface. +func (a *AddressWitnessScriptHash) IsForNet(net *chaincfg.Params) bool { + params := lparams.ConvertParams(net) + return a.hrp == params.Bech32HRPSegwit +} + +// String returns a human-readable string for the AddressWitnessScriptHash. +// This is equivalent to calling EncodeAddress, but is provided so the type +// can be used as a fmt.Stringer. +// Part of the Address interface. +func (a *AddressWitnessScriptHash) String() string { + return a.EncodeAddress() +} + +// Hrp returns the human-readable part of the bech32 encoded +// AddressWitnessScriptHash. +func (a *AddressWitnessScriptHash) Hrp() string { + return a.hrp +} + +// WitnessVersion returns the witness version of the AddressWitnessScriptHash. +func (a *AddressWitnessScriptHash) WitnessVersion() byte { + return a.witnessVersion +} + +// WitnessProgram returns the witness program of the AddressWitnessScriptHash. +func (a *AddressWitnessScriptHash) WitnessProgram() []byte { + return a.witnessProgram[:] +} + +// PayToAddrScript creates a new script to pay a transaction output to a the +// specified address. +func PayToAddrScript(addr btcutil.Address) ([]byte, error) { + const nilAddrErrStr = "unable to generate payment script for nil address" + + switch addr := addr.(type) { + case *AddressPubKeyHash: + if addr == nil { + return nil, errors.New(nilAddrErrStr) + } + return payToPubKeyHashScript(addr.ScriptAddress()) + + case *AddressScriptHash: + if addr == nil { + return nil, errors.New(nilAddrErrStr) + } + return payToScriptHashScript(addr.ScriptAddress()) + } + return nil, fmt.Errorf("unable to generate payment script for unsupported "+ + "address type %T", addr) +} + +// payToPubKeyHashScript creates a new script to pay a transaction +// output to a 20-byte pubkey hash. It is expected that the input is a valid +// hash. +func payToPubKeyHashScript(pubKeyHash []byte) ([]byte, error) { + return txscript.NewScriptBuilder().AddOp(txscript.OP_DUP).AddOp(txscript.OP_HASH160). + AddData(pubKeyHash).AddOp(txscript.OP_EQUALVERIFY).AddOp(txscript.OP_CHECKSIG). + Script() +} + +// payToScriptHashScript creates a new script to pay a transaction output to a +// script hash. It is expected that the input is a valid hash. +func payToScriptHashScript(scriptHash []byte) ([]byte, error) { + return txscript.NewScriptBuilder().AddOp(txscript.OP_HASH160).AddData(scriptHash). + AddOp(txscript.OP_EQUAL).Script() +} + +func ExtractPkScriptAddrs(pkScript []byte, chainParams *chaincfg.Params) (btcutil.Address, error) { + // No valid addresses or required signatures if the script doesn't + if len(pkScript) == 1+1+20+1 && pkScript[0] == 0xa9 && pkScript[1] == 0x14 && pkScript[22] == 0x87 { + return NewAddressScriptHashFromHash(pkScript[2:22], chainParams) + } else if len(pkScript) == 1+1+1+20+1+1 && pkScript[0] == 0x76 && pkScript[1] == 0xa9 && pkScript[2] == 0x14 && pkScript[23] == 0x88 && pkScript[24] == 0xac { + return NewAddressPubKeyHash(pkScript[3:23], chainParams) + } else if len(pkScript) == 1+1+32 && pkScript[0] == 0x00 && pkScript[1] == 0x20 { + return NewAddressWitnessScriptHash(pkScript[2:], chainParams) + } else if len(pkScript) == 1+1+20 && pkScript[0] == 0x00 && pkScript[1] == 0x14 { + return NewAddressWitnessPubKeyHash(pkScript[2:], chainParams) + } + return nil, errors.New("unknown script type") +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/litecoin/params/params.go b/vendor/github.com/OpenBazaar/multiwallet/litecoin/params/params.go new file mode 100644 index 0000000000..ce4752bcef --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/litecoin/params/params.go @@ -0,0 +1,21 @@ +package params + +import ( + "github.com/btcsuite/btcd/chaincfg" + l "github.com/ltcsuite/ltcd/chaincfg" +) + +func init() { + l.MainNetParams.ScriptHashAddrID = 0x05 +} + +func ConvertParams(params *chaincfg.Params) l.Params { + switch params.Name { + case chaincfg.MainNetParams.Name: + return l.MainNetParams + case chaincfg.TestNet3Params.Name: + return l.TestNet4Params + default: + return l.RegressionNetParams + } +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/litecoin/sign.go b/vendor/github.com/OpenBazaar/multiwallet/litecoin/sign.go new file mode 100644 index 0000000000..e90635d9ff --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/litecoin/sign.go @@ -0,0 +1,614 @@ +package litecoin + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/OpenBazaar/spvwallet" + wi "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/blockchain" + "github.com/btcsuite/btcd/btcec" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + btc "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcutil/coinset" + hd "github.com/btcsuite/btcutil/hdkeychain" + "github.com/btcsuite/btcutil/txsort" + "github.com/btcsuite/btcwallet/wallet/txauthor" + "github.com/btcsuite/btcwallet/wallet/txrules" + + laddr "github.com/OpenBazaar/multiwallet/litecoin/address" + "github.com/OpenBazaar/multiwallet/util" +) + +func (w *LitecoinWallet) buildTx(amount int64, addr btc.Address, feeLevel wi.FeeLevel, optionalOutput *wire.TxOut) (*wire.MsgTx, error) { + // Check for dust + script, _ := laddr.PayToAddrScript(addr) + if txrules.IsDustAmount(btc.Amount(amount), len(script), txrules.DefaultRelayFeePerKb) { + return nil, wi.ErrorDustAmount + } + + var additionalPrevScripts map[wire.OutPoint][]byte + var additionalKeysByAddress map[string]*btc.WIF + + // Create input source + height, _ := w.ws.ChainTip() + utxos, err := w.db.Utxos().GetAll() + if err != nil { + return nil, err + } + coinMap := util.GatherCoins(height, utxos, w.ScriptToAddress, w.km.GetKeyForScript) + + coins := make([]coinset.Coin, 0, len(coinMap)) + for k := range coinMap { + coins = append(coins, k) + } + inputSource := func(target btc.Amount) (total btc.Amount, inputs []*wire.TxIn, inputValues []btc.Amount, scripts [][]byte, err error) { + coinSelector := coinset.MaxValueAgeCoinSelector{MaxInputs: 10000, MinChangeAmount: btc.Amount(0)} + coins, err := coinSelector.CoinSelect(target, coins) + if err != nil { + return total, inputs, inputValues, scripts, wi.ErrorInsuffientFunds + } + additionalPrevScripts = make(map[wire.OutPoint][]byte) + additionalKeysByAddress = make(map[string]*btc.WIF) + for _, c := range coins.Coins() { + total += c.Value() + outpoint := wire.NewOutPoint(c.Hash(), c.Index()) + in := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + in.Sequence = 0 // Opt-in RBF so we can bump fees + inputs = append(inputs, in) + additionalPrevScripts[*outpoint] = c.PkScript() + key := coinMap[c] + addr, err := w.km.KeyToAddress(key) + if err != nil { + continue + } + privKey, err := key.ECPrivKey() + if err != nil { + continue + } + wif, _ := btc.NewWIF(privKey, w.params, true) + additionalKeysByAddress[addr.EncodeAddress()] = wif + } + return total, inputs, inputValues, scripts, nil + } + + // Get the fee per kilobyte + feePerKB := int64(w.GetFeePerByte(feeLevel)) * 1000 + + // outputs + out := wire.NewTxOut(amount, script) + + // Create change source + changeSource := func() ([]byte, error) { + addr := w.CurrentAddress(wi.INTERNAL) + script, err := laddr.PayToAddrScript(addr) + if err != nil { + return []byte{}, err + } + return script, nil + } + + outputs := []*wire.TxOut{out} + if optionalOutput != nil { + outputs = append(outputs, optionalOutput) + } + authoredTx, err := newUnsignedTransaction(outputs, btc.Amount(feePerKB), inputSource, changeSource) + if err != nil { + return nil, err + } + + // BIP 69 sorting + txsort.InPlaceSort(authoredTx.Tx) + + // Sign tx + getKey := txscript.KeyClosure(func(addr btc.Address) (*btcec.PrivateKey, bool, error) { + a, err := laddr.NewAddressPubKeyHash(addr.ScriptAddress(), w.params) + if err != nil { + return nil, false, err + } + wif := additionalKeysByAddress[a.EncodeAddress()] + return wif.PrivKey, wif.CompressPubKey, nil + }) + getScript := txscript.ScriptClosure(func( + addr btc.Address) ([]byte, error) { + return []byte{}, nil + }) + for i, txIn := range authoredTx.Tx.TxIn { + prevOutScript := additionalPrevScripts[txIn.PreviousOutPoint] + script, err := txscript.SignTxOutput(w.params, + authoredTx.Tx, i, prevOutScript, txscript.SigHashAll, getKey, + getScript, txIn.SignatureScript) + if err != nil { + return nil, errors.New("Failed to sign transaction") + } + txIn.SignatureScript = script + } + return authoredTx.Tx, nil +} + +func newUnsignedTransaction(outputs []*wire.TxOut, feePerKb btc.Amount, fetchInputs txauthor.InputSource, fetchChange txauthor.ChangeSource) (*txauthor.AuthoredTx, error) { + + var targetAmount btc.Amount + for _, txOut := range outputs { + targetAmount += btc.Amount(txOut.Value) + } + + estimatedSize := EstimateSerializeSize(1, outputs, true, P2PKH) + targetFee := txrules.FeeForSerializeSize(feePerKb, estimatedSize) + + for { + inputAmount, inputs, _, scripts, err := fetchInputs(targetAmount + targetFee) + if err != nil { + return nil, err + } + if inputAmount < targetAmount+targetFee { + return nil, errors.New("insufficient funds available to construct transaction") + } + + maxSignedSize := EstimateSerializeSize(len(inputs), outputs, true, P2PKH) + maxRequiredFee := txrules.FeeForSerializeSize(feePerKb, maxSignedSize) + remainingAmount := inputAmount - targetAmount + if remainingAmount < maxRequiredFee { + targetFee = maxRequiredFee + continue + } + + unsignedTransaction := &wire.MsgTx{ + Version: wire.TxVersion, + TxIn: inputs, + TxOut: outputs, + LockTime: 0, + } + changeIndex := -1 + changeAmount := inputAmount - targetAmount - maxRequiredFee + if changeAmount != 0 && !txrules.IsDustAmount(changeAmount, + P2PKHOutputSize, txrules.DefaultRelayFeePerKb) { + changeScript, err := fetchChange() + if err != nil { + return nil, err + } + if len(changeScript) > P2PKHPkScriptSize { + return nil, errors.New("fee estimation requires change " + + "scripts no larger than P2PKH output scripts") + } + change := wire.NewTxOut(int64(changeAmount), changeScript) + l := len(outputs) + unsignedTransaction.TxOut = append(outputs[:l:l], change) + changeIndex = l + } + + return &txauthor.AuthoredTx{ + Tx: unsignedTransaction, + PrevScripts: scripts, + TotalInput: inputAmount, + ChangeIndex: changeIndex, + }, nil + } +} + +func (w *LitecoinWallet) bumpFee(txid chainhash.Hash) (*chainhash.Hash, error) { + txn, err := w.db.Txns().Get(txid) + if err != nil { + return nil, err + } + if txn.Height > 0 { + return nil, spvwallet.BumpFeeAlreadyConfirmedError + } + if txn.Height < 0 { + return nil, spvwallet.BumpFeeTransactionDeadError + } + // Check utxos for CPFP + utxos, _ := w.db.Utxos().GetAll() + for _, u := range utxos { + if u.Op.Hash.IsEqual(&txid) && u.AtHeight == 0 { + addr, err := w.ScriptToAddress(u.ScriptPubkey) + if err != nil { + return nil, err + } + key, err := w.km.GetKeyForScript(addr.ScriptAddress()) + if err != nil { + return nil, err + } + h, err := hex.DecodeString(u.Op.Hash.String()) + if err != nil { + return nil, err + } + in := wi.TransactionInput{ + LinkedAddress: addr, + OutpointIndex: u.Op.Index, + OutpointHash: h, + Value: int64(u.Value), + } + transactionID, err := w.sweepAddress([]wi.TransactionInput{in}, nil, key, nil, wi.FEE_BUMP) + if err != nil { + return nil, err + } + return transactionID, nil + } + } + return nil, spvwallet.BumpFeeNotFoundError +} + +func (w *LitecoinWallet) sweepAddress(ins []wi.TransactionInput, address *btc.Address, key *hd.ExtendedKey, redeemScript *[]byte, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + var internalAddr btc.Address + if address != nil { + internalAddr = *address + } else { + internalAddr = w.CurrentAddress(wi.INTERNAL) + } + script, err := laddr.PayToAddrScript(internalAddr) + if err != nil { + return nil, err + } + + var val int64 + var inputs []*wire.TxIn + additionalPrevScripts := make(map[wire.OutPoint][]byte) + for _, in := range ins { + val += in.Value + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return nil, err + } + script, err := laddr.PayToAddrScript(in.LinkedAddress) + if err != nil { + return nil, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + inputs = append(inputs, input) + additionalPrevScripts[*outpoint] = script + } + out := wire.NewTxOut(val, script) + + txType := P2PKH + if redeemScript != nil { + txType = P2SH_1of2_Multisig + _, err := spvwallet.LockTimeFromRedeemScript(*redeemScript) + if err == nil { + txType = P2SH_Multisig_Timelock_1Sig + } + } + estimatedSize := EstimateSerializeSize(len(ins), []*wire.TxOut{out}, false, txType) + + // Calculate the fee + feePerByte := int(w.GetFeePerByte(feeLevel)) + fee := estimatedSize * feePerByte + + outVal := val - int64(fee) + if outVal < 0 { + outVal = 0 + } + out.Value = outVal + + tx := &wire.MsgTx{ + Version: wire.TxVersion, + TxIn: inputs, + TxOut: []*wire.TxOut{out}, + LockTime: 0, + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + // Sign tx + privKey, err := key.ECPrivKey() + if err != nil { + return nil, err + } + pk := privKey.PubKey().SerializeCompressed() + addressPub, err := btc.NewAddressPubKey(pk, w.params) + + getKey := txscript.KeyClosure(func(addr btc.Address) (*btcec.PrivateKey, bool, error) { + if addressPub.EncodeAddress() == addr.EncodeAddress() { + wif, err := btc.NewWIF(privKey, w.params, true) + if err != nil { + return nil, false, err + } + return wif.PrivKey, wif.CompressPubKey, nil + } + return nil, false, errors.New("Not found") + }) + getScript := txscript.ScriptClosure(func(addr btc.Address) ([]byte, error) { + if redeemScript == nil { + return []byte{}, nil + } + return *redeemScript, nil + }) + + // Check if time locked + var timeLocked bool + if redeemScript != nil { + rs := *redeemScript + if rs[0] == txscript.OP_IF { + timeLocked = true + tx.Version = 2 + for _, txIn := range tx.TxIn { + locktime, err := spvwallet.LockTimeFromRedeemScript(*redeemScript) + if err != nil { + return nil, err + } + txIn.Sequence = locktime + } + } + } + + hashes := txscript.NewTxSigHashes(tx) + for i, txIn := range tx.TxIn { + if redeemScript == nil { + prevOutScript := additionalPrevScripts[txIn.PreviousOutPoint] + script, err := txscript.SignTxOutput(w.params, + tx, i, prevOutScript, txscript.SigHashAll, getKey, + getScript, txIn.SignatureScript) + if err != nil { + return nil, errors.New("Failed to sign transaction") + } + txIn.SignatureScript = script + } else { + sig, err := txscript.RawTxInWitnessSignature(tx, hashes, i, ins[i].Value, *redeemScript, txscript.SigHashAll, privKey) + if err != nil { + return nil, err + } + var witness wire.TxWitness + if timeLocked { + witness = wire.TxWitness{sig, []byte{}} + } else { + witness = wire.TxWitness{[]byte{}, sig} + } + witness = append(witness, *redeemScript) + txIn.Witness = witness + } + } + + // broadcast + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + _, err = w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + txid := tx.TxHash() + return &txid, nil +} + +func (w *LitecoinWallet) createMultisigSignature(ins []wi.TransactionInput, outs []wi.TransactionOutput, key *hd.ExtendedKey, redeemScript []byte, feePerByte uint64) ([]wi.Signature, error) { + var sigs []wi.Signature + tx := wire.NewMsgTx(1) + for _, in := range ins { + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return sigs, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + tx.TxIn = append(tx.TxIn, input) + } + for _, out := range outs { + scriptPubkey, err := laddr.PayToAddrScript(out.Address) + if err != nil { + return sigs, err + } + output := wire.NewTxOut(out.Value, scriptPubkey) + tx.TxOut = append(tx.TxOut, output) + } + + // Subtract fee + txType := P2SH_2of3_Multisig + _, err := spvwallet.LockTimeFromRedeemScript(redeemScript) + if err == nil { + txType = P2SH_Multisig_Timelock_2Sigs + } + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, txType) + fee := estimatedSize * int(feePerByte) + if len(tx.TxOut) > 0 { + feePerOutput := fee / len(tx.TxOut) + for _, output := range tx.TxOut { + output.Value -= int64(feePerOutput) + } + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + signingKey, err := key.ECPrivKey() + if err != nil { + return sigs, err + } + + hashes := txscript.NewTxSigHashes(tx) + for i := range tx.TxIn { + sig, err := txscript.RawTxInWitnessSignature(tx, hashes, i, ins[i].Value, redeemScript, txscript.SigHashAll, signingKey) + if err != nil { + continue + } + bs := wi.Signature{InputIndex: uint32(i), Signature: sig} + sigs = append(sigs, bs) + } + return sigs, nil +} + +func (w *LitecoinWallet) multisign(ins []wi.TransactionInput, outs []wi.TransactionOutput, sigs1 []wi.Signature, sigs2 []wi.Signature, redeemScript []byte, feePerByte uint64, broadcast bool) ([]byte, error) { + tx := wire.NewMsgTx(1) + for _, in := range ins { + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return nil, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + tx.TxIn = append(tx.TxIn, input) + } + for _, out := range outs { + scriptPubkey, err := laddr.PayToAddrScript(out.Address) + if err != nil { + return nil, err + } + output := wire.NewTxOut(out.Value, scriptPubkey) + tx.TxOut = append(tx.TxOut, output) + } + + // Subtract fee + txType := P2SH_2of3_Multisig + _, err := spvwallet.LockTimeFromRedeemScript(redeemScript) + if err == nil { + txType = P2SH_Multisig_Timelock_2Sigs + } + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, txType) + fee := estimatedSize * int(feePerByte) + if len(tx.TxOut) > 0 { + feePerOutput := fee / len(tx.TxOut) + for _, output := range tx.TxOut { + output.Value -= int64(feePerOutput) + } + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + // Check if time locked + var timeLocked bool + if redeemScript[0] == txscript.OP_IF { + timeLocked = true + } + + for i, input := range tx.TxIn { + var sig1 []byte + var sig2 []byte + for _, sig := range sigs1 { + if int(sig.InputIndex) == i { + sig1 = sig.Signature + break + } + } + for _, sig := range sigs2 { + if int(sig.InputIndex) == i { + sig2 = sig.Signature + break + } + } + + witness := wire.TxWitness{[]byte{}, sig1, sig2} + + if timeLocked { + witness = append(witness, []byte{0x01}) + } + witness = append(witness, redeemScript) + input.Witness = witness + } + // broadcast + if broadcast { + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + _, err = w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + } + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + return buf.Bytes(), nil +} + +func (w *LitecoinWallet) generateMultisigScript(keys []hd.ExtendedKey, threshold int, timeout time.Duration, timeoutKey *hd.ExtendedKey) (addr btc.Address, redeemScript []byte, err error) { + if uint32(timeout.Hours()) > 0 && timeoutKey == nil { + return nil, nil, errors.New("Timeout key must be non nil when using an escrow timeout") + } + + if len(keys) < threshold { + return nil, nil, fmt.Errorf("unable to generate multisig script with "+ + "%d required signatures when there are only %d public "+ + "keys available", threshold, len(keys)) + } + + var ecKeys []*btcec.PublicKey + for _, key := range keys { + ecKey, err := key.ECPubKey() + if err != nil { + return nil, nil, err + } + ecKeys = append(ecKeys, ecKey) + } + + builder := txscript.NewScriptBuilder() + if uint32(timeout.Hours()) == 0 { + + builder.AddInt64(int64(threshold)) + for _, key := range ecKeys { + builder.AddData(key.SerializeCompressed()) + } + builder.AddInt64(int64(len(ecKeys))) + builder.AddOp(txscript.OP_CHECKMULTISIG) + + } else { + ecKey, err := timeoutKey.ECPubKey() + if err != nil { + return nil, nil, err + } + sequenceLock := blockchain.LockTimeToSequence(false, uint32(timeout.Hours()*6)) + builder.AddOp(txscript.OP_IF) + builder.AddInt64(int64(threshold)) + for _, key := range ecKeys { + builder.AddData(key.SerializeCompressed()) + } + builder.AddInt64(int64(len(ecKeys))) + builder.AddOp(txscript.OP_CHECKMULTISIG) + builder.AddOp(txscript.OP_ELSE). + AddInt64(int64(sequenceLock)). + AddOp(txscript.OP_CHECKSEQUENCEVERIFY). + AddOp(txscript.OP_DROP). + AddData(ecKey.SerializeCompressed()). + AddOp(txscript.OP_CHECKSIG). + AddOp(txscript.OP_ENDIF) + } + redeemScript, err = builder.Script() + if err != nil { + return nil, nil, err + } + + witnessProgram := sha256.Sum256(redeemScript) + + addr, err = laddr.NewAddressWitnessScriptHash(witnessProgram[:], w.params) + if err != nil { + return nil, nil, err + } + return addr, redeemScript, nil +} + +func (w *LitecoinWallet) estimateSpendFee(amount int64, feeLevel wi.FeeLevel) (uint64, error) { + // Since this is an estimate we can use a dummy output address. Let's use a long one so we don't under estimate. + addr, err := laddr.DecodeAddress("ltc1q65n2p3r4pwz4qppflml65en4xpdp6srjwultrun6hnddpzct5unsyyq4sf", w.params) + if err != nil { + return 0, err + } + tx, err := w.buildTx(amount, addr, feeLevel, nil) + if err != nil { + return 0, err + } + var outval int64 + for _, output := range tx.TxOut { + outval += output.Value + } + var inval int64 + utxos, err := w.db.Utxos().GetAll() + if err != nil { + return 0, err + } + for _, input := range tx.TxIn { + for _, utxo := range utxos { + if utxo.Op.Hash.IsEqual(&input.PreviousOutPoint.Hash) && utxo.Op.Index == input.PreviousOutPoint.Index { + inval += utxo.Value + break + } + } + } + if inval < outval { + return 0, errors.New("Error building transaction: inputs less than outputs") + } + return uint64(inval - outval), err +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/litecoin/txsizes.go b/vendor/github.com/OpenBazaar/multiwallet/litecoin/txsizes.go new file mode 100644 index 0000000000..e19726c5ee --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/litecoin/txsizes.go @@ -0,0 +1,249 @@ +package litecoin + +// Copyright (c) 2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* Copied here from a btcd internal package*/ + +import ( + "github.com/btcsuite/btcd/wire" +) + +// Worst case script and input/output size estimates. +const ( + // RedeemP2PKHSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2PKH output. + // It is calculated as: + // + // - OP_DATA_73 + // - 72 bytes DER signature + 1 byte sighash + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + RedeemP2PKHSigScriptSize = 1 + 73 + 1 + 33 + + // RedeemP2SHMultisigSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a 2 of 3 P2SH multisig output with compressed keys. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_PUSHDATA + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + RedeemP2SH2of3MultisigSigScriptSize = 1 + 1 + 72 + 1 + 72 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + + // RedeemP2SH1of2MultisigSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a 1 of 2 P2SH multisig output with compressed keys. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_PUSHDATA + // - OP_1 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP2 + // - OP_CHECKMULTISIG + RedeemP2SH1of2MultisigSigScriptSize = 1 + 1 + 72 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 1 + + // RedeemP2SHMultisigTimelock1SigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2SH timelocked multisig using the timeout. + // It is calculated as: + // + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_0 + // - OP_PUSHDATA + // - OP_IF + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + // - OP_ELSE + // - OP_PUSHDATA + // - 2 byte block height + // - OP_CHECKSEQUENCEVERIFY + // - OP_DROP + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_CHECKSIG + // - OP_ENDIF + RedeemP2SHMultisigTimelock1SigScriptSize = 1 + 72 + 1 + 1 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + 1 + 1 + 2 + 1 + 1 + 1 + 33 + 1 + 1 + + // RedeemP2SHMultisigTimelock2SigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2SH timelocked multisig without using the timeout. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_1 + // - OP_PUSHDATA + // - OP_IF + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + // - OP_ELSE + // - OP_PUSHDATA + // - 2 byte block height + // - OP_CHECKSEQUENCEVERIFY + // - OP_DROP + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_CHECKSIG + // - OP_ENDIF + RedeemP2SHMultisigTimelock2SigScriptSize = 1 + 1 + 72 + +1 + 72 + 1 + 1 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + 1 + 1 + 2 + 1 + 1 + 1 + 33 + 1 + 1 + + // P2PKHPkScriptSize is the size of a transaction output script that + // pays to a compressed pubkey hash. It is calculated as: + // + // - OP_DUP + // - OP_HASH160 + // - OP_DATA_20 + // - 20 bytes pubkey hash + // - OP_EQUALVERIFY + // - OP_CHECKSIG + P2PKHPkScriptSize = 1 + 1 + 1 + 20 + 1 + 1 + + // RedeemP2PKHInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2PKH output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - signature script + // - 4 bytes sequence + RedeemP2PKHInputSize = 32 + 4 + 1 + RedeemP2PKHSigScriptSize + 4 + + // RedeemP2SH2of3MultisigInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH 2 of 3 multisig output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SH2of3MultisigInputSize = 32 + 4 + 1 + 4 + (RedeemP2SH2of3MultisigSigScriptSize / 4) + + // RedeemP2SH1of2MultisigInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH 2 of 3 multisig output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SH1of2MultisigInputSize = 32 + 4 + 1 + 4 + (RedeemP2SH1of2MultisigSigScriptSize / 4) + + // RedeemP2SHMultisigTimelock1InputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed p2sh timelocked multig output with using the timeout. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SHMultisigTimelock1InputSize = 32 + 4 + 1 + 4 + (RedeemP2SHMultisigTimelock1SigScriptSize / 4) + + // RedeemP2SHMultisigTimelock2InputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH timelocked multisig output without using the timeout. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SHMultisigTimelock2InputSize = 32 + 4 + 1 + 4 + (RedeemP2SHMultisigTimelock2SigScriptSize / 4) + + // P2PKHOutputSize is the serialize size of a transaction output with a + // P2PKH output script. It is calculated as: + // + // - 8 bytes output value + // - 1 byte compact int encoding value 25 + // - 25 bytes P2PKH output script + P2PKHOutputSize = 8 + 1 + P2PKHPkScriptSize +) + +type InputType int + +const ( + P2PKH InputType = iota + P2SH_1of2_Multisig + P2SH_2of3_Multisig + P2SH_Multisig_Timelock_1Sig + P2SH_Multisig_Timelock_2Sigs +) + +// EstimateSerializeSize returns a worst case serialize size estimate for a +// signed transaction that spends inputCount number of compressed P2PKH outputs +// and contains each transaction output from txOuts. The estimated size is +// incremented for an additional P2PKH change output if addChangeOutput is true. +func EstimateSerializeSize(inputCount int, txOuts []*wire.TxOut, addChangeOutput bool, inputType InputType) int { + changeSize := 0 + outputCount := len(txOuts) + if addChangeOutput { + changeSize = P2PKHOutputSize + outputCount++ + } + + var redeemScriptSize int + switch inputType { + case P2PKH: + redeemScriptSize = RedeemP2PKHInputSize + case P2SH_1of2_Multisig: + redeemScriptSize = RedeemP2SH1of2MultisigInputSize + case P2SH_2of3_Multisig: + redeemScriptSize = RedeemP2SH2of3MultisigInputSize + case P2SH_Multisig_Timelock_1Sig: + redeemScriptSize = RedeemP2SHMultisigTimelock1InputSize + case P2SH_Multisig_Timelock_2Sigs: + redeemScriptSize = RedeemP2SHMultisigTimelock2InputSize + } + + // 10 additional bytes are for version, locktime, and segwit flags + return 10 + wire.VarIntSerializeSize(uint64(inputCount)) + + wire.VarIntSerializeSize(uint64(outputCount)) + + inputCount*redeemScriptSize + + SumOutputSerializeSizes(txOuts) + + changeSize +} + +// SumOutputSerializeSizes sums up the serialized size of the supplied outputs. +func SumOutputSerializeSizes(outputs []*wire.TxOut) (serializeSize int) { + for _, txOut := range outputs { + serializeSize += txOut.SerializeSize() + } + return serializeSize +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/litecoin/wallet.go b/vendor/github.com/OpenBazaar/multiwallet/litecoin/wallet.go new file mode 100644 index 0000000000..d416ab4367 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/litecoin/wallet.go @@ -0,0 +1,296 @@ +package litecoin + +import ( + "bytes" + "fmt" + "io" + "time" + + "github.com/OpenBazaar/spvwallet" + wi "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + hd "github.com/btcsuite/btcutil/hdkeychain" + "github.com/btcsuite/btcwallet/wallet/txrules" + "github.com/tyler-smith/go-bip39" + "golang.org/x/net/proxy" + + "github.com/OpenBazaar/multiwallet/client" + "github.com/OpenBazaar/multiwallet/config" + "github.com/OpenBazaar/multiwallet/keys" + laddr "github.com/OpenBazaar/multiwallet/litecoin/address" + "github.com/OpenBazaar/multiwallet/service" + "github.com/OpenBazaar/multiwallet/util" +) + +type LitecoinWallet struct { + db wi.Datastore + km *keys.KeyManager + params *chaincfg.Params + client client.APIClient + ws *service.WalletService + fp *spvwallet.FeeProvider + + mPrivKey *hd.ExtendedKey + mPubKey *hd.ExtendedKey +} + +func NewLitecoinWallet(cfg config.CoinConfig, mnemonic string, params *chaincfg.Params, proxy proxy.Dialer) (*LitecoinWallet, error) { + seed := bip39.NewSeed(mnemonic, "") + + mPrivKey, err := hd.NewMaster(seed, params) + if err != nil { + return nil, err + } + mPubKey, err := mPrivKey.Neuter() + if err != nil { + return nil, err + } + km, err := keys.NewKeyManager(cfg.DB.Keys(), params, mPrivKey, wi.Litecoin, litecoinAddress) + if err != nil { + return nil, err + } + + c, err := client.NewInsightClient(cfg.ClientAPI.String(), proxy) + if err != nil { + return nil, err + } + + wm := service.NewWalletService(cfg.DB, km, c, params, wi.Litecoin) + + // TODO: create litecoin fee provider + fp := spvwallet.NewFeeProvider(cfg.MaxFee, cfg.HighFee, cfg.MediumFee, cfg.LowFee, cfg.FeeAPI.String(), proxy) + + return &LitecoinWallet{cfg.DB, km, params, c, wm, fp, mPrivKey, mPubKey}, nil +} + +func litecoinAddress(key *hd.ExtendedKey, params *chaincfg.Params) (btcutil.Address, error) { + addr, err := key.Address(params) + if err != nil { + return nil, err + } + return laddr.NewAddressPubKeyHash(addr.ScriptAddress(), params) +} +func (w *LitecoinWallet) Start() { + w.ws.Start() +} + +func (w *LitecoinWallet) Params() *chaincfg.Params { + return w.params +} + +func (w *LitecoinWallet) CurrencyCode() string { + if w.params.Name == chaincfg.MainNetParams.Name { + return "ltc" + } else { + return "tltc" + } +} + +func (w *LitecoinWallet) IsDust(amount int64) bool { + return txrules.IsDustAmount(btcutil.Amount(amount), 25, txrules.DefaultRelayFeePerKb) +} + +func (w *LitecoinWallet) MasterPrivateKey() *hd.ExtendedKey { + return w.mPrivKey +} + +func (w *LitecoinWallet) MasterPublicKey() *hd.ExtendedKey { + return w.mPubKey +} + +func (w *LitecoinWallet) ChildKey(keyBytes []byte, chaincode []byte, isPrivateKey bool) (*hd.ExtendedKey, error) { + parentFP := []byte{0x00, 0x00, 0x00, 0x00} + var id []byte + if isPrivateKey { + id = w.params.HDPrivateKeyID[:] + } else { + id = w.params.HDPublicKeyID[:] + } + hdKey := hd.NewExtendedKey( + id, + keyBytes, + chaincode, + parentFP, + 0, + 0, + isPrivateKey) + return hdKey.Child(0) +} + +func (w *LitecoinWallet) CurrentAddress(purpose wi.KeyPurpose) btcutil.Address { + key, _ := w.km.GetCurrentKey(purpose) + addr, _ := litecoinAddress(key, w.params) + return btcutil.Address(addr) +} + +func (w *LitecoinWallet) NewAddress(purpose wi.KeyPurpose) btcutil.Address { + i, _ := w.db.Keys().GetUnused(purpose) + key, _ := w.km.GenerateChildKey(purpose, uint32(i[1])) + addr, _ := litecoinAddress(key, w.params) + w.db.Keys().MarkKeyAsUsed(addr.ScriptAddress()) + return btcutil.Address(addr) +} + +func (w *LitecoinWallet) DecodeAddress(addr string) (btcutil.Address, error) { + return laddr.DecodeAddress(addr, w.params) +} + +func (w *LitecoinWallet) ScriptToAddress(script []byte) (btcutil.Address, error) { + return laddr.ExtractPkScriptAddrs(script, w.params) +} + +func (w *LitecoinWallet) AddressToScript(addr btcutil.Address) ([]byte, error) { + return laddr.PayToAddrScript(addr) +} + +func (w *LitecoinWallet) HasKey(addr btcutil.Address) bool { + _, err := w.km.GetKeyForScript(addr.ScriptAddress()) + if err != nil { + return false + } + return true +} + +func (w *LitecoinWallet) Balance() (confirmed, unconfirmed int64) { + utxos, _ := w.db.Utxos().GetAll() + txns, _ := w.db.Txns().GetAll(false) + return util.CalcBalance(utxos, txns) +} + +func (w *LitecoinWallet) Transactions() ([]wi.Txn, error) { + return w.db.Txns().GetAll(false) +} + +func (w *LitecoinWallet) GetTransaction(txid chainhash.Hash) (wi.Txn, error) { + txn, err := w.db.Txns().Get(txid) + return txn, err +} + +func (w *LitecoinWallet) ChainTip() (uint32, chainhash.Hash) { + return w.ws.ChainTip() +} + +func (w *LitecoinWallet) GetFeePerByte(feeLevel wi.FeeLevel) uint64 { + return w.fp.GetFeePerByte(feeLevel) +} + +func (w *LitecoinWallet) Spend(amount int64, addr btcutil.Address, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + tx, err := w.buildTx(amount, addr, feeLevel, nil) + if err != nil { + return nil, err + } + // Broadcast + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + + _, err = w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + + ch := tx.TxHash() + return &ch, nil +} + +func (w *LitecoinWallet) BumpFee(txid chainhash.Hash) (*chainhash.Hash, error) { + return w.bumpFee(txid) +} + +func (w *LitecoinWallet) EstimateFee(ins []wi.TransactionInput, outs []wi.TransactionOutput, feePerByte uint64) uint64 { + tx := new(wire.MsgTx) + for _, out := range outs { + scriptPubKey, _ := laddr.PayToAddrScript(out.Address) + output := wire.NewTxOut(out.Value, scriptPubKey) + tx.TxOut = append(tx.TxOut, output) + } + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, P2PKH) + fee := estimatedSize * int(feePerByte) + return uint64(fee) +} + +func (w *LitecoinWallet) EstimateSpendFee(amount int64, feeLevel wi.FeeLevel) (uint64, error) { + return w.estimateSpendFee(amount, feeLevel) +} + +func (w *LitecoinWallet) SweepAddress(ins []wi.TransactionInput, address *btcutil.Address, key *hd.ExtendedKey, redeemScript *[]byte, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + return w.sweepAddress(ins, address, key, redeemScript, feeLevel) +} + +func (w *LitecoinWallet) CreateMultisigSignature(ins []wi.TransactionInput, outs []wi.TransactionOutput, key *hd.ExtendedKey, redeemScript []byte, feePerByte uint64) ([]wi.Signature, error) { + return w.createMultisigSignature(ins, outs, key, redeemScript, feePerByte) +} + +func (w *LitecoinWallet) Multisign(ins []wi.TransactionInput, outs []wi.TransactionOutput, sigs1 []wi.Signature, sigs2 []wi.Signature, redeemScript []byte, feePerByte uint64, broadcast bool) ([]byte, error) { + return w.multisign(ins, outs, sigs1, sigs2, redeemScript, feePerByte, broadcast) +} + +func (w *LitecoinWallet) GenerateMultisigScript(keys []hd.ExtendedKey, threshold int, timeout time.Duration, timeoutKey *hd.ExtendedKey) (addr btcutil.Address, redeemScript []byte, err error) { + return w.generateMultisigScript(keys, threshold, timeout, timeoutKey) +} + +func (w *LitecoinWallet) AddWatchedAddress(addr btcutil.Address) error { + script, err := w.AddressToScript(addr) + if err != nil { + return err + } + err = w.db.WatchedScripts().Put(script) + if err != nil { + return err + } + w.client.ListenAddress(addr) + return nil +} + +func (w *LitecoinWallet) AddWatchedScript(script []byte) error { + err := w.db.WatchedScripts().Put(script) + if err != nil { + return err + } + addr, err := w.ScriptToAddress(script) + if err != nil { + return err + } + w.client.ListenAddress(addr) + return nil +} + +func (w *LitecoinWallet) AddTransactionListener(callback func(wi.TransactionCallback)) { + w.ws.AddTransactionListener(callback) +} + +func (w *LitecoinWallet) ReSyncBlockchain(fromTime time.Time) { + go w.ws.UpdateState() +} + +func (w *LitecoinWallet) GetConfirmations(txid chainhash.Hash) (uint32, uint32, error) { + txn, err := w.db.Txns().Get(txid) + if err != nil { + return 0, 0, err + } + if txn.Height == 0 { + return 0, 0, nil + } + chainTip, _ := w.ChainTip() + return chainTip - uint32(txn.Height) + 1, uint32(txn.Height), nil +} + +func (w *LitecoinWallet) Close() { + w.ws.Stop() + w.client.Close() +} + +func (w *LitecoinWallet) DumpTables(wr io.Writer) { + fmt.Fprintln(wr, "Transactions-----") + txns, _ := w.db.Txns().GetAll(true) + for _, tx := range txns { + fmt.Fprintf(wr, "Hash: %s, Height: %d, Value: %d, WatchOnly: %t\n", tx.Txid, int(tx.Height), int(tx.Value), tx.WatchOnly) + } + fmt.Fprintln(wr, "\nUtxos-----") + utxos, _ := w.db.Utxos().GetAll() + for _, u := range utxos { + fmt.Fprintf(wr, "Hash: %s, Index: %d, Height: %d, Value: %d, WatchOnly: %t\n", u.Op.Hash.String(), int(u.Op.Index), int(u.AtHeight), int(u.Value), u.WatchOnly) + } +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/makefile b/vendor/github.com/OpenBazaar/multiwallet/makefile new file mode 100644 index 0000000000..fe026e12cf --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/makefile @@ -0,0 +1,5 @@ +install: + cd cmd/multiwallet && go install + +protos: + cd api/pb && protoc --go_out=plugins=grpc:. api.proto \ No newline at end of file diff --git a/vendor/github.com/OpenBazaar/multiwallet/multiwallet.go b/vendor/github.com/OpenBazaar/multiwallet/multiwallet.go new file mode 100644 index 0000000000..4e26d3a2a1 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/multiwallet.go @@ -0,0 +1,86 @@ +package multiwallet + +import ( + "errors" + "github.com/OpenBazaar/multiwallet/bitcoin" + "github.com/OpenBazaar/multiwallet/bitcoincash" + "github.com/OpenBazaar/multiwallet/config" + "github.com/OpenBazaar/multiwallet/litecoin" + "github.com/OpenBazaar/multiwallet/zcash" + "github.com/OpenBazaar/wallet-interface" + "github.com/op/go-logging" + "github.com/tyler-smith/go-bip39" + "strings" + "time" +) + +var log = logging.MustGetLogger("multiwallet") + +var UnsuppertedCoinError = errors.New("multiwallet does not contain an implementation for the given coin") + +type MultiWallet map[wallet.CoinType]wallet.Wallet + +func NewMultiWallet(cfg *config.Config) (MultiWallet, error) { + log.SetBackend(logging.AddModuleLevel(cfg.Logger)) + + if cfg.Mnemonic == "" { + ent, err := bip39.NewEntropy(128) + if err != nil { + return nil, err + } + mnemonic, err := bip39.NewMnemonic(ent) + if err != nil { + return nil, err + } + cfg.Mnemonic = mnemonic + cfg.CreationDate = time.Now() + } + + multiwallet := make(MultiWallet) + var err error + for _, coin := range cfg.Coins { + var w wallet.Wallet + switch coin.CoinType { + case wallet.Bitcoin: + w, err = bitcoin.NewBitcoinWallet(coin, cfg.Mnemonic, cfg.Params, cfg.Proxy) + if err != nil { + return nil, err + } + multiwallet[coin.CoinType] = w + case wallet.BitcoinCash: + w, err = bitcoincash.NewBitcoinCashWallet(coin, cfg.Mnemonic, cfg.Params, cfg.Proxy) + if err != nil { + return nil, err + } + multiwallet[coin.CoinType] = w + case wallet.Zcash: + w, err = zcash.NewZCashWallet(coin, cfg.Mnemonic, cfg.Params, cfg.Proxy) + if err != nil { + return nil, err + } + multiwallet[coin.CoinType] = w + case wallet.Litecoin: + w, err = litecoin.NewLitecoinWallet(coin, cfg.Mnemonic, cfg.Params, cfg.Proxy) + if err != nil { + return nil, err + } + multiwallet[coin.CoinType] = w + } + } + return multiwallet, nil +} + +func (w *MultiWallet) Start() { + for _, wallet := range *w { + wallet.Start() + } +} + +func (w *MultiWallet) WalletForCurrencyCode(currencyCode string) (wallet.Wallet, error) { + for _, wl := range *w { + if strings.ToUpper(wl.CurrencyCode()) == strings.ToUpper(currencyCode) { + return wl, nil + } + } + return nil, UnsuppertedCoinError +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/service/wallet_service.go b/vendor/github.com/OpenBazaar/multiwallet/service/wallet_service.go new file mode 100644 index 0000000000..28d7e043b3 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/service/wallet_service.go @@ -0,0 +1,541 @@ +package service + +import ( + "bytes" + "encoding/hex" + "strconv" + "sync" + "time" + + "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + "github.com/cpacia/bchutil" + "github.com/op/go-logging" + + "github.com/OpenBazaar/multiwallet/client" + "github.com/OpenBazaar/multiwallet/keys" + laddr "github.com/OpenBazaar/multiwallet/litecoin/address" + "github.com/OpenBazaar/multiwallet/util" + zaddr "github.com/OpenBazaar/multiwallet/zcash/address" +) + +var log = logging.MustGetLogger("WalletService") + +type WalletService struct { + db wallet.Datastore + km *keys.KeyManager + client client.APIClient + params *chaincfg.Params + coinType wallet.CoinType + + chainHeight uint32 + bestBlock string + + listeners []func(wallet.TransactionCallback) + + lock sync.RWMutex + + doneChan chan struct{} +} + +const nullHash = "0000000000000000000000000000000000000000000000000000000000000000" + +func NewWalletService(db wallet.Datastore, km *keys.KeyManager, client client.APIClient, params *chaincfg.Params, coinType wallet.CoinType) *WalletService { + return &WalletService{db, km, client, params, coinType, 0, nullHash, []func(wallet.TransactionCallback){}, sync.RWMutex{}, make(chan struct{})} +} + +func (ws *WalletService) Start() { + log.Noticef("Starting %s WalletService", ws.coinType.String()) + go ws.UpdateState() + go ws.listen() +} + +func (ws *WalletService) Stop() { + ws.doneChan <- struct{}{} +} + +func (ws *WalletService) ChainTip() (uint32, chainhash.Hash) { + ws.lock.RLock() + defer ws.lock.RUnlock() + ch, _ := chainhash.NewHashFromStr(ws.bestBlock) + return uint32(ws.chainHeight), *ch +} + +func (ws *WalletService) AddTransactionListener(callback func(callback wallet.TransactionCallback)) { + ws.listeners = append(ws.listeners, callback) +} + +func (ws *WalletService) listen() { + addrs := ws.getStoredAddresses() + for _, sa := range addrs { + ws.client.ListenAddress(sa.Addr) + } + + for { + select { + case <-ws.doneChan: + break + case tx := <-ws.client.TransactionNotify(): + go ws.processIncomingTransaction(tx) + case block := <-ws.client.BlockNotify(): + go ws.processIncomingBlock(block) + } + } +} + +// This is a transaction fresh off the wire. Let's save it to the db. +func (ws *WalletService) processIncomingTransaction(tx client.Transaction) { + log.Debugf("New incoming %s transaction: %s", ws.coinType.String(), tx.Txid) + addrs := ws.getStoredAddresses() + ws.lock.RLock() + chainHeight := int32(ws.chainHeight) + ws.lock.RUnlock() + ws.saveSingleTxToDB(tx, chainHeight, addrs) + utxos, err := ws.db.Utxos().GetAll() + if err != nil { + log.Errorf("Error loading %s utxos: %s", ws.coinType.String(), err.Error()) + } + + for _, sa := range addrs { + for _, out := range tx.Outputs { + for _, addr := range out.ScriptPubKey.Addresses { + if addr == sa.Addr.String() { + utxo := client.Utxo{ + Txid: tx.Txid, + ScriptPubKey: out.ScriptPubKey.Hex, + Satoshis: int64(out.Value * util.SatoshisPerCoin(ws.coinType)), + Vout: out.N, + Address: addr, + Confirmations: 0, + Amount: out.Value, + } + ws.saveSingleUtxoToDB(utxo, addrs, chainHeight) + break + } + } + } + // If spending a utxo, delete it + for _, in := range tx.Inputs { + for _, u := range utxos { + if in.Txid == u.Op.Hash.String() && in.Vout == int(u.Op.Index) { + ws.db.Utxos().Delete(u) + break + } + } + } + } +} + +// A new block was found let's update our chain height and best hash and check for a reorg +func (ws *WalletService) processIncomingBlock(block client.Block) { + log.Infof("Received new %s block at height %d: %s", ws.coinType.String(), block.Height, block.Hash) + ws.lock.RLock() + currentBest := ws.bestBlock + ws.lock.RUnlock() + + ws.lock.Lock() + ws.chainHeight = uint32(block.Height) + ws.bestBlock = block.Hash + ws.lock.Unlock() + + // REORG! Rescan all transactions and utxos to see if anything changed + if currentBest != block.PreviousBlockhash { + log.Warningf("Reorg in the %s chain! Re-scanning wallet", ws.coinType.String()) + ws.UpdateState() + return + } + + // Query db for unconfirmed txs and utxos then query API to get current height + txs, err := ws.db.Txns().GetAll(true) + if err != nil { + log.Errorf("Error loading %s txs from db: %s", ws.coinType.String(), err.Error()) + return + } + utxos, err := ws.db.Utxos().GetAll() + if err != nil { + log.Errorf("Error loading %s txs from db: %s", ws.coinType.String(), err.Error()) + return + } + addrs := ws.getStoredAddresses() + for _, tx := range txs { + if tx.Height == 0 { + go func(txn wallet.Txn) { + ret, err := ws.client.GetTransaction(txn.Txid) + if err != nil { + log.Errorf("Error fetching unconfirmed %s tx: %s", ws.coinType.String(), err.Error()) + return + } + if ret.Confirmations > 0 { + h := int32(block.Height) - int32(ret.Confirmations-1) + ws.saveSingleTxToDB(*ret, int32(block.Height), addrs) + for _, u := range utxos { + if u.Op.Hash.String() == txn.Txid { + u.AtHeight = h + ws.db.Utxos().Put(u) + } + } + } + }(tx) + } + } +} + +// updateState will query the API for both UTXOs and TXs relevant to our wallet and then update +// the db state to match the API responses. +func (ws *WalletService) UpdateState() { + // Start by fetching the chain height from the API + log.Debugf("Querying for %s chain height", ws.coinType.String()) + best, err := ws.client.GetBestBlock() + if err == nil { + log.Debugf("%s chain height: %d", ws.coinType.String(), best.Height) + ws.lock.Lock() + ws.chainHeight = uint32(best.Height) + ws.bestBlock = best.Hash + ws.lock.Unlock() + } else { + log.Errorf("Error querying API for %s chain height: %s", ws.coinType.String(), err.Error()) + } + + // Load wallet addresses and watch only addresses from the db + addrs := ws.getStoredAddresses() + + go ws.syncUtxos(addrs) + go ws.syncTxs(addrs) + +} + +// Query API for UTXOs and synchronize db state +func (ws *WalletService) syncUtxos(addrs map[string]storedAddress) { + log.Debugf("Querying for %s utxos", ws.coinType.String()) + var query []btcutil.Address + for _, sa := range addrs { + query = append(query, sa.Addr) + } + utxos, err := ws.client.GetUtxos(query) + if err != nil { + log.Errorf("Error downloading utxos for %s: %s", ws.coinType.String(), err.Error()) + } else { + log.Debugf("Downloaded %d %s utxos", len(utxos), ws.coinType.String()) + ws.saveUtxosToDB(utxos, addrs) + } +} + +// For each API response we will have to figure out height at which the UTXO has confirmed (if it has) and +// build a UTXO object suitable for saving to the database. If the database contains any UTXOs not returned +// by the API we will delete them. +func (ws *WalletService) saveUtxosToDB(utxos []client.Utxo, addrs map[string]storedAddress) { + // Get current utxos + currentUtxos, err := ws.db.Utxos().GetAll() + if err != nil { + log.Error("Error loading utxos for %s: %s", ws.coinType.String(), err.Error()) + return + } + + ws.lock.RLock() + chainHeight := int32(ws.chainHeight) + ws.lock.RUnlock() + + newUtxos := make(map[string]wallet.Utxo) + // Iterate over new utxos and put them to the db + for _, u := range utxos { + ch, err := chainhash.NewHashFromStr(u.Txid) + if err != nil { + log.Error("Error converting to chainhash for %s: %s", ws.coinType.String(), err.Error()) + continue + } + newU := wallet.Utxo{ + Op: *wire.NewOutPoint(ch, uint32(u.Vout)), + } + newUtxos[serializeUtxo(newU)] = newU + ws.saveSingleUtxoToDB(u, addrs, chainHeight) + } + // If any old utxos were not returned by the API, delete them. + for _, cur := range currentUtxos { + _, ok := newUtxos[serializeUtxo(cur)] + if !ok { + ws.db.Utxos().Delete(cur) + } + } +} + +func (ws *WalletService) saveSingleUtxoToDB(u client.Utxo, addrs map[string]storedAddress, chainHeight int32) { + ch, err := chainhash.NewHashFromStr(u.Txid) + if err != nil { + log.Error("Error converting to chainhash for %s: %s", ws.coinType.String(), err.Error()) + return + } + scriptBytes, err := hex.DecodeString(u.ScriptPubKey) + if err != nil { + log.Error("Error converting to script bytes for %s: %s", ws.coinType.String(), err.Error()) + return + } + + var watchOnly bool + sa, ok := addrs[u.Address] + if sa.WatchOnly || !ok { + watchOnly = true + } + + height := int32(0) + if u.Confirmations > 0 { + height = chainHeight - (int32(u.Confirmations) - 1) + } + + newU := wallet.Utxo{ + Op: *wire.NewOutPoint(ch, uint32(u.Vout)), + Value: u.Satoshis, + WatchOnly: watchOnly, + ScriptPubkey: scriptBytes, + AtHeight: height, + } + + ws.db.Utxos().Put(newU) +} + +// For use as a map key +func serializeUtxo(u wallet.Utxo) string { + ser := u.Op.Hash.String() + ser += strconv.Itoa(int(u.Op.Index)) + return ser +} + +// Query API for TXs and synchronize db state +func (ws *WalletService) syncTxs(addrs map[string]storedAddress) { + log.Debugf("Querying for %s transactions", ws.coinType.String()) + var query []btcutil.Address + for _, sa := range addrs { + query = append(query, sa.Addr) + } + txs, err := ws.client.GetTransactions(query) + if err != nil { + log.Errorf("Error downloading txs for %s: %s", ws.coinType.String(), err.Error()) + } else { + log.Debugf("Downloaded %d %s transactions", len(txs), ws.coinType.String()) + ws.saveTxsToDB(txs, addrs) + } +} + +// For each API response we will need to determine the net coins leaving/entering the wallet as well as determine +// if the transaction was exclusively for our `watch only` addresses. We will also build a Tx object suitable +// for saving to the db and delete any existing txs not returned by the API. Finally, for any output matching a key +// in our wallet we need to mark that key as used in the db +func (ws *WalletService) saveTxsToDB(txns []client.Transaction, addrs map[string]storedAddress) { + // Get current utxos + currentTxs, err := ws.db.Txns().GetAll(true) + if err != nil { + log.Error("Error loading utxos for %s: %s", ws.coinType.String(), err.Error()) + return + } + + ws.lock.RLock() + chainHeight := int32(ws.chainHeight) + ws.lock.RUnlock() + + newTxs := make(map[string]bool) + // Iterate over new utxos and put them to the db + for _, u := range txns { + ws.saveSingleTxToDB(u, chainHeight, addrs) + newTxs[u.Txid] = true + } + // If any old utxos were not returned by the API, delete them. + for _, cur := range currentTxs { + if !newTxs[cur.Txid] { + ch, err := chainhash.NewHashFromStr(cur.Txid) + if err != nil { + log.Errorf("error converting to chainhash for %s: %s", ws.coinType.String(), err.Error()) + continue + } + ws.db.Txns().Delete(ch) + } + } +} + +func (ws *WalletService) saveSingleTxToDB(u client.Transaction, chainHeight int32, addrs map[string]storedAddress) { + msgTx := wire.NewMsgTx(int32(u.Version)) + msgTx.LockTime = uint32(u.Locktime) + hits := 0 + value := int64(0) + + height := int32(0) + if u.Confirmations > 0 { + height = chainHeight - (int32(u.Confirmations) - 1) + } + + txHash, err := chainhash.NewHashFromStr(u.Txid) + if err != nil { + log.Errorf("error converting to txHash for %s: %s", ws.coinType.String(), err.Error()) + return + } + var relevant bool + cb := wallet.TransactionCallback{Txid: txHash.String(), Height: height} + for _, in := range u.Inputs { + ch, err := chainhash.NewHashFromStr(in.Txid) + if err != nil { + log.Errorf("error converting to chainhash for %s: %s", ws.coinType.String(), err.Error()) + continue + } + script, err := hex.DecodeString(in.ScriptSig.Hex) + if err != nil { + log.Errorf("error converting to scriptsig for %s: %s", ws.coinType.String(), err.Error()) + continue + } + op := wire.NewOutPoint(ch, uint32(in.Vout)) + // Skip the error check here as someone may have sent from an exotic script + // that we cannot turn into an address. + addr, _ := util.DecodeAddress(in.Addr, ws.params) + + txin := wire.NewTxIn(op, script, [][]byte{}) + txin.Sequence = uint32(in.Sequence) + msgTx.TxIn = append(msgTx.TxIn, txin) + h, err := hex.DecodeString(op.Hash.String()) + if err != nil { + log.Errorf("error converting outpoint hash for %s: %s", ws.coinType.String(), err.Error()) + return + } + cbin := wallet.TransactionInput{ + OutpointHash: h, + OutpointIndex: op.Index, + LinkedAddress: addr, + Value: in.Satoshis, + } + cb.Inputs = append(cb.Inputs, cbin) + + sa, ok := addrs[in.Addr] + if !ok { + continue + } + value -= in.Satoshis + if !sa.WatchOnly { + hits++ + } + relevant = true + } + for i, out := range u.Outputs { + script, err := hex.DecodeString(out.ScriptPubKey.Hex) + if err != nil { + log.Errorf("error converting to scriptPubkey for %s: %s", ws.coinType.String(), err.Error()) + continue + } + // Skip the error check here as someone may have sent from an exotic script + // that we cannot turn into an address. + addr, _ := util.DecodeAddress(out.ScriptPubKey.Addresses[0], ws.params) + + if len(out.ScriptPubKey.Addresses) == 0 { + continue + } + v := int64(out.Value * util.SatoshisPerCoin(ws.coinType)) + + txout := wire.NewTxOut(v, script) + msgTx.TxOut = append(msgTx.TxOut, txout) + cbout := wallet.TransactionOutput{Address: addr, Value: v, Index: uint32(i)} + cb.Outputs = append(cb.Outputs, cbout) + + sa, ok := addrs[out.ScriptPubKey.Addresses[0]] + if !ok { + continue + } + value += v + if !sa.WatchOnly { + hits++ + // Mark the key we received coins to as used + ws.db.Keys().MarkKeyAsUsed(sa.Addr.ScriptAddress()) + } + relevant = true + } + + if !relevant { + return + } + + cb.Value = value + cb.WatchOnly = (hits == 0) + saved, err := ws.db.Txns().Get(*txHash) + if err != nil { + ts := time.Now() + if u.Confirmations > 0 { + ts = time.Unix(u.BlockTime, 0) + } + var buf bytes.Buffer + msgTx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + ws.db.Txns().Put(buf.Bytes(), msgTx.TxHash().String(), int(value), int(height), ts, hits == 0) + cb.Timestamp = ts + ws.callbackListeners(cb) + } else { + ws.db.Txns().UpdateHeight(*txHash, int(height), time.Unix(u.BlockTime, 0)) + if saved.Height != height { + cb.Timestamp = saved.Timestamp + ws.callbackListeners(cb) + } + } +} + +func (ws *WalletService) callbackListeners(cb wallet.TransactionCallback) { + for _, callback := range ws.listeners { + callback(cb) + } +} + +type storedAddress struct { + Addr btcutil.Address + WatchOnly bool +} + +func (ws *WalletService) getStoredAddresses() map[string]storedAddress { + keys := ws.km.GetKeys() + addrs := make(map[string]storedAddress) + for _, key := range keys { + addr, err := ws.km.KeyToAddress(key) + if err != nil { + log.Errorf("Error getting %s address for key: %s", ws.coinType.String(), err.Error()) + continue + } + addrs[addr.String()] = storedAddress{addr, false} + } + watchScripts, err := ws.db.WatchedScripts().GetAll() + if err != nil { + log.Errorf("Error loading %s watch scripts: %s", ws.coinType.String(), err.Error()) + } else { + for _, script := range watchScripts { + switch ws.coinType { + case wallet.Bitcoin: + _, addrSlice, _, err := txscript.ExtractPkScriptAddrs(script, ws.params) + if err != nil { + log.Errorf("Error serializing %s script: %s", ws.coinType.String(), err.Error()) + continue + } + if len(addrs) == 0 { + log.Errorf("Error serializing %s script: %s", ws.coinType.String(), "Unknown script") + continue + } + addr := addrSlice[0] + addrs[addr.String()] = storedAddress{addr, true} + case wallet.BitcoinCash: + addr, err := bchutil.ExtractPkScriptAddrs(script, ws.params) + if err != nil { + log.Errorf("Error serializing %s script: %s", ws.coinType.String(), err.Error()) + continue + } + addrs[addr.String()] = storedAddress{addr, true} + case wallet.Zcash: + addr, err := zaddr.ExtractPkScriptAddrs(script, ws.params) + if err != nil { + log.Errorf("Error serializing %s script: %s", ws.coinType.String(), err.Error()) + continue + } + addrs[addr.String()] = storedAddress{addr, true} + case wallet.Litecoin: + addr, err := laddr.ExtractPkScriptAddrs(script, ws.params) + if err != nil { + log.Errorf("Error serializing %s script: %s", ws.coinType.String(), err.Error()) + continue + } + addrs[addr.String()] = storedAddress{addr, true} + } + } + } + return addrs +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/test_compile.sh b/vendor/github.com/OpenBazaar/multiwallet/test_compile.sh new file mode 100755 index 0000000000..3556ff811a --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/test_compile.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e +pwd +go get gopkg.in/jarcoal/httpmock.v1 +go get github.com/OpenBazaar/multiwallet +go get github.com/mattn/go-sqlite3 +go test -coverprofile=bitcoin.cover.out ./bitcoin +go test -coverprofile=client.cover.out ./client +go test -coverprofile=config.cover.out ./config +go test -coverprofile=keys.cover.out ./keys +go test -coverprofile=litecoin.cover.out ./litecoin +go test -coverprofile=service.cover.out ./service +go test -coverprofile=util.cover.out ./util +go test -coverprofile=zcash.cover.out ./zcash +go test -coverprofile=multiwallet.cover.out ./ +echo "mode: set" > coverage.out && cat *.cover.out | grep -v mode: | sort -r | \ +awk '{if($1 != last) {print $0;last=$1}}' >> coverage.out +rm -rf *.cover.out \ No newline at end of file diff --git a/vendor/github.com/OpenBazaar/multiwallet/util/address.go b/vendor/github.com/OpenBazaar/multiwallet/util/address.go new file mode 100644 index 0000000000..74f10fb142 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/util/address.go @@ -0,0 +1,27 @@ +package util + +import ( + liteaddr "github.com/OpenBazaar/multiwallet/litecoin/address" + zaddr "github.com/OpenBazaar/multiwallet/zcash/address" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcutil" + "github.com/cpacia/bchutil" + + "errors" +) + +func DecodeAddress(address string, params *chaincfg.Params) (btcutil.Address, error) { + if addr, err := btcutil.DecodeAddress(address, params); err == nil { + return addr, nil + } + if addr, err := bchutil.DecodeAddress(address, params); err == nil { + return addr, nil + } + if addr, err := liteaddr.DecodeAddress(address, params); err == nil { + return addr, nil + } + if addr, err := zaddr.DecodeAddress(address, params); err == nil { + return addr, nil + } + return nil, errors.New("unknown address") +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/util/balance.go b/vendor/github.com/OpenBazaar/multiwallet/util/balance.go new file mode 100644 index 0000000000..ca40b7479c --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/util/balance.go @@ -0,0 +1,59 @@ +package util + +import ( + "bytes" + wi "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/wire" +) + +func CalcBalance(utxos []wi.Utxo, txns []wi.Txn) (confirmed, unconfirmed int64) { + var txmap = make(map[string]wi.Txn) + for _, tx := range txns { + txmap[tx.Txid] = tx + } + + for _, utxo := range utxos { + if !utxo.WatchOnly { + if utxo.AtHeight > 0 { + confirmed += utxo.Value + } else { + if checkIfStxoIsConfirmed(utxo.Op.Hash.String(), txmap) { + confirmed += utxo.Value + } else { + unconfirmed += utxo.Value + } + } + } + } + return confirmed, unconfirmed +} + +func checkIfStxoIsConfirmed(txid string, txmap map[string]wi.Txn) bool { + // First look up tx and derserialize + txn, ok := txmap[txid] + if !ok { + return false + } + tx := wire.NewMsgTx(1) + rbuf := bytes.NewReader(txn.Bytes) + err := tx.BtcDecode(rbuf, wire.ProtocolVersion, wire.WitnessEncoding) + if err != nil { + return false + } + + // For each input, recursively check if confirmed + inputsConfirmed := true + for _, in := range tx.TxIn { + checkTx, ok := txmap[in.PreviousOutPoint.Hash.String()] + if ok { // Is an stxo. If confirmed we can return true. If no, we need to check the dependency. + if checkTx.Height == 0 { + if !checkIfStxoIsConfirmed(in.PreviousOutPoint.Hash.String(), txmap) { + inputsConfirmed = false + } + } + } else { // We don't have the tx in our db so it can't be an stxo. Return false. + return false + } + } + return inputsConfirmed +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/util/coin.go b/vendor/github.com/OpenBazaar/multiwallet/util/coin.go new file mode 100644 index 0000000000..3156fafa5c --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/util/coin.go @@ -0,0 +1,66 @@ +package util + +import ( + "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcutil/coinset" + hd "github.com/btcsuite/btcutil/hdkeychain" +) + +type Coin struct { + TxHash *chainhash.Hash + TxIndex uint32 + TxValue btcutil.Amount + TxNumConfs int64 + ScriptPubKey []byte +} + +func (c *Coin) Hash() *chainhash.Hash { return c.TxHash } +func (c *Coin) Index() uint32 { return c.TxIndex } +func (c *Coin) Value() btcutil.Amount { return c.TxValue } +func (c *Coin) PkScript() []byte { return c.ScriptPubKey } +func (c *Coin) NumConfs() int64 { return c.TxNumConfs } +func (c *Coin) ValueAge() int64 { return int64(c.TxValue) * c.TxNumConfs } + +func NewCoin(txid []byte, index uint32, value btcutil.Amount, numConfs int64, scriptPubKey []byte) (coinset.Coin, error) { + shaTxid, err := chainhash.NewHash(txid) + if err != nil { + return nil, err + } + c := &Coin{ + TxHash: shaTxid, + TxIndex: index, + TxValue: value, + TxNumConfs: numConfs, + ScriptPubKey: scriptPubKey, + } + return coinset.Coin(c), nil +} + +func GatherCoins(height uint32, utxos []wallet.Utxo, scriptToAddress func(script []byte) (btcutil.Address, error), getKeyForScript func(scriptAddress []byte) (*hd.ExtendedKey, error)) map[coinset.Coin]*hd.ExtendedKey { + m := make(map[coinset.Coin]*hd.ExtendedKey) + for _, u := range utxos { + if u.WatchOnly { + continue + } + var confirmations int32 + if u.AtHeight > 0 { + confirmations = int32(height) - u.AtHeight + } + c, err := NewCoin(u.Op.Hash.CloneBytes(), u.Op.Index, btcutil.Amount(u.Value), int64(confirmations), u.ScriptPubkey) + if err != nil { + continue + } + addr, err := scriptToAddress(u.ScriptPubkey) + if err != nil { + continue + } + key, err := getKeyForScript(addr.ScriptAddress()) + if err != nil { + continue + } + m[c] = key + } + return m +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/util/satoshis.go b/vendor/github.com/OpenBazaar/multiwallet/util/satoshis.go new file mode 100644 index 0000000000..dc246c6ba6 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/util/satoshis.go @@ -0,0 +1,8 @@ +package util + +import "github.com/OpenBazaar/wallet-interface" + +// All implemented coins currently have 100m satoshis per coin +func SatoshisPerCoin(coinType wallet.CoinType) float64 { + return 100000000 +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/zcash/address/address.go b/vendor/github.com/OpenBazaar/multiwallet/zcash/address/address.go new file mode 100644 index 0000000000..f9f77f457a --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/zcash/address/address.go @@ -0,0 +1,335 @@ +package address + +import ( + "errors" + + "bytes" + "crypto/sha256" + "fmt" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcutil/base58" + "golang.org/x/crypto/ripemd160" +) + +var ( + // ErrChecksumMismatch describes an error where decoding failed due + // to a bad checksum. + ErrChecksumMismatch = errors.New("checksum mismatch") + + // ErrUnknownAddressType describes an error where an address can not + // decoded as a specific address type due to the string encoding + // begining with an identifier byte unknown to any standard or + // registered (via chaincfg.Register) network. + ErrUnknownAddressType = errors.New("unknown address type") + + // ErrAddressCollision describes an error where an address can not + // be uniquely determined as either a pay-to-pubkey-hash or + // pay-to-script-hash address since the leading identifier is used for + // describing both address kinds, but for different networks. Rather + // than assuming or defaulting to one or the other, this error is + // returned and the caller must decide how to decode the address. + ErrAddressCollision = errors.New("address collision") + + // ErrInvalidFormat describes an error where decoding failed due to invalid version + ErrInvalidFormat = errors.New("invalid format: version and/or checksum bytes missing") + + NetIDs map[string]NetID +) + +type NetID struct { + AddressPubKeyHash []byte + AddressScriptHash []byte + ZAddress []byte +} + +func init() { + NetIDs = make(map[string]NetID) + NetIDs[chaincfg.MainNetParams.Name] = NetID{[]byte{0x1c, 0xb8}, []byte{0x1c, 0xbd}, []byte{0x16, 0x9a}} + NetIDs[chaincfg.TestNet3Params.Name] = NetID{[]byte{0x1d, 0x25}, []byte{0x1c, 0xba}, []byte{0x16, 0xb6}} + NetIDs[chaincfg.RegressionNetParams.Name] = NetID{[]byte{0x1d, 0x25}, []byte{0x1c, 0xba}, []byte{0x16, 0xb6}} +} + +// checksum: first four bytes of sha256^2 +func checksum(input []byte) (cksum [4]byte) { + h := sha256.Sum256(input) + h2 := sha256.Sum256(h[:]) + copy(cksum[:], h2[:4]) + return +} + +// CheckEncode prepends a version byte and appends a four byte checksum. +func CheckEncode(input []byte, version []byte) string { + b := make([]byte, 0, 2+len(input)+4) + b = append(b, version...) + b = append(b, input[:]...) + cksum := checksum(b) + b = append(b, cksum[:]...) + return base58.Encode(b) +} + +// CheckDecode decodes a string that was encoded with CheckEncode and verifies the checksum. +func CheckDecode(input string) (result []byte, version []byte, err error) { + decoded := base58.Decode(input) + if len(decoded) < 5 { + return nil, nil, ErrInvalidFormat + } + version = append(version, decoded[0:2]...) + var cksum [4]byte + copy(cksum[:], decoded[len(decoded)-4:]) + if checksum(decoded[:len(decoded)-4]) != cksum { + return nil, nil, ErrChecksumMismatch + } + payload := decoded[2 : len(decoded)-4] + result = append(result, payload...) + return +} + +// encodeAddress returns a human-readable payment address given a ripemd160 hash +// and netID which encodes the zcash network and address type. It is used +// in both pay-to-pubkey-hash (P2PKH) and pay-to-script-hash (P2SH) address +// encoding. +func encodeAddress(hash160 []byte, netID []byte) string { + // Format is 2 bytes for a network and address class (i.e. P2PKH vs + // P2SH), 20 bytes for a RIPEMD160 hash, and 4 bytes of checksum. + return CheckEncode(hash160[:ripemd160.Size], netID) +} + +// DecodeAddress decodes the string encoding of an address and returns +// the Address if addr is a valid encoding for a known address type. +// +// The zcash network the address is associated with is extracted if possible. +func DecodeAddress(addr string, defaultNet *chaincfg.Params) (btcutil.Address, error) { + + checkID, ok := NetIDs[defaultNet.Name] + if !ok { + return nil, errors.New("unknown network parameters") + } + + // Switch on decoded length to determine the type. + decoded, netID, err := CheckDecode(addr) + if err != nil { + if err == base58.ErrChecksum { + return nil, ErrChecksumMismatch + } + return nil, errors.New("decoded address is of unknown format") + } + switch len(decoded) { + case ripemd160.Size: // P2PKH or P2SH + isP2PKH := bytes.Equal(netID, checkID.AddressPubKeyHash) + isP2SH := bytes.Equal(netID, checkID.AddressScriptHash) + switch hash160 := decoded; { + case isP2PKH && isP2SH: + return nil, ErrAddressCollision + case isP2PKH: + return newAddressPubKeyHash(hash160, defaultNet) + case isP2SH: + return newAddressScriptHashFromHash(hash160, defaultNet) + default: + return nil, ErrUnknownAddressType + } + + default: + return nil, errors.New("decoded address is of unknown size") + } +} + +// AddressPubKeyHash is an Address for a pay-to-pubkey-hash (P2PKH) +// transaction. +type AddressPubKeyHash struct { + hash [ripemd160.Size]byte + netID []byte +} + +// NewAddressPubKeyHash returns a new AddressPubKeyHash. pkHash mustbe 20 +// bytes. +func NewAddressPubKeyHash(pkHash []byte, net *chaincfg.Params) (*AddressPubKeyHash, error) { + return newAddressPubKeyHash(pkHash, net) +} + +// newAddressPubKeyHash is the internal API to create a pubkey hash address +// with a known leading identifier byte for a network, rather than looking +// it up through its parameters. This is useful when creating a new address +// structure from a string encoding where the identifer byte is already +// known. +func newAddressPubKeyHash(pkHash []byte, net *chaincfg.Params) (*AddressPubKeyHash, error) { + // Check for a valid pubkey hash length. + if len(pkHash) != ripemd160.Size { + return nil, errors.New("pkHash must be 20 bytes") + } + + netID, ok := NetIDs[net.Name] + if !ok { + return nil, errors.New("unknown network parameters") + } + + addr := &AddressPubKeyHash{netID: netID.AddressPubKeyHash} + copy(addr.hash[:], pkHash) + return addr, nil +} + +// EncodeAddress returns the string encoding of a pay-to-pubkey-hash +// address. Part of the Address interface. +func (a *AddressPubKeyHash) EncodeAddress() string { + return encodeAddress(a.hash[:], a.netID) +} + +// ScriptAddress returns the bytes to be included in a txout script to pay +// to a pubkey hash. Part of the Address interface. +func (a *AddressPubKeyHash) ScriptAddress() []byte { + return a.hash[:] +} + +// IsForNet returns whether or not the pay-to-pubkey-hash address is associated +// with the passed zcash network. +func (a *AddressPubKeyHash) IsForNet(net *chaincfg.Params) bool { + checkID, ok := NetIDs[net.Name] + if !ok { + return false + } + return bytes.Equal(a.netID, checkID.AddressPubKeyHash) +} + +// String returns a human-readable string for the pay-to-pubkey-hash address. +// This is equivalent to calling EncodeAddress, but is provided so the type can +// be used as a fmt.Stringer. +func (a *AddressPubKeyHash) String() string { + return a.EncodeAddress() +} + +// Hash160 returns the underlying array of the pubkey hash. This can be useful +// when an array is more appropiate than a slice (for example, when used as map +// keys). +func (a *AddressPubKeyHash) Hash160() *[ripemd160.Size]byte { + return &a.hash +} + +// AddressScriptHash is an Address for a pay-to-script-hash (P2SH) +// transaction. +type AddressScriptHash struct { + hash [ripemd160.Size]byte + netID []byte +} + +// NewAddressScriptHash returns a new AddressScriptHash. +func NewAddressScriptHash(serializedScript []byte, net *chaincfg.Params) (*AddressScriptHash, error) { + scriptHash := btcutil.Hash160(serializedScript) + return newAddressScriptHashFromHash(scriptHash, net) +} + +// NewAddressScriptHashFromHash returns a new AddressScriptHash. scriptHash +// must be 20 bytes. +func NewAddressScriptHashFromHash(scriptHash []byte, net *chaincfg.Params) (*AddressScriptHash, error) { + return newAddressScriptHashFromHash(scriptHash, net) +} + +// newAddressScriptHashFromHash is the internal API to create a script hash +// address with a known leading identifier byte for a network, rather than +// looking it up through its parameters. This is useful when creating a new +// address structure from a string encoding where the identifer byte is already +// known. +func newAddressScriptHashFromHash(scriptHash []byte, net *chaincfg.Params) (*AddressScriptHash, error) { + // Check for a valid script hash length. + if len(scriptHash) != ripemd160.Size { + return nil, errors.New("scriptHash must be 20 bytes") + } + + netID, ok := NetIDs[net.Name] + if !ok { + return nil, errors.New("unknown network parameters") + } + + addr := &AddressScriptHash{netID: netID.AddressScriptHash} + copy(addr.hash[:], scriptHash) + return addr, nil +} + +// EncodeAddress returns the string encoding of a pay-to-script-hash +// address. Part of the Address interface. +func (a *AddressScriptHash) EncodeAddress() string { + return encodeAddress(a.hash[:], a.netID) +} + +// ScriptAddress returns the bytes to be included in a txout script to pay +// to a script hash. Part of the Address interface. +func (a *AddressScriptHash) ScriptAddress() []byte { + return a.hash[:] +} + +// IsForNet returns whether or not the pay-to-script-hash address is associated +// with the passed zcash network. +func (a *AddressScriptHash) IsForNet(net *chaincfg.Params) bool { + checkID, ok := NetIDs[net.Name] + if !ok { + return false + } + return bytes.Equal(a.netID, checkID.AddressScriptHash) +} + +// String returns a human-readable string for the pay-to-script-hash address. +// This is equivalent to calling EncodeAddress, but is provided so the type can +// be used as a fmt.Stringer. +func (a *AddressScriptHash) String() string { + return a.EncodeAddress() +} + +// Hash160 returns the underlying array of the script hash. This can be useful +// when an array is more appropiate than a slice (for example, when used as map +// keys). +func (a *AddressScriptHash) Hash160() *[ripemd160.Size]byte { + return &a.hash +} + +// PayToAddrScript creates a new script to pay a transaction output to a the +// specified address. +func PayToAddrScript(addr btcutil.Address) ([]byte, error) { + const nilAddrErrStr = "unable to generate payment script for nil address" + + switch addr := addr.(type) { + case *AddressPubKeyHash: + if addr == nil { + return nil, errors.New(nilAddrErrStr) + } + return payToPubKeyHashScript(addr.ScriptAddress()) + + case *AddressScriptHash: + if addr == nil { + return nil, errors.New(nilAddrErrStr) + } + return payToScriptHashScript(addr.ScriptAddress()) + } + return nil, fmt.Errorf("unable to generate payment script for unsupported "+ + "address type %T", addr) +} + +// payToPubKeyHashScript creates a new script to pay a transaction +// output to a 20-byte pubkey hash. It is expected that the input is a valid +// hash. +func payToPubKeyHashScript(pubKeyHash []byte) ([]byte, error) { + return txscript.NewScriptBuilder().AddOp(txscript.OP_DUP).AddOp(txscript.OP_HASH160). + AddData(pubKeyHash).AddOp(txscript.OP_EQUALVERIFY).AddOp(txscript.OP_CHECKSIG). + Script() +} + +// payToScriptHashScript creates a new script to pay a transaction output to a +// script hash. It is expected that the input is a valid hash. +func payToScriptHashScript(scriptHash []byte) ([]byte, error) { + return txscript.NewScriptBuilder().AddOp(txscript.OP_HASH160).AddData(scriptHash). + AddOp(txscript.OP_EQUAL).Script() +} + +// ExtractPkScriptAddrs returns the type of script, addresses and required +// signatures associated with the passed PkScript. Note that it only works for +// 'standard' transaction script types. Any data such as public keys which are +// invalid are omitted from the results. +func ExtractPkScriptAddrs(pkScript []byte, chainParams *chaincfg.Params) (btcutil.Address, error) { + // No valid addresses or required signatures if the script doesn't + // parse. + if len(pkScript) == 1+1+20+1 && pkScript[0] == 0xa9 && pkScript[1] == 0x14 && pkScript[22] == 0x87 { + return NewAddressScriptHashFromHash(pkScript[2:22], chainParams) + } else if len(pkScript) == 1+1+1+20+1+1 && pkScript[0] == 0x76 && pkScript[1] == 0xa9 && pkScript[2] == 0x14 && pkScript[23] == 0x88 && pkScript[24] == 0xac { + return NewAddressPubKeyHash(pkScript[3:23], chainParams) + } + return nil, errors.New("unknown script type") +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/zcash/sign.go b/vendor/github.com/OpenBazaar/multiwallet/zcash/sign.go new file mode 100644 index 0000000000..5f0a3b9de7 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/zcash/sign.go @@ -0,0 +1,540 @@ +package zcash + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/OpenBazaar/spvwallet" + wi "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/btcec" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + btc "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcutil/coinset" + hd "github.com/btcsuite/btcutil/hdkeychain" + "github.com/btcsuite/btcutil/txsort" + "github.com/btcsuite/btcwallet/wallet/txauthor" + "github.com/btcsuite/btcwallet/wallet/txrules" + + "github.com/OpenBazaar/multiwallet/util" + zaddr "github.com/OpenBazaar/multiwallet/zcash/address" +) + +func (w *ZCashWallet) buildTx(amount int64, addr btc.Address, feeLevel wi.FeeLevel, optionalOutput *wire.TxOut) (*wire.MsgTx, error) { + // Check for dust + script, err := zaddr.PayToAddrScript(addr) + if err != nil { + return nil, err + } + if txrules.IsDustAmount(btc.Amount(amount), len(script), txrules.DefaultRelayFeePerKb) { + return nil, wi.ErrorDustAmount + } + + var additionalPrevScripts map[wire.OutPoint][]byte + var additionalKeysByAddress map[string]*btc.WIF + + // Create input source + height, _ := w.ws.ChainTip() + utxos, err := w.db.Utxos().GetAll() + if err != nil { + return nil, err + } + coinMap := util.GatherCoins(height, utxos, w.ScriptToAddress, w.km.GetKeyForScript) + + coins := make([]coinset.Coin, 0, len(coinMap)) + for k := range coinMap { + coins = append(coins, k) + } + inputSource := func(target btc.Amount) (total btc.Amount, inputs []*wire.TxIn, inputValues []btc.Amount, scripts [][]byte, err error) { + coinSelector := coinset.MaxValueAgeCoinSelector{MaxInputs: 10000, MinChangeAmount: btc.Amount(0)} + coins, err := coinSelector.CoinSelect(target, coins) + if err != nil { + return total, inputs, inputValues, scripts, wi.ErrorInsuffientFunds + } + additionalPrevScripts = make(map[wire.OutPoint][]byte) + additionalKeysByAddress = make(map[string]*btc.WIF) + for _, c := range coins.Coins() { + total += c.Value() + outpoint := wire.NewOutPoint(c.Hash(), c.Index()) + in := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + in.Sequence = 0 // Opt-in RBF so we can bump fees + inputs = append(inputs, in) + additionalPrevScripts[*outpoint] = c.PkScript() + key := coinMap[c] + addr, err := key.Address(w.params) + if err != nil { + continue + } + privKey, err := key.ECPrivKey() + if err != nil { + continue + } + wif, _ := btc.NewWIF(privKey, w.params, true) + additionalKeysByAddress[addr.EncodeAddress()] = wif + } + return total, inputs, inputValues, scripts, nil + } + + // Get the fee per kilobyte + feePerKB := int64(w.GetFeePerByte(feeLevel)) * 1000 + + // outputs + out := wire.NewTxOut(amount, script) + + // Create change source + changeSource := func() ([]byte, error) { + addr := w.CurrentAddress(wi.INTERNAL) + script, err := zaddr.PayToAddrScript(addr) + if err != nil { + return []byte{}, err + } + return script, nil + } + + outputs := []*wire.TxOut{out} + if optionalOutput != nil { + outputs = append(outputs, optionalOutput) + } + authoredTx, err := newUnsignedTransaction(outputs, btc.Amount(feePerKB), inputSource, changeSource) + if err != nil { + return nil, err + } + + // BIP 69 sorting + txsort.InPlaceSort(authoredTx.Tx) + + // Sign tx + getKey := txscript.KeyClosure(func(addr btc.Address) (*btcec.PrivateKey, bool, error) { + addrStr := addr.EncodeAddress() + wif := additionalKeysByAddress[addrStr] + return wif.PrivKey, wif.CompressPubKey, nil + }) + getScript := txscript.ScriptClosure(func( + addr btc.Address) ([]byte, error) { + return []byte{}, nil + }) + for i, txIn := range authoredTx.Tx.TxIn { + prevOutScript := additionalPrevScripts[txIn.PreviousOutPoint] + script, err := txscript.SignTxOutput(w.params, + authoredTx.Tx, i, prevOutScript, txscript.SigHashAll, getKey, + getScript, txIn.SignatureScript) + if err != nil { + return nil, errors.New("Failed to sign transaction") + } + txIn.SignatureScript = script + } + return authoredTx.Tx, nil +} + +func newUnsignedTransaction(outputs []*wire.TxOut, feePerKb btc.Amount, fetchInputs txauthor.InputSource, fetchChange txauthor.ChangeSource) (*txauthor.AuthoredTx, error) { + + var targetAmount btc.Amount + for _, txOut := range outputs { + targetAmount += btc.Amount(txOut.Value) + } + + estimatedSize := EstimateSerializeSize(1, outputs, true, P2PKH) + targetFee := txrules.FeeForSerializeSize(feePerKb, estimatedSize) + + for { + inputAmount, inputs, _, scripts, err := fetchInputs(targetAmount + targetFee) + if err != nil { + return nil, err + } + if inputAmount < targetAmount+targetFee { + return nil, errors.New("insufficient funds available to construct transaction") + } + + maxSignedSize := EstimateSerializeSize(len(inputs), outputs, true, P2PKH) + maxRequiredFee := txrules.FeeForSerializeSize(feePerKb, maxSignedSize) + remainingAmount := inputAmount - targetAmount + if remainingAmount < maxRequiredFee { + targetFee = maxRequiredFee + continue + } + + unsignedTransaction := &wire.MsgTx{ + Version: wire.TxVersion, + TxIn: inputs, + TxOut: outputs, + LockTime: 0, + } + changeIndex := -1 + changeAmount := inputAmount - targetAmount - maxRequiredFee + if changeAmount != 0 && !txrules.IsDustAmount(changeAmount, + P2PKHOutputSize, txrules.DefaultRelayFeePerKb) { + changeScript, err := fetchChange() + if err != nil { + return nil, err + } + if len(changeScript) > P2PKHPkScriptSize { + return nil, errors.New("fee estimation requires change " + + "scripts no larger than P2PKH output scripts") + } + change := wire.NewTxOut(int64(changeAmount), changeScript) + l := len(outputs) + unsignedTransaction.TxOut = append(outputs[:l:l], change) + changeIndex = l + } + + return &txauthor.AuthoredTx{ + Tx: unsignedTransaction, + PrevScripts: scripts, + TotalInput: inputAmount, + ChangeIndex: changeIndex, + }, nil + } +} + +func (w *ZCashWallet) bumpFee(txid chainhash.Hash) (*chainhash.Hash, error) { + txn, err := w.db.Txns().Get(txid) + if err != nil { + return nil, err + } + if txn.Height > 0 { + return nil, spvwallet.BumpFeeAlreadyConfirmedError + } + if txn.Height < 0 { + return nil, spvwallet.BumpFeeTransactionDeadError + } + // Check utxos for CPFP + utxos, _ := w.db.Utxos().GetAll() + for _, u := range utxos { + if u.Op.Hash.IsEqual(&txid) && u.AtHeight == 0 { + addr, err := w.ScriptToAddress(u.ScriptPubkey) + if err != nil { + return nil, err + } + key, err := w.km.GetKeyForScript(addr.ScriptAddress()) + if err != nil { + return nil, err + } + h, err := hex.DecodeString(u.Op.Hash.String()) + if err != nil { + return nil, err + } + in := wi.TransactionInput{ + LinkedAddress: addr, + OutpointIndex: u.Op.Index, + OutpointHash: h, + Value: int64(u.Value), + } + transactionID, err := w.sweepAddress([]wi.TransactionInput{in}, nil, key, nil, wi.FEE_BUMP) + if err != nil { + return nil, err + } + return transactionID, nil + } + } + return nil, spvwallet.BumpFeeNotFoundError +} + +func (w *ZCashWallet) sweepAddress(ins []wi.TransactionInput, address *btc.Address, key *hd.ExtendedKey, redeemScript *[]byte, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + var internalAddr btc.Address + if address != nil { + internalAddr = *address + } else { + internalAddr = w.CurrentAddress(wi.INTERNAL) + } + script, err := zaddr.PayToAddrScript(internalAddr) + if err != nil { + return nil, err + } + + var val int64 + var inputs []*wire.TxIn + additionalPrevScripts := make(map[wire.OutPoint][]byte) + for _, in := range ins { + val += in.Value + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return nil, err + } + script, err := zaddr.PayToAddrScript(in.LinkedAddress) + if err != nil { + return nil, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + inputs = append(inputs, input) + additionalPrevScripts[*outpoint] = script + } + out := wire.NewTxOut(val, script) + + txType := P2PKH + if redeemScript != nil { + txType = P2SH_1of2_Multisig + _, err := spvwallet.LockTimeFromRedeemScript(*redeemScript) + if err == nil { + txType = P2SH_Multisig_Timelock_1Sig + } + } + estimatedSize := EstimateSerializeSize(len(ins), []*wire.TxOut{out}, false, txType) + + // Calculate the fee + feePerByte := int(w.GetFeePerByte(feeLevel)) + fee := estimatedSize * feePerByte + + outVal := val - int64(fee) + if outVal < 0 { + outVal = 0 + } + out.Value = outVal + + tx := &wire.MsgTx{ + Version: wire.TxVersion, + TxIn: inputs, + TxOut: []*wire.TxOut{out}, + LockTime: 0, + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + // Sign tx + privKey, err := key.ECPrivKey() + if err != nil { + return nil, err + } + pk := privKey.PubKey().SerializeCompressed() + addressPub, err := btc.NewAddressPubKey(pk, w.params) + + getKey := txscript.KeyClosure(func(addr btc.Address) (*btcec.PrivateKey, bool, error) { + if addressPub.EncodeAddress() == addr.EncodeAddress() { + wif, err := btc.NewWIF(privKey, w.params, true) + if err != nil { + return nil, false, err + } + return wif.PrivKey, wif.CompressPubKey, nil + } + return nil, false, errors.New("Not found") + }) + getScript := txscript.ScriptClosure(func(addr btc.Address) ([]byte, error) { + if redeemScript == nil { + return []byte{}, nil + } + return *redeemScript, nil + }) + + for i, txIn := range tx.TxIn { + prevOutScript := additionalPrevScripts[txIn.PreviousOutPoint] + script, err := txscript.SignTxOutput(w.params, + tx, i, prevOutScript, txscript.SigHashAll, getKey, + getScript, txIn.SignatureScript) + if err != nil { + return nil, errors.New("Failed to sign transaction") + } + txIn.SignatureScript = script + } + + // broadcast + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + _, err = w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + txid := tx.TxHash() + return &txid, nil +} + +func (w *ZCashWallet) createMultisigSignature(ins []wi.TransactionInput, outs []wi.TransactionOutput, key *hd.ExtendedKey, redeemScript []byte, feePerByte uint64) ([]wi.Signature, error) { + var sigs []wi.Signature + tx := wire.NewMsgTx(1) + for _, in := range ins { + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return sigs, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + tx.TxIn = append(tx.TxIn, input) + } + for _, out := range outs { + scriptPubkey, err := zaddr.PayToAddrScript(out.Address) + if err != nil { + return sigs, err + } + output := wire.NewTxOut(out.Value, scriptPubkey) + tx.TxOut = append(tx.TxOut, output) + } + + // Subtract fee + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, P2SH_2of3_Multisig) + fee := estimatedSize * int(feePerByte) + if len(tx.TxOut) > 0 { + feePerOutput := fee / len(tx.TxOut) + for _, output := range tx.TxOut { + output.Value -= int64(feePerOutput) + } + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + signingKey, err := key.ECPrivKey() + if err != nil { + return sigs, err + } + + for i := range tx.TxIn { + sig, err := txscript.RawTxInSignature(tx, i, redeemScript, txscript.SigHashAll, signingKey) + if err != nil { + continue + } + bs := wi.Signature{InputIndex: uint32(i), Signature: sig} + sigs = append(sigs, bs) + } + return sigs, nil +} + +func (w *ZCashWallet) multisign(ins []wi.TransactionInput, outs []wi.TransactionOutput, sigs1 []wi.Signature, sigs2 []wi.Signature, redeemScript []byte, feePerByte uint64, broadcast bool) ([]byte, error) { + tx := wire.NewMsgTx(1) + for _, in := range ins { + ch, err := chainhash.NewHashFromStr(hex.EncodeToString(in.OutpointHash)) + if err != nil { + return nil, err + } + outpoint := wire.NewOutPoint(ch, in.OutpointIndex) + input := wire.NewTxIn(outpoint, []byte{}, [][]byte{}) + tx.TxIn = append(tx.TxIn, input) + } + for _, out := range outs { + scriptPubkey, err := zaddr.PayToAddrScript(out.Address) + if err != nil { + return nil, err + } + output := wire.NewTxOut(out.Value, scriptPubkey) + tx.TxOut = append(tx.TxOut, output) + } + + // Subtract fee + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, P2SH_2of3_Multisig) + fee := estimatedSize * int(feePerByte) + if len(tx.TxOut) > 0 { + feePerOutput := fee / len(tx.TxOut) + for _, output := range tx.TxOut { + output.Value -= int64(feePerOutput) + } + } + + // BIP 69 sorting + txsort.InPlaceSort(tx) + + for i, input := range tx.TxIn { + var sig1 []byte + var sig2 []byte + for _, sig := range sigs1 { + if int(sig.InputIndex) == i { + sig1 = sig.Signature + break + } + } + for _, sig := range sigs2 { + if int(sig.InputIndex) == i { + sig2 = sig.Signature + break + } + } + + builder := txscript.NewScriptBuilder() + builder.AddOp(txscript.OP_0) + builder.AddData(sig1) + builder.AddData(sig2) + builder.AddData(redeemScript) + scriptSig, err := builder.Script() + if err != nil { + return nil, err + } + input.SignatureScript = scriptSig + } + // broadcast + if broadcast { + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + _, err := w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + } + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + return buf.Bytes(), nil +} + +func (w *ZCashWallet) generateMultisigScript(keys []hd.ExtendedKey, threshold int, timeout time.Duration, timeoutKey *hd.ExtendedKey) (addr btc.Address, redeemScript []byte, err error) { + if uint32(timeout.Hours()) > 0 && timeoutKey == nil { + return nil, nil, errors.New("Timeout key must be non nil when using an escrow timeout") + } + + if len(keys) < threshold { + return nil, nil, fmt.Errorf("unable to generate multisig script with "+ + "%d required signatures when there are only %d public "+ + "keys available", threshold, len(keys)) + } + + var ecKeys []*btcec.PublicKey + for _, key := range keys { + ecKey, err := key.ECPubKey() + if err != nil { + return nil, nil, err + } + ecKeys = append(ecKeys, ecKey) + } + + builder := txscript.NewScriptBuilder() + builder.AddInt64(int64(threshold)) + for _, key := range ecKeys { + builder.AddData(key.SerializeCompressed()) + } + builder.AddInt64(int64(len(ecKeys))) + builder.AddOp(txscript.OP_CHECKMULTISIG) + + redeemScript, err = builder.Script() + if err != nil { + return nil, nil, err + } + + addr, err = zaddr.NewAddressScriptHash(redeemScript, w.params) + if err != nil { + return nil, nil, err + } + return addr, redeemScript, nil +} + +func (w *ZCashWallet) estimateSpendFee(amount int64, feeLevel wi.FeeLevel) (uint64, error) { + // Since this is an estimate we can use a dummy output address. Let's use a long one so we don't under estimate. + addr, err := zaddr.DecodeAddress("t1hASvMj8e6TXWryuB3L5TKXJB7XfNioZP3", w.params) + if err != nil { + return 0, err + } + tx, err := w.buildTx(amount, addr, feeLevel, nil) + if err != nil { + return 0, err + } + var outval int64 + for _, output := range tx.TxOut { + outval += output.Value + } + var inval int64 + utxos, err := w.db.Utxos().GetAll() + if err != nil { + return 0, err + } + for _, input := range tx.TxIn { + for _, utxo := range utxos { + if utxo.Op.Hash.IsEqual(&input.PreviousOutPoint.Hash) && utxo.Op.Index == input.PreviousOutPoint.Index { + inval += utxo.Value + break + } + } + } + if inval < outval { + return 0, errors.New("Error building transaction: inputs less than outputs") + } + return uint64(inval - outval), err +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/zcash/txsizes.go b/vendor/github.com/OpenBazaar/multiwallet/zcash/txsizes.go new file mode 100644 index 0000000000..c66fe0b8d0 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/zcash/txsizes.go @@ -0,0 +1,249 @@ +package zcash + +// Copyright (c) 2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* Copied here from a btcd internal package*/ + +import ( + "github.com/btcsuite/btcd/wire" +) + +// Worst case script and input/output size estimates. +const ( + // RedeemP2PKHSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2PKH output. + // It is calculated as: + // + // - OP_DATA_73 + // - 72 bytes DER signature + 1 byte sighash + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + RedeemP2PKHSigScriptSize = 1 + 73 + 1 + 33 + + // RedeemP2SHMultisigSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a 2 of 3 P2SH multisig output with compressed keys. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_PUSHDATA + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + RedeemP2SH2of3MultisigSigScriptSize = 1 + 1 + 72 + 1 + 72 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + + // RedeemP2SH1of2MultisigSigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a 1 of 2 P2SH multisig output with compressed keys. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_PUSHDATA + // - OP_1 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP2 + // - OP_CHECKMULTISIG + RedeemP2SH1of2MultisigSigScriptSize = 1 + 1 + 72 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 1 + + // RedeemP2SHMultisigTimelock1SigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2SH timelocked multisig using the timeout. + // It is calculated as: + // + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_0 + // - OP_PUSHDATA + // - OP_IF + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + // - OP_ELSE + // - OP_PUSHDATA + // - 2 byte block height + // - OP_CHECKSEQUENCEVERIFY + // - OP_DROP + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_CHECKSIG + // - OP_ENDIF + RedeemP2SHMultisigTimelock1SigScriptSize = 1 + 72 + 1 + 1 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + 1 + 1 + 2 + 1 + 1 + 1 + 33 + 1 + 1 + + // RedeemP2SHMultisigTimelock2SigScriptSize is the worst case (largest) serialize size + // of a transaction input script that redeems a compressed P2SH timelocked multisig without using the timeout. + // It is calculated as: + // + // - OP_0 + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_DATA_72 + // - 72 bytes DER signature + // - OP_1 + // - OP_PUSHDATA + // - OP_IF + // - OP_2 + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP3 + // - OP_CHECKMULTISIG + // - OP_ELSE + // - OP_PUSHDATA + // - 2 byte block height + // - OP_CHECKSEQUENCEVERIFY + // - OP_DROP + // - OP_DATA_33 + // - 33 bytes serialized compressed pubkey + // - OP_CHECKSIG + // - OP_ENDIF + RedeemP2SHMultisigTimelock2SigScriptSize = 1 + 1 + 72 + +1 + 72 + 1 + 1 + 1 + 1 + 1 + 33 + 1 + 33 + 1 + 33 + 1 + 1 + 1 + 1 + 2 + 1 + 1 + 1 + 33 + 1 + 1 + + // P2PKHPkScriptSize is the size of a transaction output script that + // pays to a compressed pubkey hash. It is calculated as: + // + // - OP_DUP + // - OP_HASH160 + // - OP_DATA_20 + // - 20 bytes pubkey hash + // - OP_EQUALVERIFY + // - OP_CHECKSIG + P2PKHPkScriptSize = 1 + 1 + 1 + 20 + 1 + 1 + + // RedeemP2PKHInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2PKH output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - signature script + // - 4 bytes sequence + RedeemP2PKHInputSize = 32 + 4 + 1 + RedeemP2PKHSigScriptSize + 4 + + // RedeemP2SH2of3MultisigInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH 2 of 3 multisig output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SH2of3MultisigInputSize = 32 + 4 + 1 + 4 + (RedeemP2SH2of3MultisigSigScriptSize / 4) + + // RedeemP2SH1of2MultisigInputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH 2 of 3 multisig output. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SH1of2MultisigInputSize = 32 + 4 + 1 + 4 + (RedeemP2SH1of2MultisigSigScriptSize / 4) + + // RedeemP2SHMultisigTimelock1InputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed p2sh timelocked multig output with using the timeout. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SHMultisigTimelock1InputSize = 32 + 4 + 1 + 4 + (RedeemP2SHMultisigTimelock1SigScriptSize / 4) + + // RedeemP2SHMultisigTimelock2InputSize is the worst case (largest) serialize size of a + // transaction input redeeming a compressed P2SH timelocked multisig output without using the timeout. It is + // calculated as: + // + // - 32 bytes previous tx + // - 4 bytes output index + // - 1 byte script len + // - 4 bytes sequence + /// - witness discounted signature script + RedeemP2SHMultisigTimelock2InputSize = 32 + 4 + 1 + 4 + (RedeemP2SHMultisigTimelock2SigScriptSize / 4) + + // P2PKHOutputSize is the serialize size of a transaction output with a + // P2PKH output script. It is calculated as: + // + // - 8 bytes output value + // - 1 byte compact int encoding value 25 + // - 25 bytes P2PKH output script + P2PKHOutputSize = 8 + 1 + P2PKHPkScriptSize +) + +type InputType int + +const ( + P2PKH InputType = iota + P2SH_1of2_Multisig + P2SH_2of3_Multisig + P2SH_Multisig_Timelock_1Sig + P2SH_Multisig_Timelock_2Sigs +) + +// EstimateSerializeSize returns a worst case serialize size estimate for a +// signed transaction that spends inputCount number of compressed P2PKH outputs +// and contains each transaction output from txOuts. The estimated size is +// incremented for an additional P2PKH change output if addChangeOutput is true. +func EstimateSerializeSize(inputCount int, txOuts []*wire.TxOut, addChangeOutput bool, inputType InputType) int { + changeSize := 0 + outputCount := len(txOuts) + if addChangeOutput { + changeSize = P2PKHOutputSize + outputCount++ + } + + var redeemScriptSize int + switch inputType { + case P2PKH: + redeemScriptSize = RedeemP2PKHInputSize + case P2SH_1of2_Multisig: + redeemScriptSize = RedeemP2SH1of2MultisigInputSize + case P2SH_2of3_Multisig: + redeemScriptSize = RedeemP2SH2of3MultisigInputSize + case P2SH_Multisig_Timelock_1Sig: + redeemScriptSize = RedeemP2SHMultisigTimelock1InputSize + case P2SH_Multisig_Timelock_2Sigs: + redeemScriptSize = RedeemP2SHMultisigTimelock2InputSize + } + + // 10 additional bytes are for version, locktime, and segwit flags + return 10 + wire.VarIntSerializeSize(uint64(inputCount)) + + wire.VarIntSerializeSize(uint64(outputCount)) + + inputCount*redeemScriptSize + + SumOutputSerializeSizes(txOuts) + + changeSize +} + +// SumOutputSerializeSizes sums up the serialized size of the supplied outputs. +func SumOutputSerializeSizes(outputs []*wire.TxOut) (serializeSize int) { + for _, txOut := range outputs { + serializeSize += txOut.SerializeSize() + } + return serializeSize +} diff --git a/vendor/github.com/OpenBazaar/multiwallet/zcash/wallet.go b/vendor/github.com/OpenBazaar/multiwallet/zcash/wallet.go new file mode 100644 index 0000000000..53c5fde394 --- /dev/null +++ b/vendor/github.com/OpenBazaar/multiwallet/zcash/wallet.go @@ -0,0 +1,302 @@ +package zcash + +import ( + "bytes" + "fmt" + "io" + "time" + + wi "github.com/OpenBazaar/wallet-interface" + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + hd "github.com/btcsuite/btcutil/hdkeychain" + "github.com/btcsuite/btcwallet/wallet/txrules" + bcw "github.com/cpacia/BitcoinCash-Wallet" + er "github.com/cpacia/BitcoinCash-Wallet/exchangerates" + "github.com/tyler-smith/go-bip39" + "golang.org/x/net/proxy" + + "github.com/OpenBazaar/multiwallet/client" + "github.com/OpenBazaar/multiwallet/config" + "github.com/OpenBazaar/multiwallet/keys" + "github.com/OpenBazaar/multiwallet/service" + "github.com/OpenBazaar/multiwallet/util" + zaddr "github.com/OpenBazaar/multiwallet/zcash/address" +) + +type ZCashWallet struct { + db wi.Datastore + km *keys.KeyManager + params *chaincfg.Params + client client.APIClient + ws *service.WalletService + fp *bcw.FeeProvider + + mPrivKey *hd.ExtendedKey + mPubKey *hd.ExtendedKey +} + +func NewZCashWallet(cfg config.CoinConfig, mnemonic string, params *chaincfg.Params, proxy proxy.Dialer) (*ZCashWallet, error) { + seed := bip39.NewSeed(mnemonic, "") + + mPrivKey, err := hd.NewMaster(seed, params) + if err != nil { + return nil, err + } + mPubKey, err := mPrivKey.Neuter() + if err != nil { + return nil, err + } + km, err := keys.NewKeyManager(cfg.DB.Keys(), params, mPrivKey, wi.Zcash, zcashCashAddress) + if err != nil { + return nil, err + } + + c, err := client.NewInsightClient(cfg.ClientAPI.String(), proxy) + if err != nil { + return nil, err + } + + wm := service.NewWalletService(cfg.DB, km, c, params, wi.Zcash) + + // TODO: create zcash fee provider + fp := bcw.NewFeeProvider(cfg.MaxFee, cfg.HighFee, cfg.MediumFee, cfg.LowFee, er.NewBitcoinCashPriceFetcher(proxy)) + + return &ZCashWallet{cfg.DB, km, params, c, wm, fp, mPrivKey, mPubKey}, nil +} + +func zcashCashAddress(key *hd.ExtendedKey, params *chaincfg.Params) (btcutil.Address, error) { + addr, err := key.Address(params) + if err != nil { + return nil, err + } + return zaddr.NewAddressPubKeyHash(addr.ScriptAddress(), params) +} + +func (w *ZCashWallet) Start() { + w.ws.Start() +} + +func (w *ZCashWallet) Params() *chaincfg.Params { + return w.params +} + +func (w *ZCashWallet) CurrencyCode() string { + if w.params.Name == chaincfg.MainNetParams.Name { + return "zec" + } else { + return "tzec" + } +} + +func (w *ZCashWallet) IsDust(amount int64) bool { + return txrules.IsDustAmount(btcutil.Amount(amount), 25, txrules.DefaultRelayFeePerKb) +} + +func (w *ZCashWallet) MasterPrivateKey() *hd.ExtendedKey { + return w.mPrivKey +} + +func (w *ZCashWallet) MasterPublicKey() *hd.ExtendedKey { + return w.mPubKey +} + +func (w *ZCashWallet) ChildKey(keyBytes []byte, chaincode []byte, isPrivateKey bool) (*hd.ExtendedKey, error) { + parentFP := []byte{0x00, 0x00, 0x00, 0x00} + var id []byte + if isPrivateKey { + id = w.params.HDPrivateKeyID[:] + } else { + id = w.params.HDPublicKeyID[:] + } + hdKey := hd.NewExtendedKey( + id, + keyBytes, + chaincode, + parentFP, + 0, + 0, + isPrivateKey) + return hdKey.Child(0) +} + +func (w *ZCashWallet) CurrentAddress(purpose wi.KeyPurpose) btcutil.Address { + key, _ := w.km.GetCurrentKey(purpose) + addr, _ := zcashCashAddress(key, w.params) + return btcutil.Address(addr) +} + +func (w *ZCashWallet) NewAddress(purpose wi.KeyPurpose) btcutil.Address { + i, _ := w.db.Keys().GetUnused(purpose) + key, _ := w.km.GenerateChildKey(purpose, uint32(i[1])) + addr, _ := zcashCashAddress(key, w.params) + w.db.Keys().MarkKeyAsUsed(addr.ScriptAddress()) + return btcutil.Address(addr) +} + +func (w *ZCashWallet) DecodeAddress(addr string) (btcutil.Address, error) { + return zaddr.DecodeAddress(addr, w.params) +} + +func (w *ZCashWallet) ScriptToAddress(script []byte) (btcutil.Address, error) { + addr, err := zaddr.ExtractPkScriptAddrs(script, w.params) + if err != nil { + return nil, err + } + return addr, nil +} + +func (w *ZCashWallet) AddressToScript(addr btcutil.Address) ([]byte, error) { + return zaddr.PayToAddrScript(addr) +} + +func (w *ZCashWallet) HasKey(addr btcutil.Address) bool { + _, err := w.km.GetKeyForScript(addr.ScriptAddress()) + if err != nil { + return false + } + return true +} + +func (w *ZCashWallet) Balance() (confirmed, unconfirmed int64) { + utxos, _ := w.db.Utxos().GetAll() + txns, _ := w.db.Txns().GetAll(false) + return util.CalcBalance(utxos, txns) +} + +func (w *ZCashWallet) Transactions() ([]wi.Txn, error) { + return w.db.Txns().GetAll(false) +} + +func (w *ZCashWallet) GetTransaction(txid chainhash.Hash) (wi.Txn, error) { + txn, err := w.db.Txns().Get(txid) + return txn, err +} + +func (w *ZCashWallet) ChainTip() (uint32, chainhash.Hash) { + return w.ws.ChainTip() +} + +func (w *ZCashWallet) GetFeePerByte(feeLevel wi.FeeLevel) uint64 { + return w.fp.GetFeePerByte(feeLevel) +} + +func (w *ZCashWallet) Spend(amount int64, addr btcutil.Address, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + tx, err := w.buildTx(amount, addr, feeLevel, nil) + if err != nil { + return nil, err + } + // Broadcast + var buf bytes.Buffer + tx.BtcEncode(&buf, wire.ProtocolVersion, wire.WitnessEncoding) + + _, err = w.client.Broadcast(buf.Bytes()) + if err != nil { + return nil, err + } + + ch := tx.TxHash() + return &ch, nil +} + +func (w *ZCashWallet) BumpFee(txid chainhash.Hash) (*chainhash.Hash, error) { + return w.bumpFee(txid) +} + +func (w *ZCashWallet) EstimateFee(ins []wi.TransactionInput, outs []wi.TransactionOutput, feePerByte uint64) uint64 { + tx := new(wire.MsgTx) + for _, out := range outs { + scriptPubKey, _ := zaddr.PayToAddrScript(out.Address) + output := wire.NewTxOut(out.Value, scriptPubKey) + tx.TxOut = append(tx.TxOut, output) + } + estimatedSize := EstimateSerializeSize(len(ins), tx.TxOut, false, P2PKH) + fee := estimatedSize * int(feePerByte) + return uint64(fee) +} + +func (w *ZCashWallet) EstimateSpendFee(amount int64, feeLevel wi.FeeLevel) (uint64, error) { + return w.estimateSpendFee(amount, feeLevel) +} + +func (w *ZCashWallet) SweepAddress(ins []wi.TransactionInput, address *btcutil.Address, key *hd.ExtendedKey, redeemScript *[]byte, feeLevel wi.FeeLevel) (*chainhash.Hash, error) { + return w.sweepAddress(ins, address, key, redeemScript, feeLevel) +} + +func (w *ZCashWallet) CreateMultisigSignature(ins []wi.TransactionInput, outs []wi.TransactionOutput, key *hd.ExtendedKey, redeemScript []byte, feePerByte uint64) ([]wi.Signature, error) { + return w.createMultisigSignature(ins, outs, key, redeemScript, feePerByte) +} + +func (w *ZCashWallet) Multisign(ins []wi.TransactionInput, outs []wi.TransactionOutput, sigs1 []wi.Signature, sigs2 []wi.Signature, redeemScript []byte, feePerByte uint64, broadcast bool) ([]byte, error) { + return w.multisign(ins, outs, sigs1, sigs2, redeemScript, feePerByte, broadcast) +} + +func (w *ZCashWallet) GenerateMultisigScript(keys []hd.ExtendedKey, threshold int, timeout time.Duration, timeoutKey *hd.ExtendedKey) (addr btcutil.Address, redeemScript []byte, err error) { + return w.generateMultisigScript(keys, threshold, timeout, timeoutKey) +} + +func (w *ZCashWallet) AddWatchedAddress(addr btcutil.Address) error { + script, err := w.AddressToScript(addr) + if err != nil { + return err + } + err = w.db.WatchedScripts().Put(script) + if err != nil { + return err + } + w.client.ListenAddress(addr) + return nil +} + +func (w *ZCashWallet) AddWatchedScript(script []byte) error { + err := w.db.WatchedScripts().Put(script) + if err != nil { + return err + } + addr, err := w.ScriptToAddress(script) + if err != nil { + return err + } + w.client.ListenAddress(addr) + return nil +} + +func (w *ZCashWallet) AddTransactionListener(callback func(wi.TransactionCallback)) { + w.ws.AddTransactionListener(callback) +} + +func (w *ZCashWallet) ReSyncBlockchain(fromTime time.Time) { + go w.ws.UpdateState() +} + +func (w *ZCashWallet) GetConfirmations(txid chainhash.Hash) (uint32, uint32, error) { + txn, err := w.db.Txns().Get(txid) + if err != nil { + return 0, 0, err + } + if txn.Height == 0 { + return 0, 0, nil + } + chainTip, _ := w.ChainTip() + return chainTip - uint32(txn.Height) + 1, uint32(txn.Height), nil +} + +func (w *ZCashWallet) Close() { + w.ws.Stop() + w.client.Close() +} + +func (w *ZCashWallet) DumpTables(wr io.Writer) { + fmt.Fprintln(wr, "Transactions-----") + txns, _ := w.db.Txns().GetAll(true) + for _, tx := range txns { + fmt.Fprintf(wr, "Hash: %s, Height: %d, Value: %d, WatchOnly: %t\n", tx.Txid, int(tx.Height), int(tx.Value), tx.WatchOnly) + } + fmt.Fprintln(wr, "\nUtxos-----") + utxos, _ := w.db.Utxos().GetAll() + for _, u := range utxos { + fmt.Fprintf(wr, "Hash: %s, Index: %d, Height: %d, Value: %d, WatchOnly: %t\n", u.Op.Hash.String(), int(u.Op.Index), int(u.AtHeight), int(u.Value), u.WatchOnly) + } +} diff --git a/vendor/github.com/btcsuite/golangcrypto/AUTHORS b/vendor/github.com/btcsuite/golangcrypto/AUTHORS new file mode 100644 index 0000000000..15167cd746 --- /dev/null +++ b/vendor/github.com/btcsuite/golangcrypto/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/vendor/github.com/btcsuite/golangcrypto/CONTRIBUTORS b/vendor/github.com/btcsuite/golangcrypto/CONTRIBUTORS new file mode 100644 index 0000000000..1c4577e968 --- /dev/null +++ b/vendor/github.com/btcsuite/golangcrypto/CONTRIBUTORS @@ -0,0 +1,3 @@ +# This source code was written by the Go contributors. +# The master list of contributors is in the main Go distribution, +# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/vendor/github.com/btcsuite/golangcrypto/LICENSE b/vendor/github.com/btcsuite/golangcrypto/LICENSE new file mode 100644 index 0000000000..6a66aea5ea --- /dev/null +++ b/vendor/github.com/btcsuite/golangcrypto/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/btcsuite/golangcrypto/PATENTS b/vendor/github.com/btcsuite/golangcrypto/PATENTS new file mode 100644 index 0000000000..733099041f --- /dev/null +++ b/vendor/github.com/btcsuite/golangcrypto/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/github.com/btcsuite/golangcrypto/ripemd160/ripemd160.go b/vendor/github.com/btcsuite/golangcrypto/ripemd160/ripemd160.go new file mode 100644 index 0000000000..da690f0b92 --- /dev/null +++ b/vendor/github.com/btcsuite/golangcrypto/ripemd160/ripemd160.go @@ -0,0 +1,120 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ripemd160 implements the RIPEMD-160 hash algorithm. +package ripemd160 + +// RIPEMD-160 is designed by by Hans Dobbertin, Antoon Bosselaers, and Bart +// Preneel with specifications available at: +// http://homes.esat.kuleuven.be/~cosicart/pdf/AB-9601/AB-9601.pdf. + +import ( + "crypto" + "hash" +) + +func init() { + crypto.RegisterHash(crypto.RIPEMD160, New) +} + +// The size of the checksum in bytes. +const Size = 20 + +// The block size of the hash algorithm in bytes. +const BlockSize = 64 + +const ( + _s0 = 0x67452301 + _s1 = 0xefcdab89 + _s2 = 0x98badcfe + _s3 = 0x10325476 + _s4 = 0xc3d2e1f0 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + s [5]uint32 // running context + x [BlockSize]byte // temporary buffer + nx int // index into x + tc uint64 // total count of bytes processed +} + +func (d *digest) Reset() { + d.s[0], d.s[1], d.s[2], d.s[3], d.s[4] = _s0, _s1, _s2, _s3, _s4 + d.nx = 0 + d.tc = 0 +} + +// New returns a new hash.Hash computing the checksum. +func New() hash.Hash { + result := new(digest) + result.Reset() + return result +} + +func (d *digest) Size() int { return Size } + +func (d *digest) BlockSize() int { return BlockSize } + +func (d *digest) Write(p []byte) (nn int, err error) { + nn = len(p) + d.tc += uint64(nn) + if d.nx > 0 { + n := len(p) + if n > BlockSize-d.nx { + n = BlockSize - d.nx + } + for i := 0; i < n; i++ { + d.x[d.nx+i] = p[i] + } + d.nx += n + if d.nx == BlockSize { + _Block(d, d.x[0:]) + d.nx = 0 + } + p = p[n:] + } + n := _Block(d, p) + p = p[n:] + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum(in []byte) []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := *d0 + + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + tc := d.tc + var tmp [64]byte + tmp[0] = 0x80 + if tc%64 < 56 { + d.Write(tmp[0 : 56-tc%64]) + } else { + d.Write(tmp[0 : 64+56-tc%64]) + } + + // Length in bits. + tc <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(tc >> (8 * i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + var digest [Size]byte + for i, s := range d.s { + digest[i*4] = byte(s) + digest[i*4+1] = byte(s >> 8) + digest[i*4+2] = byte(s >> 16) + digest[i*4+3] = byte(s >> 24) + } + + return append(in, digest[:]...) +} diff --git a/vendor/github.com/btcsuite/golangcrypto/ripemd160/ripemd160block.go b/vendor/github.com/btcsuite/golangcrypto/ripemd160/ripemd160block.go new file mode 100644 index 0000000000..7bc8e6c485 --- /dev/null +++ b/vendor/github.com/btcsuite/golangcrypto/ripemd160/ripemd160block.go @@ -0,0 +1,161 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// RIPEMD-160 block step. +// In its own file so that a faster assembly or C version +// can be substituted easily. + +package ripemd160 + +// work buffer indices and roll amounts for one line +var _n = [80]uint{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 7, 4, 13, 1, 10, 6, 15, 3, 12, 0, 9, 5, 2, 14, 11, 8, + 3, 10, 14, 4, 9, 15, 8, 1, 2, 7, 0, 6, 13, 11, 5, 12, + 1, 9, 11, 10, 0, 8, 12, 4, 13, 3, 7, 15, 14, 5, 6, 2, + 4, 0, 5, 9, 7, 12, 2, 10, 14, 1, 3, 8, 11, 6, 15, 13, +} + +var _r = [80]uint{ + 11, 14, 15, 12, 5, 8, 7, 9, 11, 13, 14, 15, 6, 7, 9, 8, + 7, 6, 8, 13, 11, 9, 7, 15, 7, 12, 15, 9, 11, 7, 13, 12, + 11, 13, 6, 7, 14, 9, 13, 15, 14, 8, 13, 6, 5, 12, 7, 5, + 11, 12, 14, 15, 14, 15, 9, 8, 9, 14, 5, 6, 8, 6, 5, 12, + 9, 15, 5, 11, 6, 8, 13, 12, 5, 12, 13, 14, 11, 8, 5, 6, +} + +// same for the other parallel one +var n_ = [80]uint{ + 5, 14, 7, 0, 9, 2, 11, 4, 13, 6, 15, 8, 1, 10, 3, 12, + 6, 11, 3, 7, 0, 13, 5, 10, 14, 15, 8, 12, 4, 9, 1, 2, + 15, 5, 1, 3, 7, 14, 6, 9, 11, 8, 12, 2, 10, 0, 4, 13, + 8, 6, 4, 1, 3, 11, 15, 0, 5, 12, 2, 13, 9, 7, 10, 14, + 12, 15, 10, 4, 1, 5, 8, 7, 6, 2, 13, 14, 0, 3, 9, 11, +} + +var r_ = [80]uint{ + 8, 9, 9, 11, 13, 15, 15, 5, 7, 7, 8, 11, 14, 14, 12, 6, + 9, 13, 15, 7, 12, 8, 9, 11, 7, 7, 12, 7, 6, 15, 13, 11, + 9, 7, 15, 11, 8, 6, 6, 14, 12, 13, 5, 14, 13, 13, 7, 5, + 15, 5, 8, 11, 14, 14, 6, 14, 6, 9, 12, 9, 12, 5, 15, 8, + 8, 5, 12, 9, 12, 5, 14, 6, 8, 13, 6, 5, 15, 13, 11, 11, +} + +func _Block(md *digest, p []byte) int { + n := 0 + var x [16]uint32 + var alpha, beta uint32 + for len(p) >= BlockSize { + a, b, c, d, e := md.s[0], md.s[1], md.s[2], md.s[3], md.s[4] + aa, bb, cc, dd, ee := a, b, c, d, e + j := 0 + for i := 0; i < 16; i++ { + x[i] = uint32(p[j]) | uint32(p[j+1])<<8 | uint32(p[j+2])<<16 | uint32(p[j+3])<<24 + j += 4 + } + + // round 1 + i := 0 + for i < 16 { + alpha = a + (b ^ c ^ d) + x[_n[i]] + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb ^ (cc | ^dd)) + x[n_[i]] + 0x50a28be6 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 2 + for i < 32 { + alpha = a + (b&c | ^b&d) + x[_n[i]] + 0x5a827999 + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb&dd | cc&^dd) + x[n_[i]] + 0x5c4dd124 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 3 + for i < 48 { + alpha = a + (b | ^c ^ d) + x[_n[i]] + 0x6ed9eba1 + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb | ^cc ^ dd) + x[n_[i]] + 0x6d703ef3 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 4 + for i < 64 { + alpha = a + (b&d | c&^d) + x[_n[i]] + 0x8f1bbcdc + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb&cc | ^bb&dd) + x[n_[i]] + 0x7a6d76e9 + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // round 5 + for i < 80 { + alpha = a + (b ^ (c | ^d)) + x[_n[i]] + 0xa953fd4e + s := _r[i] + alpha = (alpha<>(32-s)) + e + beta = c<<10 | c>>22 + a, b, c, d, e = e, alpha, b, beta, d + + // parallel line + alpha = aa + (bb ^ cc ^ dd) + x[n_[i]] + s = r_[i] + alpha = (alpha<>(32-s)) + ee + beta = cc<<10 | cc>>22 + aa, bb, cc, dd, ee = ee, alpha, bb, beta, dd + + i++ + } + + // combine results + dd += c + md.s[1] + md.s[1] = md.s[2] + d + ee + md.s[2] = md.s[3] + e + aa + md.s[3] = md.s[4] + a + bb + md.s[4] = md.s[0] + b + cc + md.s[0] = dd + + p = p[BlockSize:] + n += BlockSize + } + return n +} diff --git a/vendor/github.com/ipfs/go-ipfs/core/coreunix/add.go b/vendor/github.com/ipfs/go-ipfs/core/coreunix/add.go index 6a0a4a129a..a6518e72ea 100644 --- a/vendor/github.com/ipfs/go-ipfs/core/coreunix/add.go +++ b/vendor/github.com/ipfs/go-ipfs/core/coreunix/add.go @@ -363,9 +363,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { if err != nil { return "", err } - fileAdder.PinRoot() - return nd.String(), nil } diff --git a/vendor/github.com/ipfs/go-ipfs/namesys/pubsub.go b/vendor/github.com/ipfs/go-ipfs/namesys/pubsub.go index 4f25cf87e9..978d128866 100644 --- a/vendor/github.com/ipfs/go-ipfs/namesys/pubsub.go +++ b/vendor/github.com/ipfs/go-ipfs/namesys/pubsub.go @@ -28,8 +28,6 @@ import ( cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" ) -var ErrorNotSubscribed = errors.New("cache is stale because not subscribed to name") - // PubsubPublisher is a publisher that distributes IPNS records through pubsub type PubsubPublisher struct { ctx context.Context @@ -55,6 +53,8 @@ type PubsubResolver struct { subs map[string]*floodsub.Subscription } +var ErrorNotSubscribed = errors.New("cache is stale because not subscribed to name") + // NewPubsubPublisher constructs a new Publisher that publishes IPNS records through pubsub. // The constructor interface is complicated by the need to bootstrap the pubsub topic. // This could be greatly simplified if the pubsub implementation handled bootstrap itself diff --git a/vendor/github.com/ipfs/go-ipfs/namesys/routing.go b/vendor/github.com/ipfs/go-ipfs/namesys/routing.go index d7ddba1cbb..1ffcf3ab40 100644 --- a/vendor/github.com/ipfs/go-ipfs/namesys/routing.go +++ b/vendor/github.com/ipfs/go-ipfs/namesys/routing.go @@ -23,7 +23,7 @@ import ( ) var log = logging.Logger("namesys") -const keyCachePrefix = "/pubkey/" +var keyCachePrefix= "/pubkey/" // routingResolver implements NSResolver for the main IPFS SFS-like naming type routingResolver struct { @@ -279,4 +279,4 @@ func checkEOL(e *pb.IpnsEntry) (time.Time, bool) { func putToDatabase(datastore ds.Datastore, name string, ipnsRec, pubkey []byte) { datastore.Put(dshelp.NewKeyFromBinary([]byte(name)), ipnsRec) datastore.Put(ds.NewKey(keyCachePrefix+strings.TrimPrefix(name, "/ipns/")), pubkey) -} \ No newline at end of file +} diff --git a/vendor/github.com/ltcsuite/ltcd/LICENSE b/vendor/github.com/ltcsuite/ltcd/LICENSE new file mode 100644 index 0000000000..53ba0c5608 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/LICENSE @@ -0,0 +1,16 @@ +ISC License + +Copyright (c) 2013-2017 The btcsuite developers +Copyright (c) 2015-2016 The Decred developers + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/vendor/github.com/ltcsuite/ltcd/chaincfg/README.md b/vendor/github.com/ltcsuite/ltcd/chaincfg/README.md new file mode 100644 index 0000000000..b771a97a20 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/chaincfg/README.md @@ -0,0 +1,87 @@ +chaincfg +======== + +[![Build Status](http://img.shields.io/travis/ltcsuite/ltcd.svg)] +(https://travis-ci.org/ltcsuite/ltcd) [![ISC License] +(http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) +[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)] +(http://godoc.org/github.com/ltcsuite/ltcd/chaincfg) + +Package chaincfg defines chain configuration parameters for the three standard +Bitcoin networks and provides the ability for callers to define their own custom +Bitcoin networks. + +Although this package was primarily written for ltcd, it has intentionally been +designed so it can be used as a standalone package for any projects needing to +use parameters for the standard Bitcoin networks or for projects needing to +define their own network. + +## Sample Use + +```Go +package main + +import ( + "flag" + "fmt" + "log" + + "github.com/ltcsuite/ltcutil" + "github.com/ltcsuite/ltcd/chaincfg" +) + +var testnet = flag.Bool("testnet", false, "operate on the testnet Bitcoin network") + +// By default (without -testnet), use mainnet. +var chainParams = &chaincfg.MainNetParams + +func main() { + flag.Parse() + + // Modify active network parameters if operating on testnet. + if *testnet { + chainParams = &chaincfg.TestNet4Params + } + + // later... + + // Create and print new payment address, specific to the active network. + pubKeyHash := make([]byte, 20) + addr, err := ltcutil.NewAddressPubKeyHash(pubKeyHash, chainParams) + if err != nil { + log.Fatal(err) + } + fmt.Println(addr) +} +``` + +## Installation and Updating + +```bash +$ go get -u github.com/ltcsuite/ltcd/chaincfg +``` + +## GPG Verification Key + +All official release tags are signed by Conformal so users can ensure the code +has not been tampered with and is coming from the btcsuite developers. To +verify the signature perform the following: + +- Download the public key from the Conformal website at + https://opensource.conformal.com/GIT-GPG-KEY-conformal.txt + +- Import the public key into your GPG keyring: + ```bash + gpg --import GIT-GPG-KEY-conformal.txt + ``` + +- Verify the release tag with the following command where `TAG_NAME` is a + placeholder for the specific tag: + ```bash + git tag -v TAG_NAME + ``` + +## License + +Package chaincfg is licensed under the [copyfree](http://copyfree.org) ISC +License. diff --git a/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/README.md b/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/README.md new file mode 100644 index 0000000000..a86725e82e --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/README.md @@ -0,0 +1,40 @@ +chainhash +========= + +[![Build Status](http://img.shields.io/travis/ltcsuite/ltcd.svg)](https://travis-ci.org/ltcsuite/ltcd) +[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) +[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/ltcsuite/ltcd/chaincfg/chainhash) + +chainhash provides a generic hash type and associated functions that allows the +specific hash algorithm to be abstracted. + +## Installation and Updating + +```bash +$ go get -u github.com/ltcsuite/ltcd/chaincfg/chainhash +``` + +## GPG Verification Key + +All official release tags are signed by Conformal so users can ensure the code +has not been tampered with and is coming from the btcsuite developers. To +verify the signature perform the following: + +- Download the public key from the Conformal website at + https://opensource.conformal.com/GIT-GPG-KEY-conformal.txt + +- Import the public key into your GPG keyring: + ```bash + gpg --import GIT-GPG-KEY-conformal.txt + ``` + +- Verify the release tag with the following command where `TAG_NAME` is a + placeholder for the specific tag: + ```bash + git tag -v TAG_NAME + ``` + +## License + +Package chainhash is licensed under the [copyfree](http://copyfree.org) ISC +License. diff --git a/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/doc.go b/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/doc.go new file mode 100644 index 0000000000..c3eb43d321 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/doc.go @@ -0,0 +1,5 @@ +// Package chainhash provides abstracted hash functionality. +// +// This package provides a generic hash type and associated functions that +// allows the specific hash algorithm to be abstracted. +package chainhash diff --git a/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/hash.go b/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/hash.go new file mode 100644 index 0000000000..2b1cec022b --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/hash.go @@ -0,0 +1,128 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Copyright (c) 2015 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package chainhash + +import ( + "encoding/hex" + "fmt" +) + +// HashSize of array used to store hashes. See Hash. +const HashSize = 32 + +// MaxHashStringSize is the maximum length of a Hash hash string. +const MaxHashStringSize = HashSize * 2 + +// ErrHashStrSize describes an error that indicates the caller specified a hash +// string that has too many characters. +var ErrHashStrSize = fmt.Errorf("max hash string length is %v bytes", MaxHashStringSize) + +// Hash is used in several of the bitcoin messages and common structures. It +// typically represents the double sha256 of data. +type Hash [HashSize]byte + +// String returns the Hash as the hexadecimal string of the byte-reversed +// hash. +func (hash Hash) String() string { + for i := 0; i < HashSize/2; i++ { + hash[i], hash[HashSize-1-i] = hash[HashSize-1-i], hash[i] + } + return hex.EncodeToString(hash[:]) +} + +// CloneBytes returns a copy of the bytes which represent the hash as a byte +// slice. +// +// NOTE: It is generally cheaper to just slice the hash directly thereby reusing +// the same bytes rather than calling this method. +func (hash *Hash) CloneBytes() []byte { + newHash := make([]byte, HashSize) + copy(newHash, hash[:]) + + return newHash +} + +// SetBytes sets the bytes which represent the hash. An error is returned if +// the number of bytes passed in is not HashSize. +func (hash *Hash) SetBytes(newHash []byte) error { + nhlen := len(newHash) + if nhlen != HashSize { + return fmt.Errorf("invalid hash length of %v, want %v", nhlen, + HashSize) + } + copy(hash[:], newHash) + + return nil +} + +// IsEqual returns true if target is the same as hash. +func (hash *Hash) IsEqual(target *Hash) bool { + if hash == nil && target == nil { + return true + } + if hash == nil || target == nil { + return false + } + return *hash == *target +} + +// NewHash returns a new Hash from a byte slice. An error is returned if +// the number of bytes passed in is not HashSize. +func NewHash(newHash []byte) (*Hash, error) { + var sh Hash + err := sh.SetBytes(newHash) + if err != nil { + return nil, err + } + return &sh, err +} + +// NewHashFromStr creates a Hash from a hash string. The string should be +// the hexadecimal string of a byte-reversed hash, but any missing characters +// result in zero padding at the end of the Hash. +func NewHashFromStr(hash string) (*Hash, error) { + ret := new(Hash) + err := Decode(ret, hash) + if err != nil { + return nil, err + } + return ret, nil +} + +// Decode decodes the byte-reversed hexadecimal string encoding of a Hash to a +// destination. +func Decode(dst *Hash, src string) error { + // Return error if hash string is too long. + if len(src) > MaxHashStringSize { + return ErrHashStrSize + } + + // Hex decoder expects the hash to be a multiple of two. When not, pad + // with a leading zero. + var srcBytes []byte + if len(src)%2 == 0 { + srcBytes = []byte(src) + } else { + srcBytes = make([]byte, 1+len(src)) + srcBytes[0] = '0' + copy(srcBytes[1:], src) + } + + // Hex decode the source bytes to a temporary destination. + var reversedHash Hash + _, err := hex.Decode(reversedHash[HashSize-hex.DecodedLen(len(srcBytes)):], srcBytes) + if err != nil { + return err + } + + // Reverse copy from the temporary hash to destination. Because the + // temporary was zeroed, the written result will be correctly padded. + for i, b := range reversedHash[:HashSize/2] { + dst[i], dst[HashSize-1-i] = reversedHash[HashSize-1-i], b + } + + return nil +} diff --git a/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/hashfuncs.go b/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/hashfuncs.go new file mode 100644 index 0000000000..bf74f73c39 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/chaincfg/chainhash/hashfuncs.go @@ -0,0 +1,33 @@ +// Copyright (c) 2015 The Decred developers +// Copyright (c) 2016-2017 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package chainhash + +import "crypto/sha256" + +// HashB calculates hash(b) and returns the resulting bytes. +func HashB(b []byte) []byte { + hash := sha256.Sum256(b) + return hash[:] +} + +// HashH calculates hash(b) and returns the resulting bytes as a Hash. +func HashH(b []byte) Hash { + return Hash(sha256.Sum256(b)) +} + +// DoubleHashB calculates hash(hash(b)) and returns the resulting bytes. +func DoubleHashB(b []byte) []byte { + first := sha256.Sum256(b) + second := sha256.Sum256(first[:]) + return second[:] +} + +// DoubleHashH calculates hash(hash(b)) and returns the resulting bytes as a +// Hash. +func DoubleHashH(b []byte) Hash { + first := sha256.Sum256(b) + return Hash(sha256.Sum256(first[:])) +} diff --git a/vendor/github.com/ltcsuite/ltcd/chaincfg/doc.go b/vendor/github.com/ltcsuite/ltcd/chaincfg/doc.go new file mode 100644 index 0000000000..4acfba0409 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/chaincfg/doc.go @@ -0,0 +1,61 @@ +// Package chaincfg defines chain configuration parameters. +// +// In addition to the main Bitcoin network, which is intended for the transfer +// of monetary value, there also exists two currently active standard networks: +// regression test and testnet (version 3). These networks are incompatible +// with each other (each sharing a different genesis block) and software should +// handle errors where input intended for one network is used on an application +// instance running on a different network. +// +// For library packages, chaincfg provides the ability to lookup chain +// parameters and encoding magics when passed a *Params. Older APIs not updated +// to the new convention of passing a *Params may lookup the parameters for a +// wire.BitcoinNet using ParamsForNet, but be aware that this usage is +// deprecated and will be removed from chaincfg in the future. +// +// For main packages, a (typically global) var may be assigned the address of +// one of the standard Param vars for use as the application's "active" network. +// When a network parameter is needed, it may then be looked up through this +// variable (either directly, or hidden in a library call). +// +// package main +// +// import ( +// "flag" +// "fmt" +// "log" +// +// "github.com/ltcsuite/ltcutil" +// "github.com/ltcsuite/ltcd/chaincfg" +// ) +// +// var testnet = flag.Bool("testnet", false, "operate on the testnet Bitcoin network") +// +// // By default (without -testnet), use mainnet. +// var chainParams = &chaincfg.MainNetParams +// +// func main() { +// flag.Parse() +// +// // Modify active network parameters if operating on testnet. +// if *testnet { +// chainParams = &chaincfg.TestNet4Params +// } +// +// // later... +// +// // Create and print new payment address, specific to the active network. +// pubKeyHash := make([]byte, 20) +// addr, err := ltcutil.NewAddressPubKeyHash(pubKeyHash, chainParams) +// if err != nil { +// log.Fatal(err) +// } +// fmt.Println(addr) +// } +// +// If an application does not use one of the three standard Bitcoin networks, +// a new Params struct may be created which defines the parameters for the +// non-standard network. As a general rule of thumb, all network parameters +// should be unique to the network, but parameter collisions can still occur +// (unfortunately, this is the case with regtest and testnet sharing magics). +package chaincfg diff --git a/vendor/github.com/ltcsuite/ltcd/chaincfg/genesis.go b/vendor/github.com/ltcsuite/ltcd/chaincfg/genesis.go new file mode 100644 index 0000000000..3826628110 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/chaincfg/genesis.go @@ -0,0 +1,178 @@ +// Copyright (c) 2014-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package chaincfg + +import ( + "time" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" + "github.com/ltcsuite/ltcd/wire" +) + +// genesisCoinbaseTx is the coinbase transaction for the genesis blocks for +// the main network, regression test network, and test network (version 3). +var genesisCoinbaseTx = wire.MsgTx{ + Version: 1, + TxIn: []*wire.TxIn{ + { + PreviousOutPoint: wire.OutPoint{ + Hash: chainhash.Hash{}, + Index: 0xffffffff, + }, + SignatureScript: []byte{ + 0x04, 0xff, 0xff, 0x00, 0x1d, 0x01, 0x04, 0x40, 0x4e, 0x59, 0x20, 0x54, 0x69, 0x6d, 0x65, 0x73, // |.......@NY Times| + 0x20, 0x30, 0x35, 0x2f, 0x4f, 0x63, 0x74, 0x2f, 0x32, 0x30, 0x31, 0x31, 0x20, 0x53, 0x74, 0x65, // | 05/Oct/2011 Ste| + 0x76, 0x65, 0x20, 0x4a, 0x6f, 0x62, 0x73, 0x2c, 0x20, 0x41, 0x70, 0x70, 0x6c, 0x65, 0xe2, 0x80, // |ve Jobs, Apple..| + 0x99, 0x73, 0x20, 0x56, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x2c, 0x20, 0x44, 0x69, // |.s Visionary, Di| + 0x65, 0x73, 0x20, 0x61, 0x74, 0x20, 0x35, 0x36, // |es at 56| + + }, + Sequence: 0xffffffff, + }, + }, + TxOut: []*wire.TxOut{ + { + Value: 0x12a05f200, + PkScript: []byte{ + 0x41, 0x4, 0x1, 0x84, 0x71, 0xf, 0xa6, 0x89, + 0xad, 0x50, 0x23, 0x69, 0xc, 0x80, 0xf3, 0xa4, + 0x9c, 0x8f, 0x13, 0xf8, 0xd4, 0x5b, 0x8c, 0x85, + 0x7f, 0xbc, 0xbc, 0x8b, 0xc4, 0xa8, 0xe4, 0xd3, + 0xeb, 0x4b, 0x10, 0xf4, 0xd4, 0x60, 0x4f, 0xa0, + 0x8d, 0xce, 0x60, 0x1a, 0xaf, 0xf, 0x47, 0x2, + 0x16, 0xfe, 0x1b, 0x51, 0x85, 0xb, 0x4a, 0xcf, + 0x21, 0xb1, 0x79, 0xc4, 0x50, 0x70, 0xac, 0x7b, + 0x3, 0xa9, 0xac, + }, + }, + }, + LockTime: 0, +} + +// genesisHash is the hash of the first block in the block chain for the main +// network (genesis block). +var genesisHash = chainhash.Hash([chainhash.HashSize]byte{ // Make go vet happy. + 0xe2, 0xbf, 0x04, 0x7e, 0x7e, 0x5a, 0x19, 0x1a, + 0xa4, 0xef, 0x34, 0xd3, 0x14, 0x97, 0x9d, 0xc9, + 0x98, 0x6e, 0x0f, 0x19, 0x25, 0x1e, 0xda, 0xba, + 0x59, 0x40, 0xfd, 0x1f, 0xe3, 0x65, 0xa7, 0x12, + }) + +// genesisMerkleRoot is the hash of the first transaction in the genesis block +// for the main network. +var genesisMerkleRoot = chainhash.Hash([chainhash.HashSize]byte{ // Make go vet happy. + 0xd9, 0xce, 0xd4, 0xed, 0x11, 0x30, 0xf7, 0xb7, + 0xfa, 0xad, 0x9b, 0xe2, 0x53, 0x23, 0xff, 0xaf, + 0xa3, 0x32, 0x32, 0xa1, 0x7c, 0x3e, 0xdf, 0x6c, + 0xfd, 0x97, 0xbe, 0xe6, 0xba, 0xfb, 0xdd, 0x97, + }) + +// genesisBlock defines the genesis block of the block chain which serves as the +// public transaction ledger for the main network. +var genesisBlock = wire.MsgBlock{ + Header: wire.BlockHeader{ + Version: 1, + PrevBlock: chainhash.Hash{}, // 0000000000000000000000000000000000000000000000000000000000000000 + MerkleRoot: genesisMerkleRoot, // 97ddfbbae6be97fd6cdf3e7ca13232a3afff2353e29badfab7f73011edd4ced9 + Timestamp: time.Unix(1317972665, 0), + Bits: 0x1e0ffff0, + Nonce: 2084524493, + }, + Transactions: []*wire.MsgTx{&genesisCoinbaseTx}, +} + +// regTestGenesisHash is the hash of the first block in the block chain for the +// regression test network (genesis block). +var regTestGenesisHash = chainhash.Hash([chainhash.HashSize]byte{ // Make go vet happy. + 0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, + 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, + 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, + 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f, +}) + +// regTestGenesisMerkleRoot is the hash of the first transaction in the genesis +// block for the regression test network. It is the same as the merkle root for +// the main network. +var regTestGenesisMerkleRoot = genesisMerkleRoot + +// regTestGenesisBlock defines the genesis block of the block chain which serves +// as the public transaction ledger for the regression test network. +var regTestGenesisBlock = wire.MsgBlock{ + Header: wire.BlockHeader{ + Version: 1, + PrevBlock: chainhash.Hash{}, // 0000000000000000000000000000000000000000000000000000000000000000 + MerkleRoot: regTestGenesisMerkleRoot, // 97ddfbbae6be97fd6cdf3e7ca13232a3afff2353e29badfab7f73011edd4ced9 + Timestamp: time.Unix(1296688602, 0), // 2011-02-02 23:16:42 +0000 UTC + Bits: 0x207fffff, // 545259519 [7fffff0000000000000000000000000000000000000000000000000000000000] + Nonce: 2, + }, + Transactions: []*wire.MsgTx{&genesisCoinbaseTx}, +} + +// testNet4GenesisHash is the hash of the first block in the block chain for the +// test network (version 4). +var testNet4GenesisHash = chainhash.Hash([chainhash.HashSize]byte{ // Make go vet happy. + 0xa0, 0x29, 0x3e, 0x4e, 0xeb, 0x3d, 0xa6, 0xe6, + 0xf5, 0x6f, 0x81, 0xed, 0x59, 0x5f, 0x57, 0x88, + 0xd, 0x1a, 0x21, 0x56, 0x9e, 0x13, 0xee, 0xfd, + 0xd9, 0x51, 0x28, 0x4b, 0x5a, 0x62, 0x66, 0x49, +}) + +// testNet3GenesisMerkleRoot is the hash of the first transaction in the genesis +// block for the test network (version 3). It is the same as the merkle root +// for the main network. +var testNet3GenesisMerkleRoot = genesisMerkleRoot + +// testNet4GenesisMerkleRoot is the hash of the first transaction in the genesis +// block for the test network (version 4). It is the same as the merkle root +// for the main network. +var testNet4GenesisMerkleRoot = chainhash.Hash([chainhash.HashSize]byte{ // Make go vet happy. + 0xd9, 0xce, 0xd4, 0xed, 0x11, 0x30, 0xf7, 0xb7, + 0xfa, 0xad, 0x9b, 0xe2, 0x53, 0x23, 0xff, 0xaf, + 0xa3, 0x32, 0x32, 0xa1, 0x7c, 0x3e, 0xdf, 0x6c, + 0xfd, 0x97, 0xbe, 0xe6, 0xba, 0xfb, 0xdd, 0x97, +}) + +// testNet4GenesisBlock defines the genesis block of the block chain which +// serves as the public transaction ledger for the test network (version 4). +var testNet4GenesisBlock = wire.MsgBlock{ + Header: wire.BlockHeader{ + Version: 1, + PrevBlock: chainhash.Hash{}, // 0000000000000000000000000000000000000000000000000000000000000000 + MerkleRoot: testNet4GenesisMerkleRoot, // 97ddfbbae6be97fd6cdf3e7ca13232a3afff2353e29badfab7f73011edd4ced9 + Timestamp: time.Unix(1486949366, 0), + Bits: 0x1e0ffff0, + Nonce: 293345, + }, + Transactions: []*wire.MsgTx{&genesisCoinbaseTx}, +} + +// simNetGenesisHash is the hash of the first block in the block chain for the +// simulation test network. +var simNetGenesisHash = chainhash.Hash([chainhash.HashSize]byte{ // Make go vet happy. + 0xf6, 0x7a, 0xd7, 0x69, 0x5d, 0x9b, 0x66, 0x2a, + 0x72, 0xff, 0x3d, 0x8e, 0xdb, 0xbb, 0x2d, 0xe0, + 0xbf, 0xa6, 0x7b, 0x13, 0x97, 0x4b, 0xb9, 0x91, + 0x0d, 0x11, 0x6d, 0x5c, 0xbd, 0x86, 0x3e, 0x68, +}) + +// simNetGenesisMerkleRoot is the hash of the first transaction in the genesis +// block for the simulation test network. It is the same as the merkle root for +// the main network. +var simNetGenesisMerkleRoot = genesisMerkleRoot + +// simNetGenesisBlock defines the genesis block of the block chain which serves +// as the public transaction ledger for the simulation test network. +var simNetGenesisBlock = wire.MsgBlock{ + Header: wire.BlockHeader{ + Version: 1, + PrevBlock: chainhash.Hash{}, // 0000000000000000000000000000000000000000000000000000000000000000 + MerkleRoot: simNetGenesisMerkleRoot, // 4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b + Timestamp: time.Unix(1401292357, 0), // 2014-05-28 15:52:37 +0000 UTC + Bits: 0x207fffff, // 545259519 [7fffff0000000000000000000000000000000000000000000000000000000000] + Nonce: 2, + }, + Transactions: []*wire.MsgTx{&genesisCoinbaseTx}, +} diff --git a/vendor/github.com/ltcsuite/ltcd/chaincfg/params.go b/vendor/github.com/ltcsuite/ltcd/chaincfg/params.go new file mode 100644 index 0000000000..0ed2c6f34f --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/chaincfg/params.go @@ -0,0 +1,696 @@ +// Copyright (c) 2014-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package chaincfg + +import ( + "errors" + "math" + "math/big" + "strings" + "time" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" + "github.com/ltcsuite/ltcd/wire" +) + +// These variables are the chain proof-of-work limit parameters for each default +// network. +var ( + // bigOne is 1 represented as a big.Int. It is defined here to avoid + // the overhead of creating it multiple times. + bigOne = big.NewInt(1) + + // mainPowLimit is the highest proof of work value a Litecoin block can + // have for the main network. + mainPowLimit, _ = new(big.Int).SetString("0x0fffff000000000000000000000000000000000000000000000000000000", 0) + + // regressionPowLimit is the highest proof of work value a Litecoin block + // can have for the regression test network. It is the value 2^255 - 1. + regressionPowLimit = new(big.Int).Sub(new(big.Int).Lsh(bigOne, 255), bigOne) + + // testNet3PowLimit is the highest proof of work value a Litecoin block + // can have for the test network (version 3). It is the value + // 2^224 - 1. + testNet3PowLimit = new(big.Int).Sub(new(big.Int).Lsh(bigOne, 224), bigOne) + + // testNet4PowLimit is the highest proof of work value a Litecoin block + // can have for the test network (version 4). + testNet4PowLimit, _ = new(big.Int).SetString("0x0fffff000000000000000000000000000000000000000000000000000000", 0) + + // simNetPowLimit is the highest proof of work value a Litecoin block + // can have for the simulation test network. It is the value 2^255 - 1. + simNetPowLimit = new(big.Int).Sub(new(big.Int).Lsh(bigOne, 255), bigOne) +) + +// Checkpoint identifies a known good point in the block chain. Using +// checkpoints allows a few optimizations for old blocks during initial download +// and also prevents forks from old blocks. +// +// Each checkpoint is selected based upon several factors. See the +// documentation for blockchain.IsCheckpointCandidate for details on the +// selection criteria. +type Checkpoint struct { + Height int32 + Hash *chainhash.Hash +} + +// DNSSeed identifies a DNS seed. +type DNSSeed struct { + // Host defines the hostname of the seed. + Host string + + // HasFiltering defines whether the seed supports filtering + // by service flags (wire.ServiceFlag). + HasFiltering bool +} + +// ConsensusDeployment defines details related to a specific consensus rule +// change that is voted in. This is part of BIP0009. +type ConsensusDeployment struct { + // BitNumber defines the specific bit number within the block version + // this particular soft-fork deployment refers to. + BitNumber uint8 + + // StartTime is the median block time after which voting on the + // deployment starts. + StartTime uint64 + + // ExpireTime is the median block time after which the attempted + // deployment expires. + ExpireTime uint64 +} + +// Constants that define the deployment offset in the deployments field of the +// parameters for each deployment. This is useful to be able to get the details +// of a specific deployment by name. +const ( + // DeploymentTestDummy defines the rule change deployment ID for testing + // purposes. + DeploymentTestDummy = iota + + // DeploymentCSV defines the rule change deployment ID for the CSV + // soft-fork package. The CSV package includes the depolyment of BIPS + // 68, 112, and 113. + DeploymentCSV + + // DeploymentSegwit defines the rule change deployment ID for the + // Segragated Witness (segwit) soft-fork package. The segwit package + // includes the deployment of BIPS 141, 142, 144, 145, 147 and 173. + DeploymentSegwit + + // NOTE: DefinedDeployments must always come last since it is used to + // determine how many defined deployments there currently are. + + // DefinedDeployments is the number of currently defined deployments. + DefinedDeployments +) + +// Params defines a Litecoin network by its parameters. These parameters may be +// used by Litecoin applications to differentiate networks as well as addresses +// and keys for one network from those intended for use on another network. +type Params struct { + // Name defines a human-readable identifier for the network. + Name string + + // Net defines the magic bytes used to identify the network. + Net wire.BitcoinNet + + // DefaultPort defines the default peer-to-peer port for the network. + DefaultPort string + + // DNSSeeds defines a list of DNS seeds for the network that are used + // as one method to discover peers. + DNSSeeds []DNSSeed + + // GenesisBlock defines the first block of the chain. + GenesisBlock *wire.MsgBlock + + // GenesisHash is the starting block hash. + GenesisHash *chainhash.Hash + + // PowLimit defines the highest allowed proof of work value for a block + // as a uint256. + PowLimit *big.Int + + // PowLimitBits defines the highest allowed proof of work value for a + // block in compact form. + PowLimitBits uint32 + + // These fields define the block heights at which the specified softfork + // BIP became active. + BIP0034Height int32 + BIP0065Height int32 + BIP0066Height int32 + + // CoinbaseMaturity is the number of blocks required before newly mined + // coins (coinbase transactions) can be spent. + CoinbaseMaturity uint16 + + // SubsidyReductionInterval is the interval of blocks before the subsidy + // is reduced. + SubsidyReductionInterval int32 + + // TargetTimespan is the desired amount of time that should elapse + // before the block difficulty requirement is examined to determine how + // it should be changed in order to maintain the desired block + // generation rate. + TargetTimespan time.Duration + + // TargetTimePerBlock is the desired amount of time to generate each + // block. + TargetTimePerBlock time.Duration + + // RetargetAdjustmentFactor is the adjustment factor used to limit + // the minimum and maximum amount of adjustment that can occur between + // difficulty retargets. + RetargetAdjustmentFactor int64 + + // ReduceMinDifficulty defines whether the network should reduce the + // minimum required difficulty after a long enough period of time has + // passed without finding a block. This is really only useful for test + // networks and should not be set on a main network. + ReduceMinDifficulty bool + + // MinDiffReductionTime is the amount of time after which the minimum + // required difficulty should be reduced when a block hasn't been found. + // + // NOTE: This only applies if ReduceMinDifficulty is true. + MinDiffReductionTime time.Duration + + // GenerateSupported specifies whether or not CPU mining is allowed. + GenerateSupported bool + + // Checkpoints ordered from oldest to newest. + Checkpoints []Checkpoint + + // These fields are related to voting on consensus rule changes as + // defined by BIP0009. + // + // RuleChangeActivationThreshold is the number of blocks in a threshold + // state retarget window for which a positive vote for a rule change + // must be cast in order to lock in a rule change. It should typically + // be 95% for the main network and 75% for test networks. + // + // MinerConfirmationWindow is the number of blocks in each threshold + // state retarget window. + // + // Deployments define the specific consensus rule changes to be voted + // on. + RuleChangeActivationThreshold uint32 + MinerConfirmationWindow uint32 + Deployments [DefinedDeployments]ConsensusDeployment + + // Mempool parameters + RelayNonStdTxs bool + + // Human-readable part for Bech32 encoded segwit addresses, as defined + // in BIP 173. + Bech32HRPSegwit string + + // Address encoding magics + PubKeyHashAddrID byte // First byte of a P2PKH address + ScriptHashAddrID byte // First byte of a P2SH address + PrivateKeyID byte // First byte of a WIF private key + WitnessPubKeyHashAddrID byte // First byte of a P2WPKH address + WitnessScriptHashAddrID byte // First byte of a P2WSH address + + // BIP32 hierarchical deterministic extended key magics + HDPrivateKeyID [4]byte + HDPublicKeyID [4]byte + + // BIP44 coin type used in the hierarchical deterministic path for + // address generation. + HDCoinType uint32 +} + +// MainNetParams defines the network parameters for the main Litecoin network. +var MainNetParams = Params{ + Name: "mainnet", + Net: wire.MainNet, + DefaultPort: "9333", + DNSSeeds: []DNSSeed{ + {"seed-a.litecoin.loshan.co.uk", true}, + {"dnsseed.thrasher.io", true}, + {"dnsseed.litecointools.com", false}, + {"dnsseed.litecoinpool.org", false}, + {"dnsseed.koin-project.com", false}, + }, + + // Chain parameters + GenesisBlock: &genesisBlock, + GenesisHash: &genesisHash, + PowLimit: mainPowLimit, + PowLimitBits: 504365055, + BIP0034Height: 710000, + BIP0065Height: 918684, + BIP0066Height: 811879, + CoinbaseMaturity: 100, + SubsidyReductionInterval: 840000, + TargetTimespan: (time.Hour * 24 * 3) + (time.Hour * 12), // 3.5 days + TargetTimePerBlock: (time.Minute * 2) + (time.Second * 30), // 2.5 minutes + RetargetAdjustmentFactor: 4, // 25% less, 400% more + ReduceMinDifficulty: false, + MinDiffReductionTime: 0, + GenerateSupported: false, + + // Checkpoints ordered from oldest to newest. + Checkpoints: []Checkpoint{ + {1500, newHashFromStr("841a2965955dd288cfa707a755d05a54e45f8bd476835ec9af4402a2b59a2967")}, + {4032, newHashFromStr("9ce90e427198fc0ef05e5905ce3503725b80e26afd35a987965fd7e3d9cf0846")}, + {8064, newHashFromStr("eb984353fc5190f210651f150c40b8a4bab9eeeff0b729fcb3987da694430d70")}, + {16128, newHashFromStr("602edf1859b7f9a6af809f1d9b0e6cb66fdc1d4d9dcd7a4bec03e12a1ccd153d")}, + {23420, newHashFromStr("d80fdf9ca81afd0bd2b2a90ac3a9fe547da58f2530ec874e978fce0b5101b507")}, + {50000, newHashFromStr("69dc37eb029b68f075a5012dcc0419c127672adb4f3a32882b2b3e71d07a20a6")}, + {80000, newHashFromStr("4fcb7c02f676a300503f49c764a89955a8f920b46a8cbecb4867182ecdb2e90a")}, + {120000, newHashFromStr("bd9d26924f05f6daa7f0155f32828ec89e8e29cee9e7121b026a7a3552ac6131")}, + {161500, newHashFromStr("dbe89880474f4bb4f75c227c77ba1cdc024991123b28b8418dbbf7798471ff43")}, + {179620, newHashFromStr("2ad9c65c990ac00426d18e446e0fd7be2ffa69e9a7dcb28358a50b2b78b9f709")}, + {240000, newHashFromStr("7140d1c4b4c2157ca217ee7636f24c9c73db39c4590c4e6eab2e3ea1555088aa")}, + {383640, newHashFromStr("2b6809f094a9215bafc65eb3f110a35127a34be94b7d0590a096c3f126c6f364")}, + {409004, newHashFromStr("487518d663d9f1fa08611d9395ad74d982b667fbdc0e77e9cf39b4f1355908a3")}, + {456000, newHashFromStr("bf34f71cc6366cd487930d06be22f897e34ca6a40501ac7d401be32456372004")}, + {638902, newHashFromStr("15238656e8ec63d28de29a8c75fcf3a5819afc953dcd9cc45cecc53baec74f38")}, + {721000, newHashFromStr("198a7b4de1df9478e2463bd99d75b714eab235a2e63e741641dc8a759a9840e5")}, + }, + + // Consensus rule change deployments. + // + // The miner confirmation window is defined as: + // target proof of work timespan / target proof of work spacing + RuleChangeActivationThreshold: 6048, // 75% of MinerConfirmationWindow + MinerConfirmationWindow: 8064, // + Deployments: [DefinedDeployments]ConsensusDeployment{ + DeploymentTestDummy: { + BitNumber: 28, + StartTime: 1199145601, // January 1, 2008 UTC + ExpireTime: 1230767999, // December 31, 2008 UTC + }, + DeploymentCSV: { + BitNumber: 0, + StartTime: 1485561600, // January 28, 2017 UTC + ExpireTime: 1517356801, // January 31st, 2018 UTC + }, + DeploymentSegwit: { + BitNumber: 1, + StartTime: 1485561600, // January 28, 2017 UTC + ExpireTime: 1517356801, // January 31st, 2018 UTC. + }, + }, + + // Mempool parameters + RelayNonStdTxs: false, + + // Human-readable part for Bech32 encoded segwit addresses, as defined in + // BIP 173. + Bech32HRPSegwit: "ltc", // always ltc for main net + + // Address encoding magics + PubKeyHashAddrID: 0x30, // starts with L + ScriptHashAddrID: 0x50, // starts with M + PrivateKeyID: 0xB0, // starts with 6 (uncompressed) or T (compressed) + WitnessPubKeyHashAddrID: 0x06, // starts with p2 + WitnessScriptHashAddrID: 0x0A, // starts with 7Xh + + // BIP32 hierarchical deterministic extended key magics + HDPrivateKeyID: [4]byte{0x04, 0x88, 0xad, 0xe4}, // starts with xprv + HDPublicKeyID: [4]byte{0x04, 0x88, 0xb2, 0x1e}, // starts with xpub + + // BIP44 coin type used in the hierarchical deterministic path for + // address generation. + HDCoinType: 2, +} + +// RegressionNetParams defines the network parameters for the regression test +// Litecoin network. Not to be confused with the test Litecoin network (version +// 3), this network is sometimes simply called "testnet". +var RegressionNetParams = Params{ + Name: "regtest", + Net: wire.TestNet, + DefaultPort: "18444", + DNSSeeds: []DNSSeed{}, + + // Chain parameters + GenesisBlock: ®TestGenesisBlock, + GenesisHash: ®TestGenesisHash, + PowLimit: regressionPowLimit, + PowLimitBits: 0x207fffff, + CoinbaseMaturity: 100, + BIP0034Height: 100000000, // Not active - Permit ver 1 blocks + BIP0065Height: 1351, // Used by regression tests + BIP0066Height: 1251, // Used by regression tests + SubsidyReductionInterval: 150, + TargetTimespan: time.Hour * 24 * 14, // 14 days + TargetTimePerBlock: time.Minute * 10, // 10 minutes + RetargetAdjustmentFactor: 4, // 25% less, 400% more + ReduceMinDifficulty: true, + MinDiffReductionTime: time.Minute * 20, // TargetTimePerBlock * 2 + GenerateSupported: true, + + // Checkpoints ordered from oldest to newest. + Checkpoints: nil, + + // Consensus rule change deployments. + // + // The miner confirmation window is defined as: + // target proof of work timespan / target proof of work spacing + RuleChangeActivationThreshold: 108, // 75% of MinerConfirmationWindow + MinerConfirmationWindow: 144, + Deployments: [DefinedDeployments]ConsensusDeployment{ + DeploymentTestDummy: { + BitNumber: 28, + StartTime: 0, // Always available for vote + ExpireTime: math.MaxInt64, // Never expires + }, + DeploymentCSV: { + BitNumber: 0, + StartTime: 0, // Always available for vote + ExpireTime: math.MaxInt64, // Never expires + }, + DeploymentSegwit: { + BitNumber: 1, + StartTime: 0, // Always available for vote + ExpireTime: math.MaxInt64, // Never expires. + }, + }, + + // Mempool parameters + RelayNonStdTxs: true, + + // Human-readable part for Bech32 encoded segwit addresses, as defined in + // BIP 173. + Bech32HRPSegwit: "tltc", // always tltc for test net + + // Address encoding magics + PubKeyHashAddrID: 0x6f, // starts with m or n + ScriptHashAddrID: 0xc4, // starts with 2 + PrivateKeyID: 0xef, // starts with 9 (uncompressed) or c (compressed) + + // BIP32 hierarchical deterministic extended key magics + HDPrivateKeyID: [4]byte{0x04, 0x35, 0x83, 0x94}, // starts with tprv + HDPublicKeyID: [4]byte{0x04, 0x35, 0x87, 0xcf}, // starts with tpub + + // BIP44 coin type used in the hierarchical deterministic path for + // address generation. + HDCoinType: 1, +} + +// TestNet4Params defines the network parameters for the test Litecoin network +// (version 4). Not to be confused with the regression test network, this +// network is sometimes simply called "testnet". +var TestNet4Params = Params{ + Name: "testnet4", + Net: wire.TestNet4, + DefaultPort: "19335", + DNSSeeds: []DNSSeed{ + {"testnet-seed.litecointools.com", false}, + {"seed-b.litecoin.loshan.co.uk", true}, + {"dnsseed-testnet.thrasher.io", true}, + }, + + // Chain parameters + GenesisBlock: &testNet4GenesisBlock, + GenesisHash: &testNet4GenesisHash, + PowLimit: testNet4PowLimit, + PowLimitBits: 504365055, + BIP0034Height: 76, + BIP0065Height: 76, + BIP0066Height: 76, + CoinbaseMaturity: 100, + SubsidyReductionInterval: 840000, + TargetTimespan: (time.Hour * 24 * 3) + (time.Hour * 12), // 3.5 days + TargetTimePerBlock: (time.Minute * 2) + (time.Second * 30), // 2.5 minutes + RetargetAdjustmentFactor: 4, // 25% less, 400% more + ReduceMinDifficulty: true, + MinDiffReductionTime: time.Minute * 5, // TargetTimePerBlock * 2 + GenerateSupported: false, + + // Checkpoints ordered from oldest to newest. + Checkpoints: []Checkpoint{ + {26115, newHashFromStr("817d5b509e91ab5e439652eee2f59271bbc7ba85021d720cdb6da6565b43c14f")}, + {43928, newHashFromStr("7d86614c153f5ef6ad878483118ae523e248cd0dd0345330cb148e812493cbb4")}, + {69296, newHashFromStr("66c2f58da3cfd282093b55eb09c1f5287d7a18801a8ff441830e67e8771010df")}, + {99949, newHashFromStr("8dd471cb5aecf5ead91e7e4b1e932c79a0763060f8d93671b6801d115bfc6cde")}, + {159256, newHashFromStr("ab5b0b9968842f5414804591119d6db829af606864b1959a25d6f5c114afb2b7")}, + }, + + // Consensus rule change deployments. + // + // The miner confirmation window is defined as: + // target proof of work timespan / target proof of work spacing + RuleChangeActivationThreshold: 1512, // 75% of MinerConfirmationWindow + MinerConfirmationWindow: 2016, + Deployments: [DefinedDeployments]ConsensusDeployment{ + DeploymentTestDummy: { + BitNumber: 28, + StartTime: 1199145601, // January 1, 2008 UTC + ExpireTime: 1230767999, // December 31, 2008 UTC + }, + DeploymentCSV: { + BitNumber: 0, + StartTime: 1483228800, // January 1, 2017 + ExpireTime: 1517356801, // January 31st, 2018 + }, + DeploymentSegwit: { + BitNumber: 1, + StartTime: 1483228800, // January 1, 2017 + ExpireTime: 1517356801, // January 31st, 2018 + }, + }, + + // Mempool parameters + RelayNonStdTxs: true, + + // Human-readable part for Bech32 encoded segwit addresses, as defined in + // BIP 173. + Bech32HRPSegwit: "tltc", // always tb for test net + + // Address encoding magics + PubKeyHashAddrID: 0x6f, // starts with m or n + ScriptHashAddrID: 0xc4, // starts with 2 + WitnessPubKeyHashAddrID: 0x52, // starts with QW + WitnessScriptHashAddrID: 0x31, // starts with T7n + PrivateKeyID: 0xef, // starts with 9 (uncompressed) or c (compressed) + + // BIP32 hierarchical deterministic extended key magics + HDPrivateKeyID: [4]byte{0x04, 0x35, 0x83, 0x94}, // starts with tprv + HDPublicKeyID: [4]byte{0x04, 0x35, 0x87, 0xcf}, // starts with tpub + + // BIP44 coin type used in the hierarchical deterministic path for + // address generation. + HDCoinType: 1, +} + +// SimNetParams defines the network parameters for the simulation test Litecoin +// network. This network is similar to the normal test network except it is +// intended for private use within a group of individuals doing simulation +// testing. The functionality is intended to differ in that the only nodes +// which are specifically specified are used to create the network rather than +// following normal discovery rules. This is important as otherwise it would +// just turn into another public testnet. +var SimNetParams = Params{ + Name: "simnet", + Net: wire.SimNet, + DefaultPort: "18555", + DNSSeeds: []DNSSeed{}, // NOTE: There must NOT be any seeds. + + // Chain parameters + GenesisBlock: &simNetGenesisBlock, + GenesisHash: &simNetGenesisHash, + PowLimit: simNetPowLimit, + PowLimitBits: 0x207fffff, + BIP0034Height: 0, // Always active on simnet + BIP0065Height: 0, // Always active on simnet + BIP0066Height: 0, // Always active on simnet + CoinbaseMaturity: 100, + SubsidyReductionInterval: 210000, + TargetTimespan: time.Hour * 24 * 14, // 14 days + TargetTimePerBlock: time.Minute * 10, // 10 minutes + RetargetAdjustmentFactor: 4, // 25% less, 400% more + ReduceMinDifficulty: true, + MinDiffReductionTime: time.Minute * 20, // TargetTimePerBlock * 2 + GenerateSupported: true, + + // Checkpoints ordered from oldest to newest. + Checkpoints: nil, + + // Consensus rule change deployments. + // + // The miner confirmation window is defined as: + // target proof of work timespan / target proof of work spacing + RuleChangeActivationThreshold: 75, // 75% of MinerConfirmationWindow + MinerConfirmationWindow: 100, + Deployments: [DefinedDeployments]ConsensusDeployment{ + DeploymentTestDummy: { + BitNumber: 28, + StartTime: 0, // Always available for vote + ExpireTime: math.MaxInt64, // Never expires + }, + DeploymentCSV: { + BitNumber: 0, + StartTime: 0, // Always available for vote + ExpireTime: math.MaxInt64, // Never expires + }, + DeploymentSegwit: { + BitNumber: 1, + StartTime: 0, // Always available for vote + ExpireTime: math.MaxInt64, // Never expires. + }, + }, + + // Mempool parameters + RelayNonStdTxs: true, + + // Human-readable part for Bech32 encoded segwit addresses, as defined in + // BIP 173. + Bech32HRPSegwit: "sltc", // always lsb for sim net + + // Address encoding magics + PubKeyHashAddrID: 0x3f, // starts with S + ScriptHashAddrID: 0x7b, // starts with s + PrivateKeyID: 0x64, // starts with 4 (uncompressed) or F (compressed) + WitnessPubKeyHashAddrID: 0x19, // starts with Gg + WitnessScriptHashAddrID: 0x28, // starts with ? + + // BIP32 hierarchical deterministic extended key magics + HDPrivateKeyID: [4]byte{0x04, 0x20, 0xb9, 0x00}, // starts with sprv + HDPublicKeyID: [4]byte{0x04, 0x20, 0xbd, 0x3a}, // starts with spub + + // BIP44 coin type used in the hierarchical deterministic path for + // address generation. + HDCoinType: 115, // ASCII for s +} + +var ( + // ErrDuplicateNet describes an error where the parameters for a Litecoin + // network could not be set due to the network already being a standard + // network or previously-registered into this package. + ErrDuplicateNet = errors.New("duplicate Litecoin network") + + // ErrUnknownHDKeyID describes an error where the provided id which + // is intended to identify the network for a hierarchical deterministic + // private extended key is not registered. + ErrUnknownHDKeyID = errors.New("unknown hd private extended key bytes") +) + +var ( + registeredNets = make(map[wire.BitcoinNet]struct{}) + pubKeyHashAddrIDs = make(map[byte]struct{}) + scriptHashAddrIDs = make(map[byte]struct{}) + bech32SegwitPrefixes = make(map[string]struct{}) + hdPrivToPubKeyIDs = make(map[[4]byte][]byte) +) + +// String returns the hostname of the DNS seed in human-readable form. +func (d DNSSeed) String() string { + return d.Host +} + +// Register registers the network parameters for a Litecoin network. This may +// error with ErrDuplicateNet if the network is already registered (either +// due to a previous Register call, or the network being one of the default +// networks). +// +// Network parameters should be registered into this package by a main package +// as early as possible. Then, library packages may lookup networks or network +// parameters based on inputs and work regardless of the network being standard +// or not. +func Register(params *Params) error { + if _, ok := registeredNets[params.Net]; ok { + return ErrDuplicateNet + } + registeredNets[params.Net] = struct{}{} + pubKeyHashAddrIDs[params.PubKeyHashAddrID] = struct{}{} + scriptHashAddrIDs[params.ScriptHashAddrID] = struct{}{} + hdPrivToPubKeyIDs[params.HDPrivateKeyID] = params.HDPublicKeyID[:] + + // A valid Bech32 encoded segwit address always has as prefix the + // human-readable part for the given net followed by '1'. + bech32SegwitPrefixes[params.Bech32HRPSegwit+"1"] = struct{}{} + return nil +} + +// mustRegister performs the same function as Register except it panics if there +// is an error. This should only be called from package init functions. +func mustRegister(params *Params) { + if err := Register(params); err != nil { + panic("failed to register network: " + err.Error()) + } +} + +// IsPubKeyHashAddrID returns whether the id is an identifier known to prefix a +// pay-to-pubkey-hash address on any default or registered network. This is +// used when decoding an address string into a specific address type. It is up +// to the caller to check both this and IsScriptHashAddrID and decide whether an +// address is a pubkey hash address, script hash address, neither, or +// undeterminable (if both return true). +func IsPubKeyHashAddrID(id byte) bool { + _, ok := pubKeyHashAddrIDs[id] + return ok +} + +// IsScriptHashAddrID returns whether the id is an identifier known to prefix a +// pay-to-script-hash address on any default or registered network. This is +// used when decoding an address string into a specific address type. It is up +// to the caller to check both this and IsPubKeyHashAddrID and decide whether an +// address is a pubkey hash address, script hash address, neither, or +// undeterminable (if both return true). +func IsScriptHashAddrID(id byte) bool { + _, ok := scriptHashAddrIDs[id] + return ok +} + +// IsBech32SegwitPrefix returns whether the prefix is a known prefix for segwit +// addresses on any default or registered network. This is used when decoding +// an address string into a specific address type. +func IsBech32SegwitPrefix(prefix string) bool { + prefix = strings.ToLower(prefix) + _, ok := bech32SegwitPrefixes[prefix] + return ok +} + +// HDPrivateKeyToPublicKeyID accepts a private hierarchical deterministic +// extended key id and returns the associated public key id. When the provided +// id is not registered, the ErrUnknownHDKeyID error will be returned. +func HDPrivateKeyToPublicKeyID(id []byte) ([]byte, error) { + if len(id) != 4 { + return nil, ErrUnknownHDKeyID + } + + var key [4]byte + copy(key[:], id) + pubBytes, ok := hdPrivToPubKeyIDs[key] + if !ok { + return nil, ErrUnknownHDKeyID + } + + return pubBytes, nil +} + +// newHashFromStr converts the passed big-endian hex string into a +// chainhash.Hash. It only differs from the one available in chainhash in that +// it panics on an error since it will only (and must only) be called with +// hard-coded, and therefore known good, hashes. +func newHashFromStr(hexStr string) *chainhash.Hash { + hash, err := chainhash.NewHashFromStr(hexStr) + if err != nil { + // Ordinarily I don't like panics in library code since it + // can take applications down without them having a chance to + // recover which is extremely annoying, however an exception is + // being made in this case because the only way this can panic + // is if there is an error in the hard-coded hashes. Thus it + // will only ever potentially panic on init and therefore is + // 100% predictable. + panic(err) + } + return hash +} + +func init() { + // Register all default networks when the package is initialized. + mustRegister(&MainNetParams) + mustRegister(&TestNet4Params) + mustRegister(&RegressionNetParams) + mustRegister(&SimNetParams) +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/README.md b/vendor/github.com/ltcsuite/ltcd/wire/README.md new file mode 100644 index 0000000000..91501177e9 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/README.md @@ -0,0 +1,112 @@ +wire +==== + +[![Build Status](http://img.shields.io/travis/ltcsuite/ltcd.svg)](https://travis-ci.org/ltcsuite/ltcd) [![ISC License] +(http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) +[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/ltcsuite/ltcd/wire) + +Package wire implements the bitcoin wire protocol. A comprehensive suite of +tests with 100% test coverage is provided to ensure proper functionality. + +There is an associated blog post about the release of this package +[here](https://blog.conformal.com/btcwire-the-bitcoin-wire-protocol-package-from-ltcd/). + +This package has intentionally been designed so it can be used as a standalone +package for any projects needing to interface with bitcoin peers at the wire +protocol level. + +## Installation and Updating + +```bash +$ go get -u github.com/ltcsuite/ltcd/wire +``` + +## Bitcoin Message Overview + +The bitcoin protocol consists of exchanging messages between peers. Each message +is preceded by a header which identifies information about it such as which +bitcoin network it is a part of, its type, how big it is, and a checksum to +verify validity. All encoding and decoding of message headers is handled by this +package. + +To accomplish this, there is a generic interface for bitcoin messages named +`Message` which allows messages of any type to be read, written, or passed +around through channels, functions, etc. In addition, concrete implementations +of most of the currently supported bitcoin messages are provided. For these +supported messages, all of the details of marshalling and unmarshalling to and +from the wire using bitcoin encoding are handled so the caller doesn't have to +concern themselves with the specifics. + +## Reading Messages Example + +In order to unmarshal bitcoin messages from the wire, use the `ReadMessage` +function. It accepts any `io.Reader`, but typically this will be a `net.Conn` +to a remote node running a bitcoin peer. Example syntax is: + +```Go + // Use the most recent protocol version supported by the package and the + // main bitcoin network. + pver := wire.ProtocolVersion + btcnet := wire.MainNet + + // Reads and validates the next bitcoin message from conn using the + // protocol version pver and the bitcoin network btcnet. The returns + // are a wire.Message, a []byte which contains the unmarshalled + // raw payload, and a possible error. + msg, rawPayload, err := wire.ReadMessage(conn, pver, btcnet) + if err != nil { + // Log and handle the error + } +``` + +See the package documentation for details on determining the message type. + +## Writing Messages Example + +In order to marshal bitcoin messages to the wire, use the `WriteMessage` +function. It accepts any `io.Writer`, but typically this will be a `net.Conn` +to a remote node running a bitcoin peer. Example syntax to request addresses +from a remote peer is: + +```Go + // Use the most recent protocol version supported by the package and the + // main bitcoin network. + pver := wire.ProtocolVersion + btcnet := wire.MainNet + + // Create a new getaddr bitcoin message. + msg := wire.NewMsgGetAddr() + + // Writes a bitcoin message msg to conn using the protocol version + // pver, and the bitcoin network btcnet. The return is a possible + // error. + err := wire.WriteMessage(conn, msg, pver, btcnet) + if err != nil { + // Log and handle the error + } +``` + +## GPG Verification Key + +All official release tags are signed by Conformal so users can ensure the code +has not been tampered with and is coming from the btcsuite developers. To +verify the signature perform the following: + +- Download the public key from the Conformal website at + https://opensource.conformal.com/GIT-GPG-KEY-conformal.txt + +- Import the public key into your GPG keyring: + ```bash + gpg --import GIT-GPG-KEY-conformal.txt + ``` + +- Verify the release tag with the following command where `TAG_NAME` is a + placeholder for the specific tag: + ```bash + git tag -v TAG_NAME + ``` + +## License + +Package wire is licensed under the [copyfree](http://copyfree.org) ISC +License. diff --git a/vendor/github.com/ltcsuite/ltcd/wire/blockheader.go b/vendor/github.com/ltcsuite/ltcd/wire/blockheader.go new file mode 100644 index 0000000000..8cde04731d --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/blockheader.go @@ -0,0 +1,147 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "bytes" + "io" + "time" + + "golang.org/x/crypto/scrypt" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +// MaxBlockHeaderPayload is the maximum number of bytes a block header can be. +// Version 4 bytes + Timestamp 4 bytes + Bits 4 bytes + Nonce 4 bytes + +// PrevBlock and MerkleRoot hashes. +const MaxBlockHeaderPayload = 16 + (chainhash.HashSize * 2) + +// BlockHeader defines information about a block and is used in the bitcoin +// block (MsgBlock) and headers (MsgHeaders) messages. +type BlockHeader struct { + // Version of the block. This is not the same as the protocol version. + Version int32 + + // Hash of the previous block in the block chain. + PrevBlock chainhash.Hash + + // Merkle tree reference to hash of all transactions for the block. + MerkleRoot chainhash.Hash + + // Time the block was created. This is, unfortunately, encoded as a + // uint32 on the wire and therefore is limited to 2106. + Timestamp time.Time + + // Difficulty target for the block. + Bits uint32 + + // Nonce used to generate the block. + Nonce uint32 +} + +// blockHeaderLen is a constant that represents the number of bytes for a block +// header. +const blockHeaderLen = 80 + +// BlockHash computes the block identifier hash for the given block header. +func (h *BlockHeader) BlockHash() chainhash.Hash { + // Encode the header and double sha256 everything prior to the number of + // transactions. Ignore the error returns since there is no way the + // encode could fail except being out of memory which would cause a + // run-time panic. + buf := bytes.NewBuffer(make([]byte, 0, MaxBlockHeaderPayload)) + _ = writeBlockHeader(buf, 0, h) + + return chainhash.DoubleHashH(buf.Bytes()) +} + +// PowHash returns the litecoin scrypt hash of this block header. This value is +// used to check the PoW on blocks advertised on the network. +func (h *BlockHeader) PowHash() (*chainhash.Hash, error) { + var powHash chainhash.Hash + + buf := bytes.NewBuffer(make([]byte, 0, MaxBlockHeaderPayload)) + _ = writeBlockHeader(buf, 0, h) + + scryptHash, err := scrypt.Key(buf.Bytes(), buf.Bytes(), 1024, 1, 1, 32) + if err != nil { + return nil, err + } + copy(powHash[:], scryptHash) + + return &powHash, nil +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +// See Deserialize for decoding block headers stored to disk, such as in a +// database, as opposed to decoding block headers from the wire. +func (h *BlockHeader) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + return readBlockHeader(r, pver, h) +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +// See Serialize for encoding block headers to be stored to disk, such as in a +// database, as opposed to encoding block headers for the wire. +func (h *BlockHeader) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + return writeBlockHeader(w, pver, h) +} + +// Deserialize decodes a block header from r into the receiver using a format +// that is suitable for long-term storage such as a database while respecting +// the Version field. +func (h *BlockHeader) Deserialize(r io.Reader) error { + // At the current time, there is no difference between the wire encoding + // at protocol version 0 and the stable long-term storage format. As + // a result, make use of readBlockHeader. + return readBlockHeader(r, 0, h) +} + +// Serialize encodes a block header from r into the receiver using a format +// that is suitable for long-term storage such as a database while respecting +// the Version field. +func (h *BlockHeader) Serialize(w io.Writer) error { + // At the current time, there is no difference between the wire encoding + // at protocol version 0 and the stable long-term storage format. As + // a result, make use of writeBlockHeader. + return writeBlockHeader(w, 0, h) +} + +// NewBlockHeader returns a new BlockHeader using the provided version, previous +// block hash, merkle root hash, difficulty bits, and nonce used to generate the +// block with defaults for the remaining fields. +func NewBlockHeader(version int32, prevHash, merkleRootHash *chainhash.Hash, + bits uint32, nonce uint32) *BlockHeader { + + // Limit the timestamp to one second precision since the protocol + // doesn't support better. + return &BlockHeader{ + Version: version, + PrevBlock: *prevHash, + MerkleRoot: *merkleRootHash, + Timestamp: time.Unix(time.Now().Unix(), 0), + Bits: bits, + Nonce: nonce, + } +} + +// readBlockHeader reads a bitcoin block header from r. See Deserialize for +// decoding block headers stored to disk, such as in a database, as opposed to +// decoding from the wire. +func readBlockHeader(r io.Reader, pver uint32, bh *BlockHeader) error { + return readElements(r, &bh.Version, &bh.PrevBlock, &bh.MerkleRoot, + (*uint32Time)(&bh.Timestamp), &bh.Bits, &bh.Nonce) +} + +// writeBlockHeader writes a bitcoin block header to w. See Serialize for +// encoding block headers to be stored to disk, such as in a database, as +// opposed to encoding for the wire. +func writeBlockHeader(w io.Writer, pver uint32, bh *BlockHeader) error { + sec := uint32(bh.Timestamp.Unix()) + return writeElements(w, bh.Version, &bh.PrevBlock, &bh.MerkleRoot, + sec, bh.Bits, bh.Nonce) +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/common.go b/vendor/github.com/ltcsuite/ltcd/wire/common.go new file mode 100644 index 0000000000..253c3f8847 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/common.go @@ -0,0 +1,689 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "crypto/rand" + "encoding/binary" + "fmt" + "io" + "math" + "time" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +const ( + // MaxVarIntPayload is the maximum payload size for a variable length integer. + MaxVarIntPayload = 9 + + // binaryFreeListMaxItems is the number of buffers to keep in the free + // list to use for binary serialization and deserialization. + binaryFreeListMaxItems = 1024 +) + +var ( + // littleEndian is a convenience variable since binary.LittleEndian is + // quite long. + littleEndian = binary.LittleEndian + + // bigEndian is a convenience variable since binary.BigEndian is quite + // long. + bigEndian = binary.BigEndian +) + +// binaryFreeList defines a concurrent safe free list of byte slices (up to the +// maximum number defined by the binaryFreeListMaxItems constant) that have a +// cap of 8 (thus it supports up to a uint64). It is used to provide temporary +// buffers for serializing and deserializing primitive numbers to and from their +// binary encoding in order to greatly reduce the number of allocations +// required. +// +// For convenience, functions are provided for each of the primitive unsigned +// integers that automatically obtain a buffer from the free list, perform the +// necessary binary conversion, read from or write to the given io.Reader or +// io.Writer, and return the buffer to the free list. +type binaryFreeList chan []byte + +// Borrow returns a byte slice from the free list with a length of 8. A new +// buffer is allocated if there are not any available on the free list. +func (l binaryFreeList) Borrow() []byte { + var buf []byte + select { + case buf = <-l: + default: + buf = make([]byte, 8) + } + return buf[:8] +} + +// Return puts the provided byte slice back on the free list. The buffer MUST +// have been obtained via the Borrow function and therefore have a cap of 8. +func (l binaryFreeList) Return(buf []byte) { + select { + case l <- buf: + default: + // Let it go to the garbage collector. + } +} + +// Uint8 reads a single byte from the provided reader using a buffer from the +// free list and returns it as a uint8. +func (l binaryFreeList) Uint8(r io.Reader) (uint8, error) { + buf := l.Borrow()[:1] + if _, err := io.ReadFull(r, buf); err != nil { + l.Return(buf) + return 0, err + } + rv := buf[0] + l.Return(buf) + return rv, nil +} + +// Uint16 reads two bytes from the provided reader using a buffer from the +// free list, converts it to a number using the provided byte order, and returns +// the resulting uint16. +func (l binaryFreeList) Uint16(r io.Reader, byteOrder binary.ByteOrder) (uint16, error) { + buf := l.Borrow()[:2] + if _, err := io.ReadFull(r, buf); err != nil { + l.Return(buf) + return 0, err + } + rv := byteOrder.Uint16(buf) + l.Return(buf) + return rv, nil +} + +// Uint32 reads four bytes from the provided reader using a buffer from the +// free list, converts it to a number using the provided byte order, and returns +// the resulting uint32. +func (l binaryFreeList) Uint32(r io.Reader, byteOrder binary.ByteOrder) (uint32, error) { + buf := l.Borrow()[:4] + if _, err := io.ReadFull(r, buf); err != nil { + l.Return(buf) + return 0, err + } + rv := byteOrder.Uint32(buf) + l.Return(buf) + return rv, nil +} + +// Uint64 reads eight bytes from the provided reader using a buffer from the +// free list, converts it to a number using the provided byte order, and returns +// the resulting uint64. +func (l binaryFreeList) Uint64(r io.Reader, byteOrder binary.ByteOrder) (uint64, error) { + buf := l.Borrow()[:8] + if _, err := io.ReadFull(r, buf); err != nil { + l.Return(buf) + return 0, err + } + rv := byteOrder.Uint64(buf) + l.Return(buf) + return rv, nil +} + +// PutUint8 copies the provided uint8 into a buffer from the free list and +// writes the resulting byte to the given writer. +func (l binaryFreeList) PutUint8(w io.Writer, val uint8) error { + buf := l.Borrow()[:1] + buf[0] = val + _, err := w.Write(buf) + l.Return(buf) + return err +} + +// PutUint16 serializes the provided uint16 using the given byte order into a +// buffer from the free list and writes the resulting two bytes to the given +// writer. +func (l binaryFreeList) PutUint16(w io.Writer, byteOrder binary.ByteOrder, val uint16) error { + buf := l.Borrow()[:2] + byteOrder.PutUint16(buf, val) + _, err := w.Write(buf) + l.Return(buf) + return err +} + +// PutUint32 serializes the provided uint32 using the given byte order into a +// buffer from the free list and writes the resulting four bytes to the given +// writer. +func (l binaryFreeList) PutUint32(w io.Writer, byteOrder binary.ByteOrder, val uint32) error { + buf := l.Borrow()[:4] + byteOrder.PutUint32(buf, val) + _, err := w.Write(buf) + l.Return(buf) + return err +} + +// PutUint64 serializes the provided uint64 using the given byte order into a +// buffer from the free list and writes the resulting eight bytes to the given +// writer. +func (l binaryFreeList) PutUint64(w io.Writer, byteOrder binary.ByteOrder, val uint64) error { + buf := l.Borrow()[:8] + byteOrder.PutUint64(buf, val) + _, err := w.Write(buf) + l.Return(buf) + return err +} + +// binarySerializer provides a free list of buffers to use for serializing and +// deserializing primitive integer values to and from io.Readers and io.Writers. +var binarySerializer binaryFreeList = make(chan []byte, binaryFreeListMaxItems) + +// errNonCanonicalVarInt is the common format string used for non-canonically +// encoded variable length integer errors. +var errNonCanonicalVarInt = "non-canonical varint %x - discriminant %x must " + + "encode a value greater than %x" + +// uint32Time represents a unix timestamp encoded with a uint32. It is used as +// a way to signal the readElement function how to decode a timestamp into a Go +// time.Time since it is otherwise ambiguous. +type uint32Time time.Time + +// int64Time represents a unix timestamp encoded with an int64. It is used as +// a way to signal the readElement function how to decode a timestamp into a Go +// time.Time since it is otherwise ambiguous. +type int64Time time.Time + +// readElement reads the next sequence of bytes from r using little endian +// depending on the concrete type of element pointed to. +func readElement(r io.Reader, element interface{}) error { + // Attempt to read the element based on the concrete type via fast + // type assertions first. + switch e := element.(type) { + case *int32: + rv, err := binarySerializer.Uint32(r, littleEndian) + if err != nil { + return err + } + *e = int32(rv) + return nil + + case *uint32: + rv, err := binarySerializer.Uint32(r, littleEndian) + if err != nil { + return err + } + *e = rv + return nil + + case *int64: + rv, err := binarySerializer.Uint64(r, littleEndian) + if err != nil { + return err + } + *e = int64(rv) + return nil + + case *uint64: + rv, err := binarySerializer.Uint64(r, littleEndian) + if err != nil { + return err + } + *e = rv + return nil + + case *bool: + rv, err := binarySerializer.Uint8(r) + if err != nil { + return err + } + if rv == 0x00 { + *e = false + } else { + *e = true + } + return nil + + // Unix timestamp encoded as a uint32. + case *uint32Time: + rv, err := binarySerializer.Uint32(r, binary.LittleEndian) + if err != nil { + return err + } + *e = uint32Time(time.Unix(int64(rv), 0)) + return nil + + // Unix timestamp encoded as an int64. + case *int64Time: + rv, err := binarySerializer.Uint64(r, binary.LittleEndian) + if err != nil { + return err + } + *e = int64Time(time.Unix(int64(rv), 0)) + return nil + + // Message header checksum. + case *[4]byte: + _, err := io.ReadFull(r, e[:]) + if err != nil { + return err + } + return nil + + // Message header command. + case *[CommandSize]uint8: + _, err := io.ReadFull(r, e[:]) + if err != nil { + return err + } + return nil + + // IP address. + case *[16]byte: + _, err := io.ReadFull(r, e[:]) + if err != nil { + return err + } + return nil + + case *chainhash.Hash: + _, err := io.ReadFull(r, e[:]) + if err != nil { + return err + } + return nil + + case *ServiceFlag: + rv, err := binarySerializer.Uint64(r, littleEndian) + if err != nil { + return err + } + *e = ServiceFlag(rv) + return nil + + case *InvType: + rv, err := binarySerializer.Uint32(r, littleEndian) + if err != nil { + return err + } + *e = InvType(rv) + return nil + + case *BitcoinNet: + rv, err := binarySerializer.Uint32(r, littleEndian) + if err != nil { + return err + } + *e = BitcoinNet(rv) + return nil + + case *BloomUpdateType: + rv, err := binarySerializer.Uint8(r) + if err != nil { + return err + } + *e = BloomUpdateType(rv) + return nil + + case *RejectCode: + rv, err := binarySerializer.Uint8(r) + if err != nil { + return err + } + *e = RejectCode(rv) + return nil + } + + // Fall back to the slower binary.Read if a fast path was not available + // above. + return binary.Read(r, littleEndian, element) +} + +// readElements reads multiple items from r. It is equivalent to multiple +// calls to readElement. +func readElements(r io.Reader, elements ...interface{}) error { + for _, element := range elements { + err := readElement(r, element) + if err != nil { + return err + } + } + return nil +} + +// writeElement writes the little endian representation of element to w. +func writeElement(w io.Writer, element interface{}) error { + // Attempt to write the element based on the concrete type via fast + // type assertions first. + switch e := element.(type) { + case int32: + err := binarySerializer.PutUint32(w, littleEndian, uint32(e)) + if err != nil { + return err + } + return nil + + case uint32: + err := binarySerializer.PutUint32(w, littleEndian, e) + if err != nil { + return err + } + return nil + + case int64: + err := binarySerializer.PutUint64(w, littleEndian, uint64(e)) + if err != nil { + return err + } + return nil + + case uint64: + err := binarySerializer.PutUint64(w, littleEndian, e) + if err != nil { + return err + } + return nil + + case bool: + var err error + if e { + err = binarySerializer.PutUint8(w, 0x01) + } else { + err = binarySerializer.PutUint8(w, 0x00) + } + if err != nil { + return err + } + return nil + + // Message header checksum. + case [4]byte: + _, err := w.Write(e[:]) + if err != nil { + return err + } + return nil + + // Message header command. + case [CommandSize]uint8: + _, err := w.Write(e[:]) + if err != nil { + return err + } + return nil + + // IP address. + case [16]byte: + _, err := w.Write(e[:]) + if err != nil { + return err + } + return nil + + case *chainhash.Hash: + _, err := w.Write(e[:]) + if err != nil { + return err + } + return nil + + case ServiceFlag: + err := binarySerializer.PutUint64(w, littleEndian, uint64(e)) + if err != nil { + return err + } + return nil + + case InvType: + err := binarySerializer.PutUint32(w, littleEndian, uint32(e)) + if err != nil { + return err + } + return nil + + case BitcoinNet: + err := binarySerializer.PutUint32(w, littleEndian, uint32(e)) + if err != nil { + return err + } + return nil + + case BloomUpdateType: + err := binarySerializer.PutUint8(w, uint8(e)) + if err != nil { + return err + } + return nil + + case RejectCode: + err := binarySerializer.PutUint8(w, uint8(e)) + if err != nil { + return err + } + return nil + } + + // Fall back to the slower binary.Write if a fast path was not available + // above. + return binary.Write(w, littleEndian, element) +} + +// writeElements writes multiple items to w. It is equivalent to multiple +// calls to writeElement. +func writeElements(w io.Writer, elements ...interface{}) error { + for _, element := range elements { + err := writeElement(w, element) + if err != nil { + return err + } + } + return nil +} + +// ReadVarInt reads a variable length integer from r and returns it as a uint64. +func ReadVarInt(r io.Reader, pver uint32) (uint64, error) { + discriminant, err := binarySerializer.Uint8(r) + if err != nil { + return 0, err + } + + var rv uint64 + switch discriminant { + case 0xff: + sv, err := binarySerializer.Uint64(r, littleEndian) + if err != nil { + return 0, err + } + rv = sv + + // The encoding is not canonical if the value could have been + // encoded using fewer bytes. + min := uint64(0x100000000) + if rv < min { + return 0, messageError("ReadVarInt", fmt.Sprintf( + errNonCanonicalVarInt, rv, discriminant, min)) + } + + case 0xfe: + sv, err := binarySerializer.Uint32(r, littleEndian) + if err != nil { + return 0, err + } + rv = uint64(sv) + + // The encoding is not canonical if the value could have been + // encoded using fewer bytes. + min := uint64(0x10000) + if rv < min { + return 0, messageError("ReadVarInt", fmt.Sprintf( + errNonCanonicalVarInt, rv, discriminant, min)) + } + + case 0xfd: + sv, err := binarySerializer.Uint16(r, littleEndian) + if err != nil { + return 0, err + } + rv = uint64(sv) + + // The encoding is not canonical if the value could have been + // encoded using fewer bytes. + min := uint64(0xfd) + if rv < min { + return 0, messageError("ReadVarInt", fmt.Sprintf( + errNonCanonicalVarInt, rv, discriminant, min)) + } + + default: + rv = uint64(discriminant) + } + + return rv, nil +} + +// WriteVarInt serializes val to w using a variable number of bytes depending +// on its value. +func WriteVarInt(w io.Writer, pver uint32, val uint64) error { + if val < 0xfd { + return binarySerializer.PutUint8(w, uint8(val)) + } + + if val <= math.MaxUint16 { + err := binarySerializer.PutUint8(w, 0xfd) + if err != nil { + return err + } + return binarySerializer.PutUint16(w, littleEndian, uint16(val)) + } + + if val <= math.MaxUint32 { + err := binarySerializer.PutUint8(w, 0xfe) + if err != nil { + return err + } + return binarySerializer.PutUint32(w, littleEndian, uint32(val)) + } + + err := binarySerializer.PutUint8(w, 0xff) + if err != nil { + return err + } + return binarySerializer.PutUint64(w, littleEndian, val) +} + +// VarIntSerializeSize returns the number of bytes it would take to serialize +// val as a variable length integer. +func VarIntSerializeSize(val uint64) int { + // The value is small enough to be represented by itself, so it's + // just 1 byte. + if val < 0xfd { + return 1 + } + + // Discriminant 1 byte plus 2 bytes for the uint16. + if val <= math.MaxUint16 { + return 3 + } + + // Discriminant 1 byte plus 4 bytes for the uint32. + if val <= math.MaxUint32 { + return 5 + } + + // Discriminant 1 byte plus 8 bytes for the uint64. + return 9 +} + +// ReadVarString reads a variable length string from r and returns it as a Go +// string. A variable length string is encoded as a variable length integer +// containing the length of the string followed by the bytes that represent the +// string itself. An error is returned if the length is greater than the +// maximum block payload size since it helps protect against memory exhaustion +// attacks and forced panics through malformed messages. +func ReadVarString(r io.Reader, pver uint32) (string, error) { + count, err := ReadVarInt(r, pver) + if err != nil { + return "", err + } + + // Prevent variable length strings that are larger than the maximum + // message size. It would be possible to cause memory exhaustion and + // panics without a sane upper bound on this count. + if count > MaxMessagePayload { + str := fmt.Sprintf("variable length string is too long "+ + "[count %d, max %d]", count, MaxMessagePayload) + return "", messageError("ReadVarString", str) + } + + buf := make([]byte, count) + _, err = io.ReadFull(r, buf) + if err != nil { + return "", err + } + return string(buf), nil +} + +// WriteVarString serializes str to w as a variable length integer containing +// the length of the string followed by the bytes that represent the string +// itself. +func WriteVarString(w io.Writer, pver uint32, str string) error { + err := WriteVarInt(w, pver, uint64(len(str))) + if err != nil { + return err + } + _, err = w.Write([]byte(str)) + return err +} + +// ReadVarBytes reads a variable length byte array. A byte array is encoded +// as a varInt containing the length of the array followed by the bytes +// themselves. An error is returned if the length is greater than the +// passed maxAllowed parameter which helps protect against memory exhuastion +// attacks and forced panics thorugh malformed messages. The fieldName +// parameter is only used for the error message so it provides more context in +// the error. +func ReadVarBytes(r io.Reader, pver uint32, maxAllowed uint32, + fieldName string) ([]byte, error) { + + count, err := ReadVarInt(r, pver) + if err != nil { + return nil, err + } + + // Prevent byte array larger than the max message size. It would + // be possible to cause memory exhaustion and panics without a sane + // upper bound on this count. + if count > uint64(maxAllowed) { + str := fmt.Sprintf("%s is larger than the max allowed size "+ + "[count %d, max %d]", fieldName, count, maxAllowed) + return nil, messageError("ReadVarBytes", str) + } + + b := make([]byte, count) + _, err = io.ReadFull(r, b) + if err != nil { + return nil, err + } + return b, nil +} + +// WriteVarBytes serializes a variable length byte array to w as a varInt +// containing the number of bytes, followed by the bytes themselves. +func WriteVarBytes(w io.Writer, pver uint32, bytes []byte) error { + slen := uint64(len(bytes)) + err := WriteVarInt(w, pver, slen) + if err != nil { + return err + } + + _, err = w.Write(bytes) + return err +} + +// randomUint64 returns a cryptographically random uint64 value. This +// unexported version takes a reader primarily to ensure the error paths +// can be properly tested by passing a fake reader in the tests. +func randomUint64(r io.Reader) (uint64, error) { + rv, err := binarySerializer.Uint64(r, bigEndian) + if err != nil { + return 0, err + } + return rv, nil +} + +// RandomUint64 returns a cryptographically random uint64 value. +func RandomUint64() (uint64, error) { + return randomUint64(rand.Reader) +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/doc.go b/vendor/github.com/ltcsuite/ltcd/wire/doc.go new file mode 100644 index 0000000000..65c88f02ce --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/doc.go @@ -0,0 +1,162 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* +Package wire implements the bitcoin wire protocol. + +For the complete details of the bitcoin protocol, see the official wiki entry +at https://en.bitcoin.it/wiki/Protocol_specification. The following only serves +as a quick overview to provide information on how to use the package. + +At a high level, this package provides support for marshalling and unmarshalling +supported bitcoin messages to and from the wire. This package does not deal +with the specifics of message handling such as what to do when a message is +received. This provides the caller with a high level of flexibility. + +Bitcoin Message Overview + +The bitcoin protocol consists of exchanging messages between peers. Each +message is preceded by a header which identifies information about it such as +which bitcoin network it is a part of, its type, how big it is, and a checksum +to verify validity. All encoding and decoding of message headers is handled by +this package. + +To accomplish this, there is a generic interface for bitcoin messages named +Message which allows messages of any type to be read, written, or passed around +through channels, functions, etc. In addition, concrete implementations of most +of the currently supported bitcoin messages are provided. For these supported +messages, all of the details of marshalling and unmarshalling to and from the +wire using bitcoin encoding are handled so the caller doesn't have to concern +themselves with the specifics. + +Message Interaction + +The following provides a quick summary of how the bitcoin messages are intended +to interact with one another. As stated above, these interactions are not +directly handled by this package. For more in-depth details about the +appropriate interactions, see the official bitcoin protocol wiki entry at +https://en.bitcoin.it/wiki/Protocol_specification. + +The initial handshake consists of two peers sending each other a version message +(MsgVersion) followed by responding with a verack message (MsgVerAck). Both +peers use the information in the version message (MsgVersion) to negotiate +things such as protocol version and supported services with each other. Once +the initial handshake is complete, the following chart indicates message +interactions in no particular order. + + Peer A Sends Peer B Responds + ---------------------------------------------------------------------------- + getaddr message (MsgGetAddr) addr message (MsgAddr) + getblocks message (MsgGetBlocks) inv message (MsgInv) + inv message (MsgInv) getdata message (MsgGetData) + getdata message (MsgGetData) block message (MsgBlock) -or- + tx message (MsgTx) -or- + notfound message (MsgNotFound) + getheaders message (MsgGetHeaders) headers message (MsgHeaders) + ping message (MsgPing) pong message (MsgHeaders)* -or- + (none -- Ability to send message is enough) + + NOTES: + * The pong message was not added until later protocol versions as defined + in BIP0031. The BIP0031Version constant can be used to detect a recent + enough protocol version for this purpose (version > BIP0031Version). + +Common Parameters + +There are several common parameters that arise when using this package to read +and write bitcoin messages. The following sections provide a quick overview of +these parameters so the next sections can build on them. + +Protocol Version + +The protocol version should be negotiated with the remote peer at a higher +level than this package via the version (MsgVersion) message exchange, however, +this package provides the wire.ProtocolVersion constant which indicates the +latest protocol version this package supports and is typically the value to use +for all outbound connections before a potentially lower protocol version is +negotiated. + +Litecoin Network + +The bitcoin network is a magic number which is used to identify the start of a +message and which bitcoin network the message applies to. This package provides +the following constants: + + wire.MainNet + wire.TestNet (Regression test network) + wire.TestNet4 (Test network version 4) + wire.SimNet (Simulation test network) + +Determining Message Type + +As discussed in the bitcoin message overview section, this package reads +and writes bitcoin messages using a generic interface named Message. In +order to determine the actual concrete type of the message, use a type +switch or type assertion. An example of a type switch follows: + + // Assumes msg is already a valid concrete message such as one created + // via NewMsgVersion or read via ReadMessage. + switch msg := msg.(type) { + case *wire.MsgVersion: + // The message is a pointer to a MsgVersion struct. + fmt.Printf("Protocol version: %v", msg.ProtocolVersion) + case *wire.MsgBlock: + // The message is a pointer to a MsgBlock struct. + fmt.Printf("Number of tx in block: %v", msg.Header.TxnCount) + } + +Reading Messages + +In order to unmarshall bitcoin messages from the wire, use the ReadMessage +function. It accepts any io.Reader, but typically this will be a net.Conn to +a remote node running a bitcoin peer. Example syntax is: + + // Reads and validates the next bitcoin message from conn using the + // protocol version pver and the bitcoin network btcnet. The returns + // are a wire.Message, a []byte which contains the unmarshalled + // raw payload, and a possible error. + msg, rawPayload, err := wire.ReadMessage(conn, pver, btcnet) + if err != nil { + // Log and handle the error + } + +Writing Messages + +In order to marshall bitcoin messages to the wire, use the WriteMessage +function. It accepts any io.Writer, but typically this will be a net.Conn to +a remote node running a bitcoin peer. Example syntax to request addresses +from a remote peer is: + + // Create a new getaddr bitcoin message. + msg := wire.NewMsgGetAddr() + + // Writes a bitcoin message msg to conn using the protocol version + // pver, and the bitcoin network btcnet. The return is a possible + // error. + err := wire.WriteMessage(conn, msg, pver, btcnet) + if err != nil { + // Log and handle the error + } + +Errors + +Errors returned by this package are either the raw errors provided by underlying +calls to read/write from streams such as io.EOF, io.ErrUnexpectedEOF, and +io.ErrShortWrite, or of type wire.MessageError. This allows the caller to +differentiate between general IO errors and malformed messages through type +assertions. + +Bitcoin Improvement Proposals + +This package includes spec changes outlined by the following BIPs: + + BIP0014 (https://github.com/bitcoin/bips/blob/master/bip-0014.mediawiki) + BIP0031 (https://github.com/bitcoin/bips/blob/master/bip-0031.mediawiki) + BIP0035 (https://github.com/bitcoin/bips/blob/master/bip-0035.mediawiki) + BIP0037 (https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki) + BIP0111 (https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki) + BIP0130 (https://github.com/bitcoin/bips/blob/master/bip-0130.mediawiki) + BIP0133 (https://github.com/bitcoin/bips/blob/master/bip-0133.mediawiki) +*/ +package wire diff --git a/vendor/github.com/ltcsuite/ltcd/wire/error.go b/vendor/github.com/ltcsuite/ltcd/wire/error.go new file mode 100644 index 0000000000..755c2db49a --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/error.go @@ -0,0 +1,34 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" +) + +// MessageError describes an issue with a message. +// An example of some potential issues are messages from the wrong bitcoin +// network, invalid commands, mismatched checksums, and exceeding max payloads. +// +// This provides a mechanism for the caller to type assert the error to +// differentiate between general io errors such as io.EOF and issues that +// resulted from malformed messages. +type MessageError struct { + Func string // Function name + Description string // Human readable description of the issue +} + +// Error satisfies the error interface and prints human-readable errors. +func (e *MessageError) Error() string { + if e.Func != "" { + return fmt.Sprintf("%v: %v", e.Func, e.Description) + } + return e.Description +} + +// messageError creates an error for the given function and description. +func messageError(f string, desc string) *MessageError { + return &MessageError{Func: f, Description: desc} +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/invvect.go b/vendor/github.com/ltcsuite/ltcd/wire/invvect.go new file mode 100644 index 0000000000..0bb52e3013 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/invvect.go @@ -0,0 +1,86 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +const ( + // MaxInvPerMsg is the maximum number of inventory vectors that can be in a + // single bitcoin inv message. + MaxInvPerMsg = 50000 + + // Maximum payload size for an inventory vector. + maxInvVectPayload = 4 + chainhash.HashSize + + // InvWitnessFlag denotes that the inventory vector type is requesting, + // or sending a version which includes witness data. + InvWitnessFlag = 1 << 30 +) + +// InvType represents the allowed types of inventory vectors. See InvVect. +type InvType uint32 + +// These constants define the various supported inventory vector types. +const ( + InvTypeError InvType = 0 + InvTypeTx InvType = 1 + InvTypeBlock InvType = 2 + InvTypeFilteredBlock InvType = 3 + InvTypeWitnessBlock InvType = InvTypeBlock | InvWitnessFlag + InvTypeWitnessTx InvType = InvTypeTx | InvWitnessFlag + InvTypeFilteredWitnessBlock InvType = InvTypeFilteredBlock | InvWitnessFlag +) + +// Map of service flags back to their constant names for pretty printing. +var ivStrings = map[InvType]string{ + InvTypeError: "ERROR", + InvTypeTx: "MSG_TX", + InvTypeBlock: "MSG_BLOCK", + InvTypeFilteredBlock: "MSG_FILTERED_BLOCK", + InvTypeWitnessBlock: "MSG_WITNESS_BLOCK", + InvTypeWitnessTx: "MSG_WITNESS_TX", + InvTypeFilteredWitnessBlock: "MSG_FILTERED_WITNESS_BLOCK", +} + +// String returns the InvType in human-readable form. +func (invtype InvType) String() string { + if s, ok := ivStrings[invtype]; ok { + return s + } + + return fmt.Sprintf("Unknown InvType (%d)", uint32(invtype)) +} + +// InvVect defines a bitcoin inventory vector which is used to describe data, +// as specified by the Type field, that a peer wants, has, or does not have to +// another peer. +type InvVect struct { + Type InvType // Type of data + Hash chainhash.Hash // Hash of the data +} + +// NewInvVect returns a new InvVect using the provided type and hash. +func NewInvVect(typ InvType, hash *chainhash.Hash) *InvVect { + return &InvVect{ + Type: typ, + Hash: *hash, + } +} + +// readInvVect reads an encoded InvVect from r depending on the protocol +// version. +func readInvVect(r io.Reader, pver uint32, iv *InvVect) error { + return readElements(r, &iv.Type, &iv.Hash) +} + +// writeInvVect serializes an InvVect to w depending on the protocol version. +func writeInvVect(w io.Writer, pver uint32, iv *InvVect) error { + return writeElements(w, iv.Type, &iv.Hash) +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/message.go b/vendor/github.com/ltcsuite/ltcd/wire/message.go new file mode 100644 index 0000000000..775da73777 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/message.go @@ -0,0 +1,428 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "bytes" + "fmt" + "io" + "unicode/utf8" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +// MessageHeaderSize is the number of bytes in a bitcoin message header. +// Bitcoin network (magic) 4 bytes + command 12 bytes + payload length 4 bytes + +// checksum 4 bytes. +const MessageHeaderSize = 24 + +// CommandSize is the fixed size of all commands in the common bitcoin message +// header. Shorter commands must be zero padded. +const CommandSize = 12 + +// MaxMessagePayload is the maximum bytes a message can be regardless of other +// individual limits imposed by messages themselves. +const MaxMessagePayload = (1024 * 1024 * 32) // 32MB + +// Commands used in bitcoin message headers which describe the type of message. +const ( + CmdVersion = "version" + CmdVerAck = "verack" + CmdGetAddr = "getaddr" + CmdAddr = "addr" + CmdGetBlocks = "getblocks" + CmdInv = "inv" + CmdGetData = "getdata" + CmdNotFound = "notfound" + CmdBlock = "block" + CmdTx = "tx" + CmdGetHeaders = "getheaders" + CmdHeaders = "headers" + CmdPing = "ping" + CmdPong = "pong" + CmdAlert = "alert" + CmdMemPool = "mempool" + CmdFilterAdd = "filteradd" + CmdFilterClear = "filterclear" + CmdFilterLoad = "filterload" + CmdMerkleBlock = "merkleblock" + CmdReject = "reject" + CmdSendHeaders = "sendheaders" + CmdFeeFilter = "feefilter" + CmdGetCFilter = "getcfilter" + CmdGetCFHeaders = "getcfheaders" + CmdCFilter = "cfilter" + CmdCFHeaders = "cfheaders" +) + +// MessageEncoding represents the wire message encoding format to be used. +type MessageEncoding uint32 + +const ( + // BaseEncoding encodes all messages in the default format specified + // for the Bitcoin wire protocol. + BaseEncoding MessageEncoding = 1 << iota + + // WitnessEncoding encodes all messages other than transaction messages + // using the default Bitcoin wire protocol specification. For transaction + // messages, the new encoding format detailed in BIP0144 will be used. + WitnessEncoding +) + +// LatestEncoding is the most recently specified encoding for the Bitcoin wire +// protocol. +var LatestEncoding = WitnessEncoding + +// Message is an interface that describes a bitcoin message. A type that +// implements Message has complete control over the representation of its data +// and may therefore contain additional or fewer fields than those which +// are used directly in the protocol encoded message. +type Message interface { + BtcDecode(io.Reader, uint32, MessageEncoding) error + BtcEncode(io.Writer, uint32, MessageEncoding) error + Command() string + MaxPayloadLength(uint32) uint32 +} + +// makeEmptyMessage creates a message of the appropriate concrete type based +// on the command. +func makeEmptyMessage(command string) (Message, error) { + var msg Message + switch command { + case CmdVersion: + msg = &MsgVersion{} + + case CmdVerAck: + msg = &MsgVerAck{} + + case CmdGetAddr: + msg = &MsgGetAddr{} + + case CmdAddr: + msg = &MsgAddr{} + + case CmdGetBlocks: + msg = &MsgGetBlocks{} + + case CmdBlock: + msg = &MsgBlock{} + + case CmdInv: + msg = &MsgInv{} + + case CmdGetData: + msg = &MsgGetData{} + + case CmdNotFound: + msg = &MsgNotFound{} + + case CmdTx: + msg = &MsgTx{} + + case CmdPing: + msg = &MsgPing{} + + case CmdPong: + msg = &MsgPong{} + + case CmdGetHeaders: + msg = &MsgGetHeaders{} + + case CmdHeaders: + msg = &MsgHeaders{} + + case CmdAlert: + msg = &MsgAlert{} + + case CmdMemPool: + msg = &MsgMemPool{} + + case CmdFilterAdd: + msg = &MsgFilterAdd{} + + case CmdFilterClear: + msg = &MsgFilterClear{} + + case CmdFilterLoad: + msg = &MsgFilterLoad{} + + case CmdMerkleBlock: + msg = &MsgMerkleBlock{} + + case CmdReject: + msg = &MsgReject{} + + case CmdSendHeaders: + msg = &MsgSendHeaders{} + + case CmdFeeFilter: + msg = &MsgFeeFilter{} + + case CmdGetCFilter: + msg = &MsgGetCFilter{} + + case CmdGetCFHeaders: + msg = &MsgGetCFHeaders{} + + case CmdCFilter: + msg = &MsgCFilter{} + + case CmdCFHeaders: + msg = &MsgCFHeaders{} + + default: + return nil, fmt.Errorf("unhandled command [%s]", command) + } + return msg, nil +} + +// messageHeader defines the header structure for all bitcoin protocol messages. +type messageHeader struct { + magic BitcoinNet // 4 bytes + command string // 12 bytes + length uint32 // 4 bytes + checksum [4]byte // 4 bytes +} + +// readMessageHeader reads a bitcoin message header from r. +func readMessageHeader(r io.Reader) (int, *messageHeader, error) { + // Since readElements doesn't return the amount of bytes read, attempt + // to read the entire header into a buffer first in case there is a + // short read so the proper amount of read bytes are known. This works + // since the header is a fixed size. + var headerBytes [MessageHeaderSize]byte + n, err := io.ReadFull(r, headerBytes[:]) + if err != nil { + return n, nil, err + } + hr := bytes.NewReader(headerBytes[:]) + + // Create and populate a messageHeader struct from the raw header bytes. + hdr := messageHeader{} + var command [CommandSize]byte + readElements(hr, &hdr.magic, &command, &hdr.length, &hdr.checksum) + + // Strip trailing zeros from command string. + hdr.command = string(bytes.TrimRight(command[:], string(0))) + + return n, &hdr, nil +} + +// discardInput reads n bytes from reader r in chunks and discards the read +// bytes. This is used to skip payloads when various errors occur and helps +// prevent rogue nodes from causing massive memory allocation through forging +// header length. +func discardInput(r io.Reader, n uint32) { + maxSize := uint32(10 * 1024) // 10k at a time + numReads := n / maxSize + bytesRemaining := n % maxSize + if n > 0 { + buf := make([]byte, maxSize) + for i := uint32(0); i < numReads; i++ { + io.ReadFull(r, buf) + } + } + if bytesRemaining > 0 { + buf := make([]byte, bytesRemaining) + io.ReadFull(r, buf) + } +} + +// WriteMessageN writes a bitcoin Message to w including the necessary header +// information and returns the number of bytes written. This function is the +// same as WriteMessage except it also returns the number of bytes written. +func WriteMessageN(w io.Writer, msg Message, pver uint32, btcnet BitcoinNet) (int, error) { + return WriteMessageWithEncodingN(w, msg, pver, btcnet, BaseEncoding) +} + +// WriteMessage writes a bitcoin Message to w including the necessary header +// information. This function is the same as WriteMessageN except it doesn't +// doesn't return the number of bytes written. This function is mainly provided +// for backwards compatibility with the original API, but it's also useful for +// callers that don't care about byte counts. +func WriteMessage(w io.Writer, msg Message, pver uint32, btcnet BitcoinNet) error { + _, err := WriteMessageN(w, msg, pver, btcnet) + return err +} + +// WriteMessageWithEncodingN writes a bitcoin Message to w including the +// necessary header information and returns the number of bytes written. +// This function is the same as WriteMessageN except it also allows the caller +// to specify the message encoding format to be used when serializing wire +// messages. +func WriteMessageWithEncodingN(w io.Writer, msg Message, pver uint32, + btcnet BitcoinNet, encoding MessageEncoding) (int, error) { + + totalBytes := 0 + + // Enforce max command size. + var command [CommandSize]byte + cmd := msg.Command() + if len(cmd) > CommandSize { + str := fmt.Sprintf("command [%s] is too long [max %v]", + cmd, CommandSize) + return totalBytes, messageError("WriteMessage", str) + } + copy(command[:], []byte(cmd)) + + // Encode the message payload. + var bw bytes.Buffer + err := msg.BtcEncode(&bw, pver, encoding) + if err != nil { + return totalBytes, err + } + payload := bw.Bytes() + lenp := len(payload) + + // Enforce maximum overall message payload. + if lenp > MaxMessagePayload { + str := fmt.Sprintf("message payload is too large - encoded "+ + "%d bytes, but maximum message payload is %d bytes", + lenp, MaxMessagePayload) + return totalBytes, messageError("WriteMessage", str) + } + + // Enforce maximum message payload based on the message type. + mpl := msg.MaxPayloadLength(pver) + if uint32(lenp) > mpl { + str := fmt.Sprintf("message payload is too large - encoded "+ + "%d bytes, but maximum message payload size for "+ + "messages of type [%s] is %d.", lenp, cmd, mpl) + return totalBytes, messageError("WriteMessage", str) + } + + // Create header for the message. + hdr := messageHeader{} + hdr.magic = btcnet + hdr.command = cmd + hdr.length = uint32(lenp) + copy(hdr.checksum[:], chainhash.DoubleHashB(payload)[0:4]) + + // Encode the header for the message. This is done to a buffer + // rather than directly to the writer since writeElements doesn't + // return the number of bytes written. + hw := bytes.NewBuffer(make([]byte, 0, MessageHeaderSize)) + writeElements(hw, hdr.magic, command, hdr.length, hdr.checksum) + + // Write header. + n, err := w.Write(hw.Bytes()) + totalBytes += n + if err != nil { + return totalBytes, err + } + + // Write payload. + n, err = w.Write(payload) + totalBytes += n + return totalBytes, err +} + +// ReadMessageWithEncodingN reads, validates, and parses the next bitcoin Message +// from r for the provided protocol version and bitcoin network. It returns the +// number of bytes read in addition to the parsed Message and raw bytes which +// comprise the message. This function is the same as ReadMessageN except it +// allows the caller to specify which message encoding is to to consult when +// decoding wire messages. +func ReadMessageWithEncodingN(r io.Reader, pver uint32, btcnet BitcoinNet, + enc MessageEncoding) (int, Message, []byte, error) { + + totalBytes := 0 + n, hdr, err := readMessageHeader(r) + totalBytes += n + if err != nil { + return totalBytes, nil, nil, err + } + + // Enforce maximum message payload. + if hdr.length > MaxMessagePayload { + str := fmt.Sprintf("message payload is too large - header "+ + "indicates %d bytes, but max message payload is %d "+ + "bytes.", hdr.length, MaxMessagePayload) + return totalBytes, nil, nil, messageError("ReadMessage", str) + + } + + // Check for messages from the wrong bitcoin network. + if hdr.magic != btcnet { + discardInput(r, hdr.length) + str := fmt.Sprintf("message from other network [%v]", hdr.magic) + return totalBytes, nil, nil, messageError("ReadMessage", str) + } + + // Check for malformed commands. + command := hdr.command + if !utf8.ValidString(command) { + discardInput(r, hdr.length) + str := fmt.Sprintf("invalid command %v", []byte(command)) + return totalBytes, nil, nil, messageError("ReadMessage", str) + } + + // Create struct of appropriate message type based on the command. + msg, err := makeEmptyMessage(command) + if err != nil { + discardInput(r, hdr.length) + return totalBytes, nil, nil, messageError("ReadMessage", + err.Error()) + } + + // Check for maximum length based on the message type as a malicious client + // could otherwise create a well-formed header and set the length to max + // numbers in order to exhaust the machine's memory. + mpl := msg.MaxPayloadLength(pver) + if hdr.length > mpl { + discardInput(r, hdr.length) + str := fmt.Sprintf("payload exceeds max length - header "+ + "indicates %v bytes, but max payload size for "+ + "messages of type [%v] is %v.", hdr.length, command, mpl) + return totalBytes, nil, nil, messageError("ReadMessage", str) + } + + // Read payload. + payload := make([]byte, hdr.length) + n, err = io.ReadFull(r, payload) + totalBytes += n + if err != nil { + return totalBytes, nil, nil, err + } + + // Test checksum. + checksum := chainhash.DoubleHashB(payload)[0:4] + if !bytes.Equal(checksum[:], hdr.checksum[:]) { + str := fmt.Sprintf("payload checksum failed - header "+ + "indicates %v, but actual checksum is %v.", + hdr.checksum, checksum) + return totalBytes, nil, nil, messageError("ReadMessage", str) + } + + // Unmarshal message. NOTE: This must be a *bytes.Buffer since the + // MsgVersion BtcDecode function requires it. + pr := bytes.NewBuffer(payload) + err = msg.BtcDecode(pr, pver, enc) + if err != nil { + return totalBytes, nil, nil, err + } + + return totalBytes, msg, payload, nil +} + +// ReadMessageN reads, validates, and parses the next bitcoin Message from r for +// the provided protocol version and bitcoin network. It returns the number of +// bytes read in addition to the parsed Message and raw bytes which comprise the +// message. This function is the same as ReadMessage except it also returns the +// number of bytes read. +func ReadMessageN(r io.Reader, pver uint32, btcnet BitcoinNet) (int, Message, []byte, error) { + return ReadMessageWithEncodingN(r, pver, btcnet, BaseEncoding) +} + +// ReadMessage reads, validates, and parses the next bitcoin Message from r for +// the provided protocol version and bitcoin network. It returns the parsed +// Message and raw bytes which comprise the message. This function only differs +// from ReadMessageN in that it doesn't return the number of bytes read. This +// function is mainly provided for backwards compatibility with the original +// API, but it's also useful for callers that don't care about byte counts. +func ReadMessage(r io.Reader, pver uint32, btcnet BitcoinNet) (Message, []byte, error) { + _, msg, buf, err := ReadMessageN(r, pver, btcnet) + return msg, buf, err +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgaddr.go b/vendor/github.com/ltcsuite/ltcd/wire/msgaddr.go new file mode 100644 index 0000000000..ce945c8c06 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgaddr.go @@ -0,0 +1,143 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +// MaxAddrPerMsg is the maximum number of addresses that can be in a single +// bitcoin addr message (MsgAddr). +const MaxAddrPerMsg = 1000 + +// MsgAddr implements the Message interface and represents a bitcoin +// addr message. It is used to provide a list of known active peers on the +// network. An active peer is considered one that has transmitted a message +// within the last 3 hours. Nodes which have not transmitted in that time +// frame should be forgotten. Each message is limited to a maximum number of +// addresses, which is currently 1000. As a result, multiple messages must +// be used to relay the full list. +// +// Use the AddAddress function to build up the list of known addresses when +// sending an addr message to another peer. +type MsgAddr struct { + AddrList []*NetAddress +} + +// AddAddress adds a known active peer to the message. +func (msg *MsgAddr) AddAddress(na *NetAddress) error { + if len(msg.AddrList)+1 > MaxAddrPerMsg { + str := fmt.Sprintf("too many addresses in message [max %v]", + MaxAddrPerMsg) + return messageError("MsgAddr.AddAddress", str) + } + + msg.AddrList = append(msg.AddrList, na) + return nil +} + +// AddAddresses adds multiple known active peers to the message. +func (msg *MsgAddr) AddAddresses(netAddrs ...*NetAddress) error { + for _, na := range netAddrs { + err := msg.AddAddress(na) + if err != nil { + return err + } + } + return nil +} + +// ClearAddresses removes all addresses from the message. +func (msg *MsgAddr) ClearAddresses() { + msg.AddrList = []*NetAddress{} +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgAddr) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + + // Limit to max addresses per message. + if count > MaxAddrPerMsg { + str := fmt.Sprintf("too many addresses for message "+ + "[count %v, max %v]", count, MaxAddrPerMsg) + return messageError("MsgAddr.BtcDecode", str) + } + + addrList := make([]NetAddress, count) + msg.AddrList = make([]*NetAddress, 0, count) + for i := uint64(0); i < count; i++ { + na := &addrList[i] + err := readNetAddress(r, pver, na, true) + if err != nil { + return err + } + msg.AddAddress(na) + } + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgAddr) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + // Protocol versions before MultipleAddressVersion only allowed 1 address + // per message. + count := len(msg.AddrList) + if pver < MultipleAddressVersion && count > 1 { + str := fmt.Sprintf("too many addresses for message of "+ + "protocol version %v [count %v, max 1]", pver, count) + return messageError("MsgAddr.BtcEncode", str) + + } + if count > MaxAddrPerMsg { + str := fmt.Sprintf("too many addresses for message "+ + "[count %v, max %v]", count, MaxAddrPerMsg) + return messageError("MsgAddr.BtcEncode", str) + } + + err := WriteVarInt(w, pver, uint64(count)) + if err != nil { + return err + } + + for _, na := range msg.AddrList { + err = writeNetAddress(w, pver, na, true) + if err != nil { + return err + } + } + + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgAddr) Command() string { + return CmdAddr +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgAddr) MaxPayloadLength(pver uint32) uint32 { + if pver < MultipleAddressVersion { + // Num addresses (varInt) + a single net addresses. + return MaxVarIntPayload + maxNetAddressPayload(pver) + } + + // Num addresses (varInt) + max allowed addresses. + return MaxVarIntPayload + (MaxAddrPerMsg * maxNetAddressPayload(pver)) +} + +// NewMsgAddr returns a new bitcoin addr message that conforms to the +// Message interface. See MsgAddr for details. +func NewMsgAddr() *MsgAddr { + return &MsgAddr{ + AddrList: make([]*NetAddress, 0, MaxAddrPerMsg), + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgalert.go b/vendor/github.com/ltcsuite/ltcd/wire/msgalert.go new file mode 100644 index 0000000000..71c4e220fe --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgalert.go @@ -0,0 +1,407 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "bytes" + "fmt" + "io" +) + +// MsgAlert contains a payload and a signature: +// +// =============================================== +// | Field | Data Type | Size | +// =============================================== +// | payload | []uchar | ? | +// ----------------------------------------------- +// | signature | []uchar | ? | +// ----------------------------------------------- +// +// Here payload is an Alert serialized into a byte array to ensure that +// versions using incompatible alert formats can still relay +// alerts among one another. +// +// An Alert is the payload deserialized as follows: +// +// =============================================== +// | Field | Data Type | Size | +// =============================================== +// | Version | int32 | 4 | +// ----------------------------------------------- +// | RelayUntil | int64 | 8 | +// ----------------------------------------------- +// | Expiration | int64 | 8 | +// ----------------------------------------------- +// | ID | int32 | 4 | +// ----------------------------------------------- +// | Cancel | int32 | 4 | +// ----------------------------------------------- +// | SetCancel | set | ? | +// ----------------------------------------------- +// | MinVer | int32 | 4 | +// ----------------------------------------------- +// | MaxVer | int32 | 4 | +// ----------------------------------------------- +// | SetSubVer | set | ? | +// ----------------------------------------------- +// | Priority | int32 | 4 | +// ----------------------------------------------- +// | Comment | string | ? | +// ----------------------------------------------- +// | StatusBar | string | ? | +// ----------------------------------------------- +// | Reserved | string | ? | +// ----------------------------------------------- +// | Total (Fixed) | 45 | +// ----------------------------------------------- +// +// NOTE: +// * string is a VarString i.e VarInt length followed by the string itself +// * set is a VarInt followed by as many number of strings +// * set is a VarInt followed by as many number of ints +// * fixedAlertSize = 40 + 5*min(VarInt) = 40 + 5*1 = 45 +// +// Now we can define bounds on Alert size, SetCancel and SetSubVer + +// Fixed size of the alert payload +const fixedAlertSize = 45 + +// maxSignatureSize is the max size of an ECDSA signature. +// NOTE: Since this size is fixed and < 255, the size of VarInt required = 1. +const maxSignatureSize = 72 + +// maxAlertSize is the maximum size an alert. +// +// MessagePayload = VarInt(Alert) + Alert + VarInt(Signature) + Signature +// MaxMessagePayload = maxAlertSize + max(VarInt) + maxSignatureSize + 1 +const maxAlertSize = MaxMessagePayload - maxSignatureSize - MaxVarIntPayload - 1 + +// maxCountSetCancel is the maximum number of cancel IDs that could possibly +// fit into a maximum size alert. +// +// maxAlertSize = fixedAlertSize + max(SetCancel) + max(SetSubVer) + 3*(string) +// for caculating maximum number of cancel IDs, set all other var sizes to 0 +// maxAlertSize = fixedAlertSize + (MaxVarIntPayload-1) + x*sizeOf(int32) +// x = (maxAlertSize - fixedAlertSize - MaxVarIntPayload + 1) / 4 +const maxCountSetCancel = (maxAlertSize - fixedAlertSize - MaxVarIntPayload + 1) / 4 + +// maxCountSetSubVer is the maximum number of subversions that could possibly +// fit into a maximum size alert. +// +// maxAlertSize = fixedAlertSize + max(SetCancel) + max(SetSubVer) + 3*(string) +// for caculating maximum number of subversions, set all other var sizes to 0 +// maxAlertSize = fixedAlertSize + (MaxVarIntPayload-1) + x*sizeOf(string) +// x = (maxAlertSize - fixedAlertSize - MaxVarIntPayload + 1) / sizeOf(string) +// subversion would typically be something like "/Satoshi:0.7.2/" (15 bytes) +// so assuming < 255 bytes, sizeOf(string) = sizeOf(uint8) + 255 = 256 +const maxCountSetSubVer = (maxAlertSize - fixedAlertSize - MaxVarIntPayload + 1) / 256 + +// Alert contains the data deserialized from the MsgAlert payload. +type Alert struct { + // Alert format version + Version int32 + + // Timestamp beyond which nodes should stop relaying this alert + RelayUntil int64 + + // Timestamp beyond which this alert is no longer in effect and + // should be ignored + Expiration int64 + + // A unique ID number for this alert + ID int32 + + // All alerts with an ID less than or equal to this number should + // cancelled, deleted and not accepted in the future + Cancel int32 + + // All alert IDs contained in this set should be cancelled as above + SetCancel []int32 + + // This alert only applies to versions greater than or equal to this + // version. Other versions should still relay it. + MinVer int32 + + // This alert only applies to versions less than or equal to this version. + // Other versions should still relay it. + MaxVer int32 + + // If this set contains any elements, then only nodes that have their + // subVer contained in this set are affected by the alert. Other versions + // should still relay it. + SetSubVer []string + + // Relative priority compared to other alerts + Priority int32 + + // A comment on the alert that is not displayed + Comment string + + // The alert message that is displayed to the user + StatusBar string + + // Reserved + Reserved string +} + +// Serialize encodes the alert to w using the alert protocol encoding format. +func (alert *Alert) Serialize(w io.Writer, pver uint32) error { + err := writeElements(w, alert.Version, alert.RelayUntil, + alert.Expiration, alert.ID, alert.Cancel) + if err != nil { + return err + } + + count := len(alert.SetCancel) + if count > maxCountSetCancel { + str := fmt.Sprintf("too many cancel alert IDs for alert "+ + "[count %v, max %v]", count, maxCountSetCancel) + return messageError("Alert.Serialize", str) + } + err = WriteVarInt(w, pver, uint64(count)) + if err != nil { + return err + } + for i := 0; i < count; i++ { + err = writeElement(w, alert.SetCancel[i]) + if err != nil { + return err + } + } + + err = writeElements(w, alert.MinVer, alert.MaxVer) + if err != nil { + return err + } + + count = len(alert.SetSubVer) + if count > maxCountSetSubVer { + str := fmt.Sprintf("too many sub versions for alert "+ + "[count %v, max %v]", count, maxCountSetSubVer) + return messageError("Alert.Serialize", str) + } + err = WriteVarInt(w, pver, uint64(count)) + if err != nil { + return err + } + for i := 0; i < count; i++ { + err = WriteVarString(w, pver, alert.SetSubVer[i]) + if err != nil { + return err + } + } + + err = writeElement(w, alert.Priority) + if err != nil { + return err + } + err = WriteVarString(w, pver, alert.Comment) + if err != nil { + return err + } + err = WriteVarString(w, pver, alert.StatusBar) + if err != nil { + return err + } + return WriteVarString(w, pver, alert.Reserved) +} + +// Deserialize decodes from r into the receiver using the alert protocol +// encoding format. +func (alert *Alert) Deserialize(r io.Reader, pver uint32) error { + err := readElements(r, &alert.Version, &alert.RelayUntil, + &alert.Expiration, &alert.ID, &alert.Cancel) + if err != nil { + return err + } + + // SetCancel: first read a VarInt that contains + // count - the number of Cancel IDs, then + // iterate count times and read them + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + if count > maxCountSetCancel { + str := fmt.Sprintf("too many cancel alert IDs for alert "+ + "[count %v, max %v]", count, maxCountSetCancel) + return messageError("Alert.Deserialize", str) + } + alert.SetCancel = make([]int32, count) + for i := 0; i < int(count); i++ { + err := readElement(r, &alert.SetCancel[i]) + if err != nil { + return err + } + } + + err = readElements(r, &alert.MinVer, &alert.MaxVer) + if err != nil { + return err + } + + // SetSubVer: similar to SetCancel + // but read count number of sub-version strings + count, err = ReadVarInt(r, pver) + if err != nil { + return err + } + if count > maxCountSetSubVer { + str := fmt.Sprintf("too many sub versions for alert "+ + "[count %v, max %v]", count, maxCountSetSubVer) + return messageError("Alert.Deserialize", str) + } + alert.SetSubVer = make([]string, count) + for i := 0; i < int(count); i++ { + alert.SetSubVer[i], err = ReadVarString(r, pver) + if err != nil { + return err + } + } + + err = readElement(r, &alert.Priority) + if err != nil { + return err + } + alert.Comment, err = ReadVarString(r, pver) + if err != nil { + return err + } + alert.StatusBar, err = ReadVarString(r, pver) + if err != nil { + return err + } + alert.Reserved, err = ReadVarString(r, pver) + return err +} + +// NewAlert returns an new Alert with values provided. +func NewAlert(version int32, relayUntil int64, expiration int64, + id int32, cancel int32, setCancel []int32, minVer int32, + maxVer int32, setSubVer []string, priority int32, comment string, + statusBar string) *Alert { + return &Alert{ + Version: version, + RelayUntil: relayUntil, + Expiration: expiration, + ID: id, + Cancel: cancel, + SetCancel: setCancel, + MinVer: minVer, + MaxVer: maxVer, + SetSubVer: setSubVer, + Priority: priority, + Comment: comment, + StatusBar: statusBar, + Reserved: "", + } +} + +// NewAlertFromPayload returns an Alert with values deserialized from the +// serialized payload. +func NewAlertFromPayload(serializedPayload []byte, pver uint32) (*Alert, error) { + var alert Alert + r := bytes.NewReader(serializedPayload) + err := alert.Deserialize(r, pver) + if err != nil { + return nil, err + } + return &alert, nil +} + +// MsgAlert implements the Message interface and defines a bitcoin alert +// message. +// +// This is a signed message that provides notifications that the client should +// display if the signature matches the key. bitcoind/bitcoin-qt only checks +// against a signature from the core developers. +type MsgAlert struct { + // SerializedPayload is the alert payload serialized as a string so that the + // version can change but the Alert can still be passed on by older + // clients. + SerializedPayload []byte + + // Signature is the ECDSA signature of the message. + Signature []byte + + // Deserialized Payload + Payload *Alert +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgAlert) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + var err error + + msg.SerializedPayload, err = ReadVarBytes(r, pver, MaxMessagePayload, + "alert serialized payload") + if err != nil { + return err + } + + msg.Payload, err = NewAlertFromPayload(msg.SerializedPayload, pver) + if err != nil { + msg.Payload = nil + } + + msg.Signature, err = ReadVarBytes(r, pver, MaxMessagePayload, + "alert signature") + return err +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgAlert) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + var err error + var serializedpayload []byte + if msg.Payload != nil { + // try to Serialize Payload if possible + r := new(bytes.Buffer) + err = msg.Payload.Serialize(r, pver) + if err != nil { + // Serialize failed - ignore & fallback + // to SerializedPayload + serializedpayload = msg.SerializedPayload + } else { + serializedpayload = r.Bytes() + } + } else { + serializedpayload = msg.SerializedPayload + } + slen := uint64(len(serializedpayload)) + if slen == 0 { + return messageError("MsgAlert.BtcEncode", "empty serialized payload") + } + err = WriteVarBytes(w, pver, serializedpayload) + if err != nil { + return err + } + return WriteVarBytes(w, pver, msg.Signature) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgAlert) Command() string { + return CmdAlert +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgAlert) MaxPayloadLength(pver uint32) uint32 { + // Since this can vary depending on the message, make it the max + // size allowed. + return MaxMessagePayload +} + +// NewMsgAlert returns a new bitcoin alert message that conforms to the Message +// interface. See MsgAlert for details. +func NewMsgAlert(serializedPayload []byte, signature []byte) *MsgAlert { + return &MsgAlert{ + SerializedPayload: serializedPayload, + Signature: signature, + Payload: nil, + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgblock.go b/vendor/github.com/ltcsuite/ltcd/wire/msgblock.go new file mode 100644 index 0000000000..9651057605 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgblock.go @@ -0,0 +1,290 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "bytes" + "fmt" + "io" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +// defaultTransactionAlloc is the default size used for the backing array +// for transactions. The transaction array will dynamically grow as needed, but +// this figure is intended to provide enough space for the number of +// transactions in the vast majority of blocks without needing to grow the +// backing array multiple times. +const defaultTransactionAlloc = 2048 + +// MaxBlocksPerMsg is the maximum number of blocks allowed per message. +const MaxBlocksPerMsg = 500 + +// MaxBlockPayload is the maximum bytes a block message can be in bytes. +// After Segregated Witness, the max block payload has been raised to 4MB. +const MaxBlockPayload = 4000000 + +// maxTxPerBlock is the maximum number of transactions that could +// possibly fit into a block. +const maxTxPerBlock = (MaxBlockPayload / minTxPayload) + 1 + +// TxLoc holds locator data for the offset and length of where a transaction is +// located within a MsgBlock data buffer. +type TxLoc struct { + TxStart int + TxLen int +} + +// MsgBlock implements the Message interface and represents a bitcoin +// block message. It is used to deliver block and transaction information in +// response to a getdata message (MsgGetData) for a given block hash. +type MsgBlock struct { + Header BlockHeader + Transactions []*MsgTx +} + +// AddTransaction adds a transaction to the message. +func (msg *MsgBlock) AddTransaction(tx *MsgTx) error { + msg.Transactions = append(msg.Transactions, tx) + return nil + +} + +// ClearTransactions removes all transactions from the message. +func (msg *MsgBlock) ClearTransactions() { + msg.Transactions = make([]*MsgTx, 0, defaultTransactionAlloc) +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +// See Deserialize for decoding blocks stored to disk, such as in a database, as +// opposed to decoding blocks from the wire. +func (msg *MsgBlock) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + err := readBlockHeader(r, pver, &msg.Header) + if err != nil { + return err + } + + txCount, err := ReadVarInt(r, pver) + if err != nil { + return err + } + + // Prevent more transactions than could possibly fit into a block. + // It would be possible to cause memory exhaustion and panics without + // a sane upper bound on this count. + if txCount > maxTxPerBlock { + str := fmt.Sprintf("too many transactions to fit into a block "+ + "[count %d, max %d]", txCount, maxTxPerBlock) + return messageError("MsgBlock.BtcDecode", str) + } + + msg.Transactions = make([]*MsgTx, 0, txCount) + for i := uint64(0); i < txCount; i++ { + tx := MsgTx{} + err := tx.BtcDecode(r, pver, enc) + if err != nil { + return err + } + msg.Transactions = append(msg.Transactions, &tx) + } + + return nil +} + +// Deserialize decodes a block from r into the receiver using a format that is +// suitable for long-term storage such as a database while respecting the +// Version field in the block. This function differs from BtcDecode in that +// BtcDecode decodes from the bitcoin wire protocol as it was sent across the +// network. The wire encoding can technically differ depending on the protocol +// version and doesn't even really need to match the format of a stored block at +// all. As of the time this comment was written, the encoded block is the same +// in both instances, but there is a distinct difference and separating the two +// allows the API to be flexible enough to deal with changes. +func (msg *MsgBlock) Deserialize(r io.Reader) error { + // At the current time, there is no difference between the wire encoding + // at protocol version 0 and the stable long-term storage format. As + // a result, make use of BtcDecode. + // + // Passing an encoding type of WitnessEncoding to BtcEncode for the + // MessageEncoding parameter indicates that the transactions within the + // block are expected to be serialized according to the new + // serialization structure defined in BIP0141. + return msg.BtcDecode(r, 0, WitnessEncoding) +} + +// DeserializeNoWitness decodes a block from r into the receiver similar to +// Deserialize, however DeserializeWitness strips all (if any) witness data +// from the transactions within the block before encoding them. +func (msg *MsgBlock) DeserializeNoWitness(r io.Reader) error { + return msg.BtcDecode(r, 0, BaseEncoding) +} + +// DeserializeTxLoc decodes r in the same manner Deserialize does, but it takes +// a byte buffer instead of a generic reader and returns a slice containing the +// start and length of each transaction within the raw data that is being +// deserialized. +func (msg *MsgBlock) DeserializeTxLoc(r *bytes.Buffer) ([]TxLoc, error) { + fullLen := r.Len() + + // At the current time, there is no difference between the wire encoding + // at protocol version 0 and the stable long-term storage format. As + // a result, make use of existing wire protocol functions. + err := readBlockHeader(r, 0, &msg.Header) + if err != nil { + return nil, err + } + + txCount, err := ReadVarInt(r, 0) + if err != nil { + return nil, err + } + + // Prevent more transactions than could possibly fit into a block. + // It would be possible to cause memory exhaustion and panics without + // a sane upper bound on this count. + if txCount > maxTxPerBlock { + str := fmt.Sprintf("too many transactions to fit into a block "+ + "[count %d, max %d]", txCount, maxTxPerBlock) + return nil, messageError("MsgBlock.DeserializeTxLoc", str) + } + + // Deserialize each transaction while keeping track of its location + // within the byte stream. + msg.Transactions = make([]*MsgTx, 0, txCount) + txLocs := make([]TxLoc, txCount) + for i := uint64(0); i < txCount; i++ { + txLocs[i].TxStart = fullLen - r.Len() + tx := MsgTx{} + err := tx.Deserialize(r) + if err != nil { + return nil, err + } + msg.Transactions = append(msg.Transactions, &tx) + txLocs[i].TxLen = (fullLen - r.Len()) - txLocs[i].TxStart + } + + return txLocs, nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +// See Serialize for encoding blocks to be stored to disk, such as in a +// database, as opposed to encoding blocks for the wire. +func (msg *MsgBlock) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + err := writeBlockHeader(w, pver, &msg.Header) + if err != nil { + return err + } + + err = WriteVarInt(w, pver, uint64(len(msg.Transactions))) + if err != nil { + return err + } + + for _, tx := range msg.Transactions { + err = tx.BtcEncode(w, pver, enc) + if err != nil { + return err + } + } + + return nil +} + +// Serialize encodes the block to w using a format that suitable for long-term +// storage such as a database while respecting the Version field in the block. +// This function differs from BtcEncode in that BtcEncode encodes the block to +// the bitcoin wire protocol in order to be sent across the network. The wire +// encoding can technically differ depending on the protocol version and doesn't +// even really need to match the format of a stored block at all. As of the +// time this comment was written, the encoded block is the same in both +// instances, but there is a distinct difference and separating the two allows +// the API to be flexible enough to deal with changes. +func (msg *MsgBlock) Serialize(w io.Writer) error { + // At the current time, there is no difference between the wire encoding + // at protocol version 0 and the stable long-term storage format. As + // a result, make use of BtcEncode. + // + // Passing WitnessEncoding as the encoding type here indicates that + // each of the transactions should be serialized using the witness + // serialization structure defined in BIP0141. + return msg.BtcEncode(w, 0, WitnessEncoding) +} + +// SerializeNoWitness encodes a block to w using an identical format to +// Serialize, with all (if any) witness data stripped from all transactions. +// This method is provided in additon to the regular Serialize, in order to +// allow one to selectively encode transaction witness data to non-upgraded +// peers which are unaware of the new encoding. +func (msg *MsgBlock) SerializeNoWitness(w io.Writer) error { + return msg.BtcEncode(w, 0, BaseEncoding) +} + +// SerializeSize returns the number of bytes it would take to serialize the +// block, factoring in any witness data within transaction. +func (msg *MsgBlock) SerializeSize() int { + // Block header bytes + Serialized varint size for the number of + // transactions. + n := blockHeaderLen + VarIntSerializeSize(uint64(len(msg.Transactions))) + + for _, tx := range msg.Transactions { + n += tx.SerializeSize() + } + + return n +} + +// SerializeSizeStripped returns the number of bytes it would take to serialize +// the block, excluding any witness data (if any). +func (msg *MsgBlock) SerializeSizeStripped() int { + // Block header bytes + Serialized varint size for the number of + // transactions. + n := blockHeaderLen + VarIntSerializeSize(uint64(len(msg.Transactions))) + + for _, tx := range msg.Transactions { + n += tx.SerializeSizeStripped() + } + + return n +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgBlock) Command() string { + return CmdBlock +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgBlock) MaxPayloadLength(pver uint32) uint32 { + // Block header at 80 bytes + transaction count + max transactions + // which can vary up to the MaxBlockPayload (including the block header + // and transaction count). + return MaxBlockPayload +} + +// BlockHash computes the block identifier hash for this block. +func (msg *MsgBlock) BlockHash() chainhash.Hash { + return msg.Header.BlockHash() +} + +// TxHashes returns a slice of hashes of all of transactions in this block. +func (msg *MsgBlock) TxHashes() ([]chainhash.Hash, error) { + hashList := make([]chainhash.Hash, 0, len(msg.Transactions)) + for _, tx := range msg.Transactions { + hashList = append(hashList, tx.TxHash()) + } + return hashList, nil +} + +// NewMsgBlock returns a new bitcoin block message that conforms to the +// Message interface. See MsgBlock for details. +func NewMsgBlock(blockHeader *BlockHeader) *MsgBlock { + return &MsgBlock{ + Header: *blockHeader, + Transactions: make([]*MsgTx, 0, defaultTransactionAlloc), + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgcfheaders.go b/vendor/github.com/ltcsuite/ltcd/wire/msgcfheaders.go new file mode 100644 index 0000000000..f0fe143033 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgcfheaders.go @@ -0,0 +1,166 @@ +// Copyright (c) 2017 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +const ( + // MaxCFHeaderPayload is the maximum byte size of a committed + // filter header. + MaxCFHeaderPayload = chainhash.HashSize + + // MaxCFHeadersPerMsg is the maximum number of committed filter headers + // that can be in a single bitcoin cfheaders message. + MaxCFHeadersPerMsg = 2000 +) + +// MsgCFHeaders implements the Message interface and represents a bitcoin +// cfheaders message. It is used to deliver committed filter header information +// in response to a getcfheaders message (MsgGetCFHeaders). The maximum number +// of committed filter headers per message is currently 2000. See +// MsgGetCFHeaders for details on requesting the headers. +type MsgCFHeaders struct { + StopHash chainhash.Hash + Extended bool + HeaderHashes []*chainhash.Hash +} + +// AddCFHeader adds a new committed filter header to the message. +func (msg *MsgCFHeaders) AddCFHeader(headerHash *chainhash.Hash) error { + if len(msg.HeaderHashes)+1 > MaxCFHeadersPerMsg { + str := fmt.Sprintf("too many block headers in message [max %v]", + MaxBlockHeadersPerMsg) + return messageError("MsgCFHeaders.AddCFHeader", str) + } + + msg.HeaderHashes = append(msg.HeaderHashes, headerHash) + return nil +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgCFHeaders) BtcDecode(r io.Reader, pver uint32, _ MessageEncoding) error { + // Read stop hash + err := readElement(r, &msg.StopHash) + if err != nil { + return err + } + + // Read extended flag + err = readElement(r, &msg.Extended) + if err != nil { + return err + } + + // Read number of filter headers + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + + // Limit to max committed filter headers per message. + if count > MaxCFHeadersPerMsg { + str := fmt.Sprintf("too many committed filter headers for "+ + "message [count %v, max %v]", count, + MaxBlockHeadersPerMsg) + return messageError("MsgCFHeaders.BtcDecode", str) + } + + // Create a contiguous slice of headers to deserialize into in order to + // reduce the number of allocations. + msg.HeaderHashes = make([]*chainhash.Hash, 0, count) + for i := uint64(0); i < count; i++ { + var cfh chainhash.Hash + err := readElement(r, &cfh) + if err != nil { + return err + } + msg.AddCFHeader(&cfh) + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgCFHeaders) BtcEncode(w io.Writer, pver uint32, _ MessageEncoding) error { + // Write stop hash + err := writeElement(w, msg.StopHash) + if err != nil { + return err + } + + // Write extended flag + err = writeElement(w, msg.Extended) + if err != nil { + return err + } + + // Limit to max committed headers per message. + count := len(msg.HeaderHashes) + if count > MaxCFHeadersPerMsg { + str := fmt.Sprintf("too many committed filter headers for "+ + "message [count %v, max %v]", count, + MaxBlockHeadersPerMsg) + return messageError("MsgCFHeaders.BtcEncode", str) + } + + err = WriteVarInt(w, pver, uint64(count)) + if err != nil { + return err + } + + for _, cfh := range msg.HeaderHashes { + err := writeElement(w, cfh) + if err != nil { + return err + } + } + + return nil +} + +// Deserialize decodes a filter header from r into the receiver using a format +// that is suitable for long-term storage such as a database. This function +// differs from BtcDecode in that BtcDecode decodes from the bitcoin wire +// protocol as it was sent across the network. The wire encoding can +// technically differ depending on the protocol version and doesn't even really +// need to match the format of a stored filter header at all. As of the time +// this comment was written, the encoded filter header is the same in both +// instances, but there is a distinct difference and separating the two allows +// the API to be flexible enough to deal with changes. +func (msg *MsgCFHeaders) Deserialize(r io.Reader) error { + // At the current time, there is no difference between the wire encoding + // and the stable long-term storage format. As a result, make use of + // BtcDecode. + return msg.BtcDecode(r, 0, BaseEncoding) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgCFHeaders) Command() string { + return CmdCFHeaders +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgCFHeaders) MaxPayloadLength(pver uint32) uint32 { + // Hash size + num headers (varInt) + (header size * max headers). + return chainhash.HashSize + 1 + MaxVarIntPayload + + (MaxCFHeaderPayload * MaxCFHeadersPerMsg) +} + +// NewMsgCFHeaders returns a new bitcoin cfheaders message that conforms to +// the Message interface. See MsgCFHeaders for details. +func NewMsgCFHeaders() *MsgCFHeaders { + return &MsgCFHeaders{ + HeaderHashes: make([]*chainhash.Hash, 0, MaxCFHeadersPerMsg), + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgcfilter.go b/vendor/github.com/ltcsuite/ltcd/wire/msgcfilter.go new file mode 100644 index 0000000000..a6eb758d6e --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgcfilter.go @@ -0,0 +1,106 @@ +// Copyright (c) 2017 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +const ( + // MaxCFilterDataSize is the maximum byte size of a committed filter. + MaxCFilterDataSize = 262144 +) + +type MsgCFilter struct { + BlockHash chainhash.Hash + Extended bool + Data []byte +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgCFilter) BtcDecode(r io.Reader, pver uint32, _ MessageEncoding) error { + var err error + // Read the hash of the filter's block + err = readElement(r, &msg.BlockHash) + if err != nil { + return err + } + // Read extended flag + err = readElement(r, &msg.Extended) + if err != nil { + return err + } + // Read filter data + msg.Data, err = ReadVarBytes(r, pver, MaxCFilterDataSize, + "cfilter data") + return err +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgCFilter) BtcEncode(w io.Writer, pver uint32, _ MessageEncoding) error { + size := len(msg.Data) + if size > MaxCFilterDataSize { + str := fmt.Sprintf("cfilter size too large for message "+ + "[size %v, max %v]", size, MaxCFilterDataSize) + return messageError("MsgCFilter.BtcEncode", str) + } + + err := writeElement(w, msg.BlockHash) + if err != nil { + return err + } + + err = writeElement(w, msg.Extended) + if err != nil { + return err + } + + return WriteVarBytes(w, pver, msg.Data) +} + +// Deserialize decodes a filter from r into the receiver using a format that is +// suitable for long-term storage such as a database. This function differs +// from BtcDecode in that BtcDecode decodes from the bitcoin wire protocol as +// it was sent across the network. The wire encoding can technically differ +// depending on the protocol version and doesn't even really need to match the +// format of a stored filter at all. As of the time this comment was written, +// the encoded filter is the same in both instances, but there is a distinct +// difference and separating the two allows the API to be flexible enough to +// deal with changes. +func (msg *MsgCFilter) Deserialize(r io.Reader) error { + // At the current time, there is no difference between the wire encoding + // and the stable long-term storage format. As a result, make use of + // BtcDecode. + return msg.BtcDecode(r, 0, BaseEncoding) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgCFilter) Command() string { + return CmdCFilter +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgCFilter) MaxPayloadLength(pver uint32) uint32 { + return uint32(VarIntSerializeSize(MaxCFilterDataSize)) + + MaxCFilterDataSize + chainhash.HashSize + 1 +} + +// NewMsgCFilter returns a new bitcoin cfilter message that conforms to the +// Message interface. See MsgCFilter for details. +func NewMsgCFilter(blockHash *chainhash.Hash, extended bool, + data []byte) *MsgCFilter { + return &MsgCFilter{ + BlockHash: *blockHash, + Extended: extended, + Data: data, + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgfeefilter.go b/vendor/github.com/ltcsuite/ltcd/wire/msgfeefilter.go new file mode 100644 index 0000000000..754647a275 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgfeefilter.go @@ -0,0 +1,64 @@ +// Copyright (c) 2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +// MsgFeeFilter implements the Message interface and represents a bitcoin +// feefilter message. It is used to request the receiving peer does not +// announce any transactions below the specified minimum fee rate. +// +// This message was not added until protocol versions starting with +// FeeFilterVersion. +type MsgFeeFilter struct { + MinFee int64 +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgFeeFilter) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + if pver < FeeFilterVersion { + str := fmt.Sprintf("feefilter message invalid for protocol "+ + "version %d", pver) + return messageError("MsgFeeFilter.BtcDecode", str) + } + + return readElement(r, &msg.MinFee) +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgFeeFilter) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + if pver < FeeFilterVersion { + str := fmt.Sprintf("feefilter message invalid for protocol "+ + "version %d", pver) + return messageError("MsgFeeFilter.BtcEncode", str) + } + + return writeElement(w, msg.MinFee) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgFeeFilter) Command() string { + return CmdFeeFilter +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgFeeFilter) MaxPayloadLength(pver uint32) uint32 { + return 8 +} + +// NewMsgFeeFilter returns a new bitcoin feefilter message that conforms to +// the Message interface. See MsgFeeFilter for details. +func NewMsgFeeFilter(minfee int64) *MsgFeeFilter { + return &MsgFeeFilter{ + MinFee: minfee, + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgfilteradd.go b/vendor/github.com/ltcsuite/ltcd/wire/msgfilteradd.go new file mode 100644 index 0000000000..21b67cabb7 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgfilteradd.go @@ -0,0 +1,81 @@ +// Copyright (c) 2014-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +const ( + // MaxFilterAddDataSize is the maximum byte size of a data + // element to add to the Bloom filter. It is equal to the + // maximum element size of a script. + MaxFilterAddDataSize = 520 +) + +// MsgFilterAdd implements the Message interface and represents a bitcoin +// filteradd message. It is used to add a data element to an existing Bloom +// filter. +// +// This message was not added until protocol version BIP0037Version. +type MsgFilterAdd struct { + Data []byte +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgFilterAdd) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + if pver < BIP0037Version { + str := fmt.Sprintf("filteradd message invalid for protocol "+ + "version %d", pver) + return messageError("MsgFilterAdd.BtcDecode", str) + } + + var err error + msg.Data, err = ReadVarBytes(r, pver, MaxFilterAddDataSize, + "filteradd data") + return err +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgFilterAdd) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + if pver < BIP0037Version { + str := fmt.Sprintf("filteradd message invalid for protocol "+ + "version %d", pver) + return messageError("MsgFilterAdd.BtcEncode", str) + } + + size := len(msg.Data) + if size > MaxFilterAddDataSize { + str := fmt.Sprintf("filteradd size too large for message "+ + "[size %v, max %v]", size, MaxFilterAddDataSize) + return messageError("MsgFilterAdd.BtcEncode", str) + } + + return WriteVarBytes(w, pver, msg.Data) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgFilterAdd) Command() string { + return CmdFilterAdd +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgFilterAdd) MaxPayloadLength(pver uint32) uint32 { + return uint32(VarIntSerializeSize(MaxFilterAddDataSize)) + + MaxFilterAddDataSize +} + +// NewMsgFilterAdd returns a new bitcoin filteradd message that conforms to the +// Message interface. See MsgFilterAdd for details. +func NewMsgFilterAdd(data []byte) *MsgFilterAdd { + return &MsgFilterAdd{ + Data: data, + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgfilterclear.go b/vendor/github.com/ltcsuite/ltcd/wire/msgfilterclear.go new file mode 100644 index 0000000000..7562fe2c0c --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgfilterclear.go @@ -0,0 +1,59 @@ +// Copyright (c) 2014-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +// MsgFilterClear implements the Message interface and represents a bitcoin +// filterclear message which is used to reset a Bloom filter. +// +// This message was not added until protocol version BIP0037Version and has +// no payload. +type MsgFilterClear struct{} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgFilterClear) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + if pver < BIP0037Version { + str := fmt.Sprintf("filterclear message invalid for protocol "+ + "version %d", pver) + return messageError("MsgFilterClear.BtcDecode", str) + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgFilterClear) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + if pver < BIP0037Version { + str := fmt.Sprintf("filterclear message invalid for protocol "+ + "version %d", pver) + return messageError("MsgFilterClear.BtcEncode", str) + } + + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgFilterClear) Command() string { + return CmdFilterClear +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgFilterClear) MaxPayloadLength(pver uint32) uint32 { + return 0 +} + +// NewMsgFilterClear returns a new bitcoin filterclear message that conforms to the Message +// interface. See MsgFilterClear for details. +func NewMsgFilterClear() *MsgFilterClear { + return &MsgFilterClear{} +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgfilterload.go b/vendor/github.com/ltcsuite/ltcd/wire/msgfilterload.go new file mode 100644 index 0000000000..dbda737622 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgfilterload.go @@ -0,0 +1,136 @@ +// Copyright (c) 2014-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +// BloomUpdateType specifies how the filter is updated when a match is found +type BloomUpdateType uint8 + +const ( + // BloomUpdateNone indicates the filter is not adjusted when a match is + // found. + BloomUpdateNone BloomUpdateType = 0 + + // BloomUpdateAll indicates if the filter matches any data element in a + // public key script, the outpoint is serialized and inserted into the + // filter. + BloomUpdateAll BloomUpdateType = 1 + + // BloomUpdateP2PubkeyOnly indicates if the filter matches a data + // element in a public key script and the script is of the standard + // pay-to-pubkey or multisig, the outpoint is serialized and inserted + // into the filter. + BloomUpdateP2PubkeyOnly BloomUpdateType = 2 +) + +const ( + // MaxFilterLoadHashFuncs is the maximum number of hash functions to + // load into the Bloom filter. + MaxFilterLoadHashFuncs = 50 + + // MaxFilterLoadFilterSize is the maximum size in bytes a filter may be. + MaxFilterLoadFilterSize = 36000 +) + +// MsgFilterLoad implements the Message interface and represents a bitcoin +// filterload message which is used to reset a Bloom filter. +// +// This message was not added until protocol version BIP0037Version. +type MsgFilterLoad struct { + Filter []byte + HashFuncs uint32 + Tweak uint32 + Flags BloomUpdateType +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgFilterLoad) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + if pver < BIP0037Version { + str := fmt.Sprintf("filterload message invalid for protocol "+ + "version %d", pver) + return messageError("MsgFilterLoad.BtcDecode", str) + } + + var err error + msg.Filter, err = ReadVarBytes(r, pver, MaxFilterLoadFilterSize, + "filterload filter size") + if err != nil { + return err + } + + err = readElements(r, &msg.HashFuncs, &msg.Tweak, &msg.Flags) + if err != nil { + return err + } + + if msg.HashFuncs > MaxFilterLoadHashFuncs { + str := fmt.Sprintf("too many filter hash functions for message "+ + "[count %v, max %v]", msg.HashFuncs, MaxFilterLoadHashFuncs) + return messageError("MsgFilterLoad.BtcDecode", str) + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgFilterLoad) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + if pver < BIP0037Version { + str := fmt.Sprintf("filterload message invalid for protocol "+ + "version %d", pver) + return messageError("MsgFilterLoad.BtcEncode", str) + } + + size := len(msg.Filter) + if size > MaxFilterLoadFilterSize { + str := fmt.Sprintf("filterload filter size too large for message "+ + "[size %v, max %v]", size, MaxFilterLoadFilterSize) + return messageError("MsgFilterLoad.BtcEncode", str) + } + + if msg.HashFuncs > MaxFilterLoadHashFuncs { + str := fmt.Sprintf("too many filter hash functions for message "+ + "[count %v, max %v]", msg.HashFuncs, MaxFilterLoadHashFuncs) + return messageError("MsgFilterLoad.BtcEncode", str) + } + + err := WriteVarBytes(w, pver, msg.Filter) + if err != nil { + return err + } + + return writeElements(w, msg.HashFuncs, msg.Tweak, msg.Flags) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgFilterLoad) Command() string { + return CmdFilterLoad +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgFilterLoad) MaxPayloadLength(pver uint32) uint32 { + // Num filter bytes (varInt) + filter + 4 bytes hash funcs + + // 4 bytes tweak + 1 byte flags. + return uint32(VarIntSerializeSize(MaxFilterLoadFilterSize)) + + MaxFilterLoadFilterSize + 9 +} + +// NewMsgFilterLoad returns a new bitcoin filterload message that conforms to +// the Message interface. See MsgFilterLoad for details. +func NewMsgFilterLoad(filter []byte, hashFuncs uint32, tweak uint32, flags BloomUpdateType) *MsgFilterLoad { + return &MsgFilterLoad{ + Filter: filter, + HashFuncs: hashFuncs, + Tweak: tweak, + Flags: flags, + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msggetaddr.go b/vendor/github.com/ltcsuite/ltcd/wire/msggetaddr.go new file mode 100644 index 0000000000..2af8018f8b --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msggetaddr.go @@ -0,0 +1,47 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "io" +) + +// MsgGetAddr implements the Message interface and represents a bitcoin +// getaddr message. It is used to request a list of known active peers on the +// network from a peer to help identify potential nodes. The list is returned +// via one or more addr messages (MsgAddr). +// +// This message has no payload. +type MsgGetAddr struct{} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgGetAddr) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgGetAddr) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgGetAddr) Command() string { + return CmdGetAddr +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgGetAddr) MaxPayloadLength(pver uint32) uint32 { + return 0 +} + +// NewMsgGetAddr returns a new bitcoin getaddr message that conforms to the +// Message interface. See MsgGetAddr for details. +func NewMsgGetAddr() *MsgGetAddr { + return &MsgGetAddr{} +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msggetblocks.go b/vendor/github.com/ltcsuite/ltcd/wire/msggetblocks.go new file mode 100644 index 0000000000..900c7307a8 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msggetblocks.go @@ -0,0 +1,139 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +// MaxBlockLocatorsPerMsg is the maximum number of block locator hashes allowed +// per message. +const MaxBlockLocatorsPerMsg = 500 + +// MsgGetBlocks implements the Message interface and represents a bitcoin +// getblocks message. It is used to request a list of blocks starting after the +// last known hash in the slice of block locator hashes. The list is returned +// via an inv message (MsgInv) and is limited by a specific hash to stop at or +// the maximum number of blocks per message, which is currently 500. +// +// Set the HashStop field to the hash at which to stop and use +// AddBlockLocatorHash to build up the list of block locator hashes. +// +// The algorithm for building the block locator hashes should be to add the +// hashes in reverse order until you reach the genesis block. In order to keep +// the list of locator hashes to a reasonable number of entries, first add the +// most recent 10 block hashes, then double the step each loop iteration to +// exponentially decrease the number of hashes the further away from head and +// closer to the genesis block you get. +type MsgGetBlocks struct { + ProtocolVersion uint32 + BlockLocatorHashes []*chainhash.Hash + HashStop chainhash.Hash +} + +// AddBlockLocatorHash adds a new block locator hash to the message. +func (msg *MsgGetBlocks) AddBlockLocatorHash(hash *chainhash.Hash) error { + if len(msg.BlockLocatorHashes)+1 > MaxBlockLocatorsPerMsg { + str := fmt.Sprintf("too many block locator hashes for message [max %v]", + MaxBlockLocatorsPerMsg) + return messageError("MsgGetBlocks.AddBlockLocatorHash", str) + } + + msg.BlockLocatorHashes = append(msg.BlockLocatorHashes, hash) + return nil +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgGetBlocks) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + err := readElement(r, &msg.ProtocolVersion) + if err != nil { + return err + } + + // Read num block locator hashes and limit to max. + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + if count > MaxBlockLocatorsPerMsg { + str := fmt.Sprintf("too many block locator hashes for message "+ + "[count %v, max %v]", count, MaxBlockLocatorsPerMsg) + return messageError("MsgGetBlocks.BtcDecode", str) + } + + // Create a contiguous slice of hashes to deserialize into in order to + // reduce the number of allocations. + locatorHashes := make([]chainhash.Hash, count) + msg.BlockLocatorHashes = make([]*chainhash.Hash, 0, count) + for i := uint64(0); i < count; i++ { + hash := &locatorHashes[i] + err := readElement(r, hash) + if err != nil { + return err + } + msg.AddBlockLocatorHash(hash) + } + + return readElement(r, &msg.HashStop) +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgGetBlocks) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + count := len(msg.BlockLocatorHashes) + if count > MaxBlockLocatorsPerMsg { + str := fmt.Sprintf("too many block locator hashes for message "+ + "[count %v, max %v]", count, MaxBlockLocatorsPerMsg) + return messageError("MsgGetBlocks.BtcEncode", str) + } + + err := writeElement(w, msg.ProtocolVersion) + if err != nil { + return err + } + + err = WriteVarInt(w, pver, uint64(count)) + if err != nil { + return err + } + + for _, hash := range msg.BlockLocatorHashes { + err = writeElement(w, hash) + if err != nil { + return err + } + } + + return writeElement(w, &msg.HashStop) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgGetBlocks) Command() string { + return CmdGetBlocks +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgGetBlocks) MaxPayloadLength(pver uint32) uint32 { + // Protocol version 4 bytes + num hashes (varInt) + max block locator + // hashes + hash stop. + return 4 + MaxVarIntPayload + (MaxBlockLocatorsPerMsg * chainhash.HashSize) + chainhash.HashSize +} + +// NewMsgGetBlocks returns a new bitcoin getblocks message that conforms to the +// Message interface using the passed parameters and defaults for the remaining +// fields. +func NewMsgGetBlocks(hashStop *chainhash.Hash) *MsgGetBlocks { + return &MsgGetBlocks{ + ProtocolVersion: ProtocolVersion, + BlockLocatorHashes: make([]*chainhash.Hash, 0, MaxBlockLocatorsPerMsg), + HashStop: *hashStop, + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msggetcfheaders.go b/vendor/github.com/ltcsuite/ltcd/wire/msggetcfheaders.go new file mode 100644 index 0000000000..a09a3f2944 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msggetcfheaders.go @@ -0,0 +1,135 @@ +// Copyright (c) 2017 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +// MsgGetCFHeaders is a message similar to MsgGetHeaders, but for committed +// filter headers. It allows to set the Extended field to get headers in the +// chain of basic (false) or extended (true) headers. +type MsgGetCFHeaders struct { + ProtocolVersion uint32 + BlockLocatorHashes []*chainhash.Hash + HashStop chainhash.Hash + Extended bool +} + +// AddBlockLocatorHash adds a new block locator hash to the message. +func (msg *MsgGetCFHeaders) AddBlockLocatorHash(hash *chainhash.Hash) error { + if len(msg.BlockLocatorHashes)+1 > MaxBlockLocatorsPerMsg { + str := fmt.Sprintf("too many block locator hashes for message [max %v]", + MaxBlockLocatorsPerMsg) + return messageError("MsgGetCFHeaders.AddBlockLocatorHash", str) + } + + msg.BlockLocatorHashes = append(msg.BlockLocatorHashes, hash) + return nil +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgGetCFHeaders) BtcDecode(r io.Reader, pver uint32, _ MessageEncoding) error { + err := readElement(r, &msg.ProtocolVersion) + if err != nil { + return err + } + + // Read num block locator hashes and limit to max. + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + if count > MaxBlockLocatorsPerMsg { + str := fmt.Sprintf("too many block locator hashes for message "+ + "[count %v, max %v]", count, MaxBlockLocatorsPerMsg) + return messageError("MsgGetHeaders.BtcDecode", str) + } + + // Create a contiguous slice of hashes to deserialize into in order to + // reduce the number of allocations. + locatorHashes := make([]chainhash.Hash, count) + msg.BlockLocatorHashes = make([]*chainhash.Hash, 0, count) + for i := uint64(0); i < count; i++ { + hash := &locatorHashes[i] + err := readElement(r, hash) + if err != nil { + return err + } + msg.AddBlockLocatorHash(hash) + } + + err = readElement(r, &msg.HashStop) + if err != nil { + return err + } + + return readElement(r, &msg.Extended) +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgGetCFHeaders) BtcEncode(w io.Writer, pver uint32, _ MessageEncoding) error { + // Limit to max block locator hashes per message. + count := len(msg.BlockLocatorHashes) + if count > MaxBlockLocatorsPerMsg { + str := fmt.Sprintf("too many block locator hashes for message "+ + "[count %v, max %v]", count, MaxBlockLocatorsPerMsg) + return messageError("MsgGetHeaders.BtcEncode", str) + } + + err := writeElement(w, msg.ProtocolVersion) + if err != nil { + return err + } + + err = WriteVarInt(w, pver, uint64(count)) + if err != nil { + return err + } + + for _, hash := range msg.BlockLocatorHashes { + err := writeElement(w, hash) + if err != nil { + return err + } + } + + err = writeElement(w, &msg.HashStop) + if err != nil { + return err + } + + return writeElement(w, msg.Extended) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgGetCFHeaders) Command() string { + return CmdGetCFHeaders +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgGetCFHeaders) MaxPayloadLength(pver uint32) uint32 { + // Version 4 bytes + num block locator hashes (varInt) + max allowed + // block locators + hash stop + Extended flag 1 byte. + return 4 + MaxVarIntPayload + (MaxBlockLocatorsPerMsg * + chainhash.HashSize) + chainhash.HashSize + 1 +} + +// NewMsgGetCFHeaders returns a new bitcoin getcfheader message that conforms to +// the Message interface using the passed parameters and defaults for the +// remaining fields. +func NewMsgGetCFHeaders() *MsgGetCFHeaders { + return &MsgGetCFHeaders{ + BlockLocatorHashes: make([]*chainhash.Hash, 0, + MaxBlockLocatorsPerMsg), + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msggetcfilter.go b/vendor/github.com/ltcsuite/ltcd/wire/msggetcfilter.go new file mode 100644 index 0000000000..1986d35914 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msggetcfilter.go @@ -0,0 +1,57 @@ +// Copyright (c) 2017 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "io" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +type MsgGetCFilter struct { + BlockHash chainhash.Hash + Extended bool +} + +func (msg *MsgGetCFilter) BtcDecode(r io.Reader, pver uint32, _ MessageEncoding) error { + err := readElement(r, &msg.BlockHash) + if err != nil { + return err + } + return readElement(r, &msg.Extended) +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgGetCFilter) BtcEncode(w io.Writer, pver uint32, _ MessageEncoding) error { + err := writeElement(w, &msg.BlockHash) + if err != nil { + return err + } + return writeElement(w, msg.Extended) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgGetCFilter) Command() string { + return CmdGetCFilter +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgGetCFilter) MaxPayloadLength(pver uint32) uint32 { + // Block hash + Extended flag. + return chainhash.HashSize + 1 +} + +// NewMsgGetCFilter returns a new bitcoin getcfilter message that conforms to +// the Message interface using the passed parameters and defaults for the +// remaining fields. +func NewMsgGetCFilter(blockHash *chainhash.Hash, extended bool) *MsgGetCFilter { + return &MsgGetCFilter{ + BlockHash: *blockHash, + Extended: extended, + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msggetdata.go b/vendor/github.com/ltcsuite/ltcd/wire/msggetdata.go new file mode 100644 index 0000000000..5837fac5ba --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msggetdata.go @@ -0,0 +1,133 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +// MsgGetData implements the Message interface and represents a bitcoin +// getdata message. It is used to request data such as blocks and transactions +// from another peer. It should be used in response to the inv (MsgInv) message +// to request the actual data referenced by each inventory vector the receiving +// peer doesn't already have. Each message is limited to a maximum number of +// inventory vectors, which is currently 50,000. As a result, multiple messages +// must be used to request larger amounts of data. +// +// Use the AddInvVect function to build up the list of inventory vectors when +// sending a getdata message to another peer. +type MsgGetData struct { + InvList []*InvVect +} + +// AddInvVect adds an inventory vector to the message. +func (msg *MsgGetData) AddInvVect(iv *InvVect) error { + if len(msg.InvList)+1 > MaxInvPerMsg { + str := fmt.Sprintf("too many invvect in message [max %v]", + MaxInvPerMsg) + return messageError("MsgGetData.AddInvVect", str) + } + + msg.InvList = append(msg.InvList, iv) + return nil +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgGetData) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + + // Limit to max inventory vectors per message. + if count > MaxInvPerMsg { + str := fmt.Sprintf("too many invvect in message [%v]", count) + return messageError("MsgGetData.BtcDecode", str) + } + + // Create a contiguous slice of inventory vectors to deserialize into in + // order to reduce the number of allocations. + invList := make([]InvVect, count) + msg.InvList = make([]*InvVect, 0, count) + for i := uint64(0); i < count; i++ { + iv := &invList[i] + err := readInvVect(r, pver, iv) + if err != nil { + return err + } + msg.AddInvVect(iv) + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgGetData) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + // Limit to max inventory vectors per message. + count := len(msg.InvList) + if count > MaxInvPerMsg { + str := fmt.Sprintf("too many invvect in message [%v]", count) + return messageError("MsgGetData.BtcEncode", str) + } + + err := WriteVarInt(w, pver, uint64(count)) + if err != nil { + return err + } + + for _, iv := range msg.InvList { + err := writeInvVect(w, pver, iv) + if err != nil { + return err + } + } + + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgGetData) Command() string { + return CmdGetData +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgGetData) MaxPayloadLength(pver uint32) uint32 { + // Num inventory vectors (varInt) + max allowed inventory vectors. + return MaxVarIntPayload + (MaxInvPerMsg * maxInvVectPayload) +} + +// NewMsgGetData returns a new bitcoin getdata message that conforms to the +// Message interface. See MsgGetData for details. +func NewMsgGetData() *MsgGetData { + return &MsgGetData{ + InvList: make([]*InvVect, 0, defaultInvListAlloc), + } +} + +// NewMsgGetDataSizeHint returns a new bitcoin getdata message that conforms to +// the Message interface. See MsgGetData for details. This function differs +// from NewMsgGetData in that it allows a default allocation size for the +// backing array which houses the inventory vector list. This allows callers +// who know in advance how large the inventory list will grow to avoid the +// overhead of growing the internal backing array several times when appending +// large amounts of inventory vectors with AddInvVect. Note that the specified +// hint is just that - a hint that is used for the default allocation size. +// Adding more (or less) inventory vectors will still work properly. The size +// hint is limited to MaxInvPerMsg. +func NewMsgGetDataSizeHint(sizeHint uint) *MsgGetData { + // Limit the specified hint to the maximum allow per message. + if sizeHint > MaxInvPerMsg { + sizeHint = MaxInvPerMsg + } + + return &MsgGetData{ + InvList: make([]*InvVect, 0, sizeHint), + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msggetheaders.go b/vendor/github.com/ltcsuite/ltcd/wire/msggetheaders.go new file mode 100644 index 0000000000..67ce621d6d --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msggetheaders.go @@ -0,0 +1,136 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +// MsgGetHeaders implements the Message interface and represents a bitcoin +// getheaders message. It is used to request a list of block headers for +// blocks starting after the last known hash in the slice of block locator +// hashes. The list is returned via a headers message (MsgHeaders) and is +// limited by a specific hash to stop at or the maximum number of block headers +// per message, which is currently 2000. +// +// Set the HashStop field to the hash at which to stop and use +// AddBlockLocatorHash to build up the list of block locator hashes. +// +// The algorithm for building the block locator hashes should be to add the +// hashes in reverse order until you reach the genesis block. In order to keep +// the list of locator hashes to a resonable number of entries, first add the +// most recent 10 block hashes, then double the step each loop iteration to +// exponentially decrease the number of hashes the further away from head and +// closer to the genesis block you get. +type MsgGetHeaders struct { + ProtocolVersion uint32 + BlockLocatorHashes []*chainhash.Hash + HashStop chainhash.Hash +} + +// AddBlockLocatorHash adds a new block locator hash to the message. +func (msg *MsgGetHeaders) AddBlockLocatorHash(hash *chainhash.Hash) error { + if len(msg.BlockLocatorHashes)+1 > MaxBlockLocatorsPerMsg { + str := fmt.Sprintf("too many block locator hashes for message [max %v]", + MaxBlockLocatorsPerMsg) + return messageError("MsgGetHeaders.AddBlockLocatorHash", str) + } + + msg.BlockLocatorHashes = append(msg.BlockLocatorHashes, hash) + return nil +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgGetHeaders) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + err := readElement(r, &msg.ProtocolVersion) + if err != nil { + return err + } + + // Read num block locator hashes and limit to max. + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + if count > MaxBlockLocatorsPerMsg { + str := fmt.Sprintf("too many block locator hashes for message "+ + "[count %v, max %v]", count, MaxBlockLocatorsPerMsg) + return messageError("MsgGetHeaders.BtcDecode", str) + } + + // Create a contiguous slice of hashes to deserialize into in order to + // reduce the number of allocations. + locatorHashes := make([]chainhash.Hash, count) + msg.BlockLocatorHashes = make([]*chainhash.Hash, 0, count) + for i := uint64(0); i < count; i++ { + hash := &locatorHashes[i] + err := readElement(r, hash) + if err != nil { + return err + } + msg.AddBlockLocatorHash(hash) + } + + return readElement(r, &msg.HashStop) +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgGetHeaders) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + // Limit to max block locator hashes per message. + count := len(msg.BlockLocatorHashes) + if count > MaxBlockLocatorsPerMsg { + str := fmt.Sprintf("too many block locator hashes for message "+ + "[count %v, max %v]", count, MaxBlockLocatorsPerMsg) + return messageError("MsgGetHeaders.BtcEncode", str) + } + + err := writeElement(w, msg.ProtocolVersion) + if err != nil { + return err + } + + err = WriteVarInt(w, pver, uint64(count)) + if err != nil { + return err + } + + for _, hash := range msg.BlockLocatorHashes { + err := writeElement(w, hash) + if err != nil { + return err + } + } + + return writeElement(w, &msg.HashStop) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgGetHeaders) Command() string { + return CmdGetHeaders +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgGetHeaders) MaxPayloadLength(pver uint32) uint32 { + // Version 4 bytes + num block locator hashes (varInt) + max allowed block + // locators + hash stop. + return 4 + MaxVarIntPayload + (MaxBlockLocatorsPerMsg * + chainhash.HashSize) + chainhash.HashSize +} + +// NewMsgGetHeaders returns a new bitcoin getheaders message that conforms to +// the Message interface. See MsgGetHeaders for details. +func NewMsgGetHeaders() *MsgGetHeaders { + return &MsgGetHeaders{ + BlockLocatorHashes: make([]*chainhash.Hash, 0, + MaxBlockLocatorsPerMsg), + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgheaders.go b/vendor/github.com/ltcsuite/ltcd/wire/msgheaders.go new file mode 100644 index 0000000000..7d18d930e0 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgheaders.go @@ -0,0 +1,136 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +// MaxBlockHeadersPerMsg is the maximum number of block headers that can be in +// a single bitcoin headers message. +const MaxBlockHeadersPerMsg = 2000 + +// MsgHeaders implements the Message interface and represents a bitcoin headers +// message. It is used to deliver block header information in response +// to a getheaders message (MsgGetHeaders). The maximum number of block headers +// per message is currently 2000. See MsgGetHeaders for details on requesting +// the headers. +type MsgHeaders struct { + Headers []*BlockHeader +} + +// AddBlockHeader adds a new block header to the message. +func (msg *MsgHeaders) AddBlockHeader(bh *BlockHeader) error { + if len(msg.Headers)+1 > MaxBlockHeadersPerMsg { + str := fmt.Sprintf("too many block headers in message [max %v]", + MaxBlockHeadersPerMsg) + return messageError("MsgHeaders.AddBlockHeader", str) + } + + msg.Headers = append(msg.Headers, bh) + return nil +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgHeaders) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + + // Limit to max block headers per message. + if count > MaxBlockHeadersPerMsg { + str := fmt.Sprintf("too many block headers for message "+ + "[count %v, max %v]", count, MaxBlockHeadersPerMsg) + return messageError("MsgHeaders.BtcDecode", str) + } + + // Create a contiguous slice of headers to deserialize into in order to + // reduce the number of allocations. + headers := make([]BlockHeader, count) + msg.Headers = make([]*BlockHeader, 0, count) + for i := uint64(0); i < count; i++ { + bh := &headers[i] + err := readBlockHeader(r, pver, bh) + if err != nil { + return err + } + + txCount, err := ReadVarInt(r, pver) + if err != nil { + return err + } + + // Ensure the transaction count is zero for headers. + if txCount > 0 { + str := fmt.Sprintf("block headers may not contain "+ + "transactions [count %v]", txCount) + return messageError("MsgHeaders.BtcDecode", str) + } + msg.AddBlockHeader(bh) + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgHeaders) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + // Limit to max block headers per message. + count := len(msg.Headers) + if count > MaxBlockHeadersPerMsg { + str := fmt.Sprintf("too many block headers for message "+ + "[count %v, max %v]", count, MaxBlockHeadersPerMsg) + return messageError("MsgHeaders.BtcEncode", str) + } + + err := WriteVarInt(w, pver, uint64(count)) + if err != nil { + return err + } + + for _, bh := range msg.Headers { + err := writeBlockHeader(w, pver, bh) + if err != nil { + return err + } + + // The wire protocol encoding always includes a 0 for the number + // of transactions on header messages. This is really just an + // artifact of the way the original implementation serializes + // block headers, but it is required. + err = WriteVarInt(w, pver, 0) + if err != nil { + return err + } + } + + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgHeaders) Command() string { + return CmdHeaders +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgHeaders) MaxPayloadLength(pver uint32) uint32 { + // Num headers (varInt) + max allowed headers (header length + 1 byte + // for the number of transactions which is always 0). + return MaxVarIntPayload + ((MaxBlockHeaderPayload + 1) * + MaxBlockHeadersPerMsg) +} + +// NewMsgHeaders returns a new bitcoin headers message that conforms to the +// Message interface. See MsgHeaders for details. +func NewMsgHeaders() *MsgHeaders { + return &MsgHeaders{ + Headers: make([]*BlockHeader, 0, MaxBlockHeadersPerMsg), + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msginv.go b/vendor/github.com/ltcsuite/ltcd/wire/msginv.go new file mode 100644 index 0000000000..5377b179c3 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msginv.go @@ -0,0 +1,141 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +// defaultInvListAlloc is the default size used for the backing array for an +// inventory list. The array will dynamically grow as needed, but this +// figure is intended to provide enough space for the max number of inventory +// vectors in a *typical* inventory message without needing to grow the backing +// array multiple times. Technically, the list can grow to MaxInvPerMsg, but +// rather than using that large figure, this figure more accurately reflects the +// typical case. +const defaultInvListAlloc = 1000 + +// MsgInv implements the Message interface and represents a bitcoin inv message. +// It is used to advertise a peer's known data such as blocks and transactions +// through inventory vectors. It may be sent unsolicited to inform other peers +// of the data or in response to a getblocks message (MsgGetBlocks). Each +// message is limited to a maximum number of inventory vectors, which is +// currently 50,000. +// +// Use the AddInvVect function to build up the list of inventory vectors when +// sending an inv message to another peer. +type MsgInv struct { + InvList []*InvVect +} + +// AddInvVect adds an inventory vector to the message. +func (msg *MsgInv) AddInvVect(iv *InvVect) error { + if len(msg.InvList)+1 > MaxInvPerMsg { + str := fmt.Sprintf("too many invvect in message [max %v]", + MaxInvPerMsg) + return messageError("MsgInv.AddInvVect", str) + } + + msg.InvList = append(msg.InvList, iv) + return nil +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgInv) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + + // Limit to max inventory vectors per message. + if count > MaxInvPerMsg { + str := fmt.Sprintf("too many invvect in message [%v]", count) + return messageError("MsgInv.BtcDecode", str) + } + + // Create a contiguous slice of inventory vectors to deserialize into in + // order to reduce the number of allocations. + invList := make([]InvVect, count) + msg.InvList = make([]*InvVect, 0, count) + for i := uint64(0); i < count; i++ { + iv := &invList[i] + err := readInvVect(r, pver, iv) + if err != nil { + return err + } + msg.AddInvVect(iv) + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgInv) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + // Limit to max inventory vectors per message. + count := len(msg.InvList) + if count > MaxInvPerMsg { + str := fmt.Sprintf("too many invvect in message [%v]", count) + return messageError("MsgInv.BtcEncode", str) + } + + err := WriteVarInt(w, pver, uint64(count)) + if err != nil { + return err + } + + for _, iv := range msg.InvList { + err := writeInvVect(w, pver, iv) + if err != nil { + return err + } + } + + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgInv) Command() string { + return CmdInv +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgInv) MaxPayloadLength(pver uint32) uint32 { + // Num inventory vectors (varInt) + max allowed inventory vectors. + return MaxVarIntPayload + (MaxInvPerMsg * maxInvVectPayload) +} + +// NewMsgInv returns a new bitcoin inv message that conforms to the Message +// interface. See MsgInv for details. +func NewMsgInv() *MsgInv { + return &MsgInv{ + InvList: make([]*InvVect, 0, defaultInvListAlloc), + } +} + +// NewMsgInvSizeHint returns a new bitcoin inv message that conforms to the +// Message interface. See MsgInv for details. This function differs from +// NewMsgInv in that it allows a default allocation size for the backing array +// which houses the inventory vector list. This allows callers who know in +// advance how large the inventory list will grow to avoid the overhead of +// growing the internal backing array several times when appending large amounts +// of inventory vectors with AddInvVect. Note that the specified hint is just +// that - a hint that is used for the default allocation size. Adding more +// (or less) inventory vectors will still work properly. The size hint is +// limited to MaxInvPerMsg. +func NewMsgInvSizeHint(sizeHint uint) *MsgInv { + // Limit the specified hint to the maximum allow per message. + if sizeHint > MaxInvPerMsg { + sizeHint = MaxInvPerMsg + } + + return &MsgInv{ + InvList: make([]*InvVect, 0, sizeHint), + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgmempool.go b/vendor/github.com/ltcsuite/ltcd/wire/msgmempool.go new file mode 100644 index 0000000000..25760c52eb --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgmempool.go @@ -0,0 +1,60 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +// MsgMemPool implements the Message interface and represents a bitcoin mempool +// message. It is used to request a list of transactions still in the active +// memory pool of a relay. +// +// This message has no payload and was not added until protocol versions +// starting with BIP0035Version. +type MsgMemPool struct{} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgMemPool) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + if pver < BIP0035Version { + str := fmt.Sprintf("mempool message invalid for protocol "+ + "version %d", pver) + return messageError("MsgMemPool.BtcDecode", str) + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgMemPool) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + if pver < BIP0035Version { + str := fmt.Sprintf("mempool message invalid for protocol "+ + "version %d", pver) + return messageError("MsgMemPool.BtcEncode", str) + } + + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgMemPool) Command() string { + return CmdMemPool +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgMemPool) MaxPayloadLength(pver uint32) uint32 { + return 0 +} + +// NewMsgMemPool returns a new bitcoin pong message that conforms to the Message +// interface. See MsgPong for details. +func NewMsgMemPool() *MsgMemPool { + return &MsgMemPool{} +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgmerkleblock.go b/vendor/github.com/ltcsuite/ltcd/wire/msgmerkleblock.go new file mode 100644 index 0000000000..b786f18562 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgmerkleblock.go @@ -0,0 +1,159 @@ +// Copyright (c) 2014-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +// maxFlagsPerMerkleBlock is the maximum number of flag bytes that could +// possibly fit into a merkle block. Since each transaction is represented by +// a single bit, this is the max number of transactions per block divided by +// 8 bits per byte. Then an extra one to cover partials. +const maxFlagsPerMerkleBlock = maxTxPerBlock / 8 + +// MsgMerkleBlock implements the Message interface and represents a bitcoin +// merkleblock message which is used to reset a Bloom filter. +// +// This message was not added until protocol version BIP0037Version. +type MsgMerkleBlock struct { + Header BlockHeader + Transactions uint32 + Hashes []*chainhash.Hash + Flags []byte +} + +// AddTxHash adds a new transaction hash to the message. +func (msg *MsgMerkleBlock) AddTxHash(hash *chainhash.Hash) error { + if len(msg.Hashes)+1 > maxTxPerBlock { + str := fmt.Sprintf("too many tx hashes for message [max %v]", + maxTxPerBlock) + return messageError("MsgMerkleBlock.AddTxHash", str) + } + + msg.Hashes = append(msg.Hashes, hash) + return nil +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgMerkleBlock) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + if pver < BIP0037Version { + str := fmt.Sprintf("merkleblock message invalid for protocol "+ + "version %d", pver) + return messageError("MsgMerkleBlock.BtcDecode", str) + } + + err := readBlockHeader(r, pver, &msg.Header) + if err != nil { + return err + } + + err = readElement(r, &msg.Transactions) + if err != nil { + return err + } + + // Read num block locator hashes and limit to max. + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + if count > maxTxPerBlock { + str := fmt.Sprintf("too many transaction hashes for message "+ + "[count %v, max %v]", count, maxTxPerBlock) + return messageError("MsgMerkleBlock.BtcDecode", str) + } + + // Create a contiguous slice of hashes to deserialize into in order to + // reduce the number of allocations. + hashes := make([]chainhash.Hash, count) + msg.Hashes = make([]*chainhash.Hash, 0, count) + for i := uint64(0); i < count; i++ { + hash := &hashes[i] + err := readElement(r, hash) + if err != nil { + return err + } + msg.AddTxHash(hash) + } + + msg.Flags, err = ReadVarBytes(r, pver, maxFlagsPerMerkleBlock, + "merkle block flags size") + return err +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgMerkleBlock) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + if pver < BIP0037Version { + str := fmt.Sprintf("merkleblock message invalid for protocol "+ + "version %d", pver) + return messageError("MsgMerkleBlock.BtcEncode", str) + } + + // Read num transaction hashes and limit to max. + numHashes := len(msg.Hashes) + if numHashes > maxTxPerBlock { + str := fmt.Sprintf("too many transaction hashes for message "+ + "[count %v, max %v]", numHashes, maxTxPerBlock) + return messageError("MsgMerkleBlock.BtcDecode", str) + } + numFlagBytes := len(msg.Flags) + if numFlagBytes > maxFlagsPerMerkleBlock { + str := fmt.Sprintf("too many flag bytes for message [count %v, "+ + "max %v]", numFlagBytes, maxFlagsPerMerkleBlock) + return messageError("MsgMerkleBlock.BtcDecode", str) + } + + err := writeBlockHeader(w, pver, &msg.Header) + if err != nil { + return err + } + + err = writeElement(w, msg.Transactions) + if err != nil { + return err + } + + err = WriteVarInt(w, pver, uint64(numHashes)) + if err != nil { + return err + } + for _, hash := range msg.Hashes { + err = writeElement(w, hash) + if err != nil { + return err + } + } + + return WriteVarBytes(w, pver, msg.Flags) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgMerkleBlock) Command() string { + return CmdMerkleBlock +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgMerkleBlock) MaxPayloadLength(pver uint32) uint32 { + return MaxBlockPayload +} + +// NewMsgMerkleBlock returns a new bitcoin merkleblock message that conforms to +// the Message interface. See MsgMerkleBlock for details. +func NewMsgMerkleBlock(bh *BlockHeader) *MsgMerkleBlock { + return &MsgMerkleBlock{ + Header: *bh, + Transactions: 0, + Hashes: make([]*chainhash.Hash, 0), + Flags: make([]byte, 0), + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgnotfound.go b/vendor/github.com/ltcsuite/ltcd/wire/msgnotfound.go new file mode 100644 index 0000000000..e867681668 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgnotfound.go @@ -0,0 +1,110 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +// MsgNotFound defines a bitcoin notfound message which is sent in response to +// a getdata message if any of the requested data in not available on the peer. +// Each message is limited to a maximum number of inventory vectors, which is +// currently 50,000. +// +// Use the AddInvVect function to build up the list of inventory vectors when +// sending a notfound message to another peer. +type MsgNotFound struct { + InvList []*InvVect +} + +// AddInvVect adds an inventory vector to the message. +func (msg *MsgNotFound) AddInvVect(iv *InvVect) error { + if len(msg.InvList)+1 > MaxInvPerMsg { + str := fmt.Sprintf("too many invvect in message [max %v]", + MaxInvPerMsg) + return messageError("MsgNotFound.AddInvVect", str) + } + + msg.InvList = append(msg.InvList, iv) + return nil +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgNotFound) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + + // Limit to max inventory vectors per message. + if count > MaxInvPerMsg { + str := fmt.Sprintf("too many invvect in message [%v]", count) + return messageError("MsgNotFound.BtcDecode", str) + } + + // Create a contiguous slice of inventory vectors to deserialize into in + // order to reduce the number of allocations. + invList := make([]InvVect, count) + msg.InvList = make([]*InvVect, 0, count) + for i := uint64(0); i < count; i++ { + iv := &invList[i] + err := readInvVect(r, pver, iv) + if err != nil { + return err + } + msg.AddInvVect(iv) + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgNotFound) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + // Limit to max inventory vectors per message. + count := len(msg.InvList) + if count > MaxInvPerMsg { + str := fmt.Sprintf("too many invvect in message [%v]", count) + return messageError("MsgNotFound.BtcEncode", str) + } + + err := WriteVarInt(w, pver, uint64(count)) + if err != nil { + return err + } + + for _, iv := range msg.InvList { + err := writeInvVect(w, pver, iv) + if err != nil { + return err + } + } + + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgNotFound) Command() string { + return CmdNotFound +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgNotFound) MaxPayloadLength(pver uint32) uint32 { + // Max var int 9 bytes + max InvVects at 36 bytes each. + // Num inventory vectors (varInt) + max allowed inventory vectors. + return MaxVarIntPayload + (MaxInvPerMsg * maxInvVectPayload) +} + +// NewMsgNotFound returns a new bitcoin notfound message that conforms to the +// Message interface. See MsgNotFound for details. +func NewMsgNotFound() *MsgNotFound { + return &MsgNotFound{ + InvList: make([]*InvVect, 0, defaultInvListAlloc), + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgping.go b/vendor/github.com/ltcsuite/ltcd/wire/msgping.go new file mode 100644 index 0000000000..b2f346e0e1 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgping.go @@ -0,0 +1,87 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "io" +) + +// MsgPing implements the Message interface and represents a bitcoin ping +// message. +// +// For versions BIP0031Version and earlier, it is used primarily to confirm +// that a connection is still valid. A transmission error is typically +// interpreted as a closed connection and that the peer should be removed. +// For versions AFTER BIP0031Version it contains an identifier which can be +// returned in the pong message to determine network timing. +// +// The payload for this message just consists of a nonce used for identifying +// it later. +type MsgPing struct { + // Unique value associated with message that is used to identify + // specific ping message. + Nonce uint64 +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgPing) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + // There was no nonce for BIP0031Version and earlier. + // NOTE: > is not a mistake here. The BIP0031 was defined as AFTER + // the version unlike most others. + if pver > BIP0031Version { + err := readElement(r, &msg.Nonce) + if err != nil { + return err + } + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgPing) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + // There was no nonce for BIP0031Version and earlier. + // NOTE: > is not a mistake here. The BIP0031 was defined as AFTER + // the version unlike most others. + if pver > BIP0031Version { + err := writeElement(w, msg.Nonce) + if err != nil { + return err + } + } + + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgPing) Command() string { + return CmdPing +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgPing) MaxPayloadLength(pver uint32) uint32 { + plen := uint32(0) + // There was no nonce for BIP0031Version and earlier. + // NOTE: > is not a mistake here. The BIP0031 was defined as AFTER + // the version unlike most others. + if pver > BIP0031Version { + // Nonce 8 bytes. + plen += 8 + } + + return plen +} + +// NewMsgPing returns a new bitcoin ping message that conforms to the Message +// interface. See MsgPing for details. +func NewMsgPing(nonce uint64) *MsgPing { + return &MsgPing{ + Nonce: nonce, + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgpong.go b/vendor/github.com/ltcsuite/ltcd/wire/msgpong.go new file mode 100644 index 0000000000..eec80d8d5b --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgpong.go @@ -0,0 +1,78 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +// MsgPong implements the Message interface and represents a bitcoin pong +// message which is used primarily to confirm that a connection is still valid +// in response to a bitcoin ping message (MsgPing). +// +// This message was not added until protocol versions AFTER BIP0031Version. +type MsgPong struct { + // Unique value associated with message that is used to identify + // specific ping message. + Nonce uint64 +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgPong) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + // NOTE: <= is not a mistake here. The BIP0031 was defined as AFTER + // the version unlike most others. + if pver <= BIP0031Version { + str := fmt.Sprintf("pong message invalid for protocol "+ + "version %d", pver) + return messageError("MsgPong.BtcDecode", str) + } + + return readElement(r, &msg.Nonce) +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgPong) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + // NOTE: <= is not a mistake here. The BIP0031 was defined as AFTER + // the version unlike most others. + if pver <= BIP0031Version { + str := fmt.Sprintf("pong message invalid for protocol "+ + "version %d", pver) + return messageError("MsgPong.BtcEncode", str) + } + + return writeElement(w, msg.Nonce) +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgPong) Command() string { + return CmdPong +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgPong) MaxPayloadLength(pver uint32) uint32 { + plen := uint32(0) + // The pong message did not exist for BIP0031Version and earlier. + // NOTE: > is not a mistake here. The BIP0031 was defined as AFTER + // the version unlike most others. + if pver > BIP0031Version { + // Nonce 8 bytes. + plen += 8 + } + + return plen +} + +// NewMsgPong returns a new bitcoin pong message that conforms to the Message +// interface. See MsgPong for details. +func NewMsgPong(nonce uint64) *MsgPong { + return &MsgPong{ + Nonce: nonce, + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgreject.go b/vendor/github.com/ltcsuite/ltcd/wire/msgreject.go new file mode 100644 index 0000000000..38a9bc21b6 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgreject.go @@ -0,0 +1,186 @@ +// Copyright (c) 2014-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +// RejectCode represents a numeric value by which a remote peer indicates +// why a message was rejected. +type RejectCode uint8 + +// These constants define the various supported reject codes. +const ( + RejectMalformed RejectCode = 0x01 + RejectInvalid RejectCode = 0x10 + RejectObsolete RejectCode = 0x11 + RejectDuplicate RejectCode = 0x12 + RejectNonstandard RejectCode = 0x40 + RejectDust RejectCode = 0x41 + RejectInsufficientFee RejectCode = 0x42 + RejectCheckpoint RejectCode = 0x43 +) + +// Map of reject codes back strings for pretty printing. +var rejectCodeStrings = map[RejectCode]string{ + RejectMalformed: "REJECT_MALFORMED", + RejectInvalid: "REJECT_INVALID", + RejectObsolete: "REJECT_OBSOLETE", + RejectDuplicate: "REJECT_DUPLICATE", + RejectNonstandard: "REJECT_NONSTANDARD", + RejectDust: "REJECT_DUST", + RejectInsufficientFee: "REJECT_INSUFFICIENTFEE", + RejectCheckpoint: "REJECT_CHECKPOINT", +} + +// String returns the RejectCode in human-readable form. +func (code RejectCode) String() string { + if s, ok := rejectCodeStrings[code]; ok { + return s + } + + return fmt.Sprintf("Unknown RejectCode (%d)", uint8(code)) +} + +// MsgReject implements the Message interface and represents a bitcoin reject +// message. +// +// This message was not added until protocol version RejectVersion. +type MsgReject struct { + // Cmd is the command for the message which was rejected such as + // as CmdBlock or CmdTx. This can be obtained from the Command function + // of a Message. + Cmd string + + // RejectCode is a code indicating why the command was rejected. It + // is encoded as a uint8 on the wire. + Code RejectCode + + // Reason is a human-readable string with specific details (over and + // above the reject code) about why the command was rejected. + Reason string + + // Hash identifies a specific block or transaction that was rejected + // and therefore only applies the MsgBlock and MsgTx messages. + Hash chainhash.Hash +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgReject) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + if pver < RejectVersion { + str := fmt.Sprintf("reject message invalid for protocol "+ + "version %d", pver) + return messageError("MsgReject.BtcDecode", str) + } + + // Command that was rejected. + cmd, err := ReadVarString(r, pver) + if err != nil { + return err + } + msg.Cmd = cmd + + // Code indicating why the command was rejected. + err = readElement(r, &msg.Code) + if err != nil { + return err + } + + // Human readable string with specific details (over and above the + // reject code above) about why the command was rejected. + reason, err := ReadVarString(r, pver) + if err != nil { + return err + } + msg.Reason = reason + + // CmdBlock and CmdTx messages have an additional hash field that + // identifies the specific block or transaction. + if msg.Cmd == CmdBlock || msg.Cmd == CmdTx { + err := readElement(r, &msg.Hash) + if err != nil { + return err + } + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgReject) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + if pver < RejectVersion { + str := fmt.Sprintf("reject message invalid for protocol "+ + "version %d", pver) + return messageError("MsgReject.BtcEncode", str) + } + + // Command that was rejected. + err := WriteVarString(w, pver, msg.Cmd) + if err != nil { + return err + } + + // Code indicating why the command was rejected. + err = writeElement(w, msg.Code) + if err != nil { + return err + } + + // Human readable string with specific details (over and above the + // reject code above) about why the command was rejected. + err = WriteVarString(w, pver, msg.Reason) + if err != nil { + return err + } + + // CmdBlock and CmdTx messages have an additional hash field that + // identifies the specific block or transaction. + if msg.Cmd == CmdBlock || msg.Cmd == CmdTx { + err := writeElement(w, &msg.Hash) + if err != nil { + return err + } + } + + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgReject) Command() string { + return CmdReject +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgReject) MaxPayloadLength(pver uint32) uint32 { + plen := uint32(0) + // The reject message did not exist before protocol version + // RejectVersion. + if pver >= RejectVersion { + // Unfortunately the bitcoin protocol does not enforce a sane + // limit on the length of the reason, so the max payload is the + // overall maximum message payload. + plen = MaxMessagePayload + } + + return plen +} + +// NewMsgReject returns a new bitcoin reject message that conforms to the +// Message interface. See MsgReject for details. +func NewMsgReject(command string, code RejectCode, reason string) *MsgReject { + return &MsgReject{ + Cmd: command, + Code: code, + Reason: reason, + } +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgsendheaders.go b/vendor/github.com/ltcsuite/ltcd/wire/msgsendheaders.go new file mode 100644 index 0000000000..19505351ec --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgsendheaders.go @@ -0,0 +1,60 @@ +// Copyright (c) 2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "io" +) + +// MsgSendHeaders implements the Message interface and represents a bitcoin +// sendheaders message. It is used to request the peer send block headers +// rather than inventory vectors. +// +// This message has no payload and was not added until protocol versions +// starting with SendHeadersVersion. +type MsgSendHeaders struct{} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgSendHeaders) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + if pver < SendHeadersVersion { + str := fmt.Sprintf("sendheaders message invalid for protocol "+ + "version %d", pver) + return messageError("MsgSendHeaders.BtcDecode", str) + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgSendHeaders) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + if pver < SendHeadersVersion { + str := fmt.Sprintf("sendheaders message invalid for protocol "+ + "version %d", pver) + return messageError("MsgSendHeaders.BtcEncode", str) + } + + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgSendHeaders) Command() string { + return CmdSendHeaders +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgSendHeaders) MaxPayloadLength(pver uint32) uint32 { + return 0 +} + +// NewMsgSendHeaders returns a new bitcoin sendheaders message that conforms to +// the Message interface. See MsgSendHeaders for details. +func NewMsgSendHeaders() *MsgSendHeaders { + return &MsgSendHeaders{} +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgtx.go b/vendor/github.com/ltcsuite/ltcd/wire/msgtx.go new file mode 100644 index 0000000000..e11de31be7 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgtx.go @@ -0,0 +1,1027 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "bytes" + "fmt" + "io" + "strconv" + + "github.com/ltcsuite/ltcd/chaincfg/chainhash" +) + +const ( + // TxVersion is the current latest supported transaction version. + TxVersion = 1 + + // MaxTxInSequenceNum is the maximum sequence number the sequence field + // of a transaction input can be. + MaxTxInSequenceNum uint32 = 0xffffffff + + // MaxPrevOutIndex is the maximum index the index field of a previous + // outpoint can be. + MaxPrevOutIndex uint32 = 0xffffffff + + // SequenceLockTimeDisabled is a flag that if set on a transaction + // input's sequence number, the sequence number will not be interpreted + // as a relative locktime. + SequenceLockTimeDisabled = 1 << 31 + + // SequenceLockTimeIsSeconds is a flag that if set on a transaction + // input's sequence number, the relative locktime has units of 512 + // seconds. + SequenceLockTimeIsSeconds = 1 << 22 + + // SequenceLockTimeMask is a mask that extracts the relative locktime + // when masked against the transaction input sequence number. + SequenceLockTimeMask = 0x0000ffff + + // SequenceLockTimeGranularity is the defined time based granularity + // for seconds-based relative time locks. When converting from seconds + // to a sequence number, the value is right shifted by this amount, + // therefore the granularity of relative time locks in 512 or 2^9 + // seconds. Enforced relative lock times are multiples of 512 seconds. + SequenceLockTimeGranularity = 9 + + // defaultTxInOutAlloc is the default size used for the backing array for + // transaction inputs and outputs. The array will dynamically grow as needed, + // but this figure is intended to provide enough space for the number of + // inputs and outputs in a typical transaction without needing to grow the + // backing array multiple times. + defaultTxInOutAlloc = 15 + + // minTxInPayload is the minimum payload size for a transaction input. + // PreviousOutPoint.Hash + PreviousOutPoint.Index 4 bytes + Varint for + // SignatureScript length 1 byte + Sequence 4 bytes. + minTxInPayload = 9 + chainhash.HashSize + + // maxTxInPerMessage is the maximum number of transactions inputs that + // a transaction which fits into a message could possibly have. + maxTxInPerMessage = (MaxMessagePayload / minTxInPayload) + 1 + + // minTxOutPayload is the minimum payload size for a transaction output. + // Value 8 bytes + Varint for PkScript length 1 byte. + minTxOutPayload = 9 + + // maxTxOutPerMessage is the maximum number of transactions outputs that + // a transaction which fits into a message could possibly have. + maxTxOutPerMessage = (MaxMessagePayload / minTxOutPayload) + 1 + + // minTxPayload is the minimum payload size for a transaction. Note + // that any realistically usable transaction must have at least one + // input or output, but that is a rule enforced at a higher layer, so + // it is intentionally not included here. + // Version 4 bytes + Varint number of transaction inputs 1 byte + Varint + // number of transaction outputs 1 byte + LockTime 4 bytes + min input + // payload + min output payload. + minTxPayload = 10 + + // freeListMaxScriptSize is the size of each buffer in the free list + // that is used for deserializing scripts from the wire before they are + // concatenated into a single contiguous buffers. This value was chosen + // because it is slightly more than twice the size of the vast majority + // of all "standard" scripts. Larger scripts are still deserialized + // properly as the free list will simply be bypassed for them. + freeListMaxScriptSize = 512 + + // freeListMaxItems is the number of buffers to keep in the free list + // to use for script deserialization. This value allows up to 100 + // scripts per transaction being simultaneously deserialized by 125 + // peers. Thus, the peak usage of the free list is 12,500 * 512 = + // 6,400,000 bytes. + freeListMaxItems = 12500 + + // maxWitnessItemsPerInput is the maximum number of witness items to + // be read for the witness data for a single TxIn. This number is + // derived using a possble lower bound for the encoding of a witness + // item: 1 byte for length + 1 byte for the witness item itself, or two + // bytes. This value is then divided by the currently allowed maximum + // "cost" for a transaction. + maxWitnessItemsPerInput = 500000 + + // maxWitnessItemSize is the maximum allowed size for an item within + // an input's witness data. This number is derived from the fact that + // for script validation, each pushed item onto the stack must be less + // than 10k bytes. + maxWitnessItemSize = 11000 +) + +// witnessMarkerBytes are a pair of bytes specific to the witness encoding. If +// this sequence is encoutered, then it indicates a transaction has iwtness +// data. The first byte is an always 0x00 marker byte, which allows decoders to +// distinguish a serialized transaction with witnesses from a regular (legacy) +// one. The second byte is the Flag field, which at the moment is always 0x01, +// but may be extended in the future to accommodate auxiliary non-committed +// fields. +var witessMarkerBytes = []byte{0x00, 0x01} + +// scriptFreeList defines a free list of byte slices (up to the maximum number +// defined by the freeListMaxItems constant) that have a cap according to the +// freeListMaxScriptSize constant. It is used to provide temporary buffers for +// deserializing scripts in order to greatly reduce the number of allocations +// required. +// +// The caller can obtain a buffer from the free list by calling the Borrow +// function and should return it via the Return function when done using it. +type scriptFreeList chan []byte + +// Borrow returns a byte slice from the free list with a length according the +// provided size. A new buffer is allocated if there are any items available. +// +// When the size is larger than the max size allowed for items on the free list +// a new buffer of the appropriate size is allocated and returned. It is safe +// to attempt to return said buffer via the Return function as it will be +// ignored and allowed to go the garbage collector. +func (c scriptFreeList) Borrow(size uint64) []byte { + if size > freeListMaxScriptSize { + return make([]byte, size) + } + + var buf []byte + select { + case buf = <-c: + default: + buf = make([]byte, freeListMaxScriptSize) + } + return buf[:size] +} + +// Return puts the provided byte slice back on the free list when it has a cap +// of the expected length. The buffer is expected to have been obtained via +// the Borrow function. Any slices that are not of the appropriate size, such +// as those whose size is greater than the largest allowed free list item size +// are simply ignored so they can go to the garbage collector. +func (c scriptFreeList) Return(buf []byte) { + // Ignore any buffers returned that aren't the expected size for the + // free list. + if cap(buf) != freeListMaxScriptSize { + return + } + + // Return the buffer to the free list when it's not full. Otherwise let + // it be garbage collected. + select { + case c <- buf: + default: + // Let it go to the garbage collector. + } +} + +// Create the concurrent safe free list to use for script deserialization. As +// previously described, this free list is maintained to significantly reduce +// the number of allocations. +var scriptPool scriptFreeList = make(chan []byte, freeListMaxItems) + +// OutPoint defines a bitcoin data type that is used to track previous +// transaction outputs. +type OutPoint struct { + Hash chainhash.Hash + Index uint32 +} + +// NewOutPoint returns a new bitcoin transaction outpoint point with the +// provided hash and index. +func NewOutPoint(hash *chainhash.Hash, index uint32) *OutPoint { + return &OutPoint{ + Hash: *hash, + Index: index, + } +} + +// String returns the OutPoint in the human-readable form "hash:index". +func (o OutPoint) String() string { + // Allocate enough for hash string, colon, and 10 digits. Although + // at the time of writing, the number of digits can be no greater than + // the length of the decimal representation of maxTxOutPerMessage, the + // maximum message payload may increase in the future and this + // optimization may go unnoticed, so allocate space for 10 decimal + // digits, which will fit any uint32. + buf := make([]byte, 2*chainhash.HashSize+1, 2*chainhash.HashSize+1+10) + copy(buf, o.Hash.String()) + buf[2*chainhash.HashSize] = ':' + buf = strconv.AppendUint(buf, uint64(o.Index), 10) + return string(buf) +} + +// TxIn defines a bitcoin transaction input. +type TxIn struct { + PreviousOutPoint OutPoint + SignatureScript []byte + Witness TxWitness + Sequence uint32 +} + +// SerializeSize returns the number of bytes it would take to serialize the +// the transaction input. +func (t *TxIn) SerializeSize() int { + // Outpoint Hash 32 bytes + Outpoint Index 4 bytes + Sequence 4 bytes + + // serialized varint size for the length of SignatureScript + + // SignatureScript bytes. + return 40 + VarIntSerializeSize(uint64(len(t.SignatureScript))) + + len(t.SignatureScript) +} + +// NewTxIn returns a new bitcoin transaction input with the provided +// previous outpoint point and signature script with a default sequence of +// MaxTxInSequenceNum. +func NewTxIn(prevOut *OutPoint, signatureScript []byte, witness [][]byte) *TxIn { + return &TxIn{ + PreviousOutPoint: *prevOut, + SignatureScript: signatureScript, + Witness: witness, + Sequence: MaxTxInSequenceNum, + } +} + +// TxWitness defines the witness for a TxIn. A witness is to be interpreted as +// a slice of byte slices, or a stack with one or many elements. +type TxWitness [][]byte + +// SerializeSize returns the number of bytes it would take to serialize the the +// transaction input's witness. +func (t TxWitness) SerializeSize() int { + // A varint to signal the number of elements the witness has. + n := VarIntSerializeSize(uint64(len(t))) + + // For each element in the witness, we'll need a varint to signal the + // size of the element, then finally the number of bytes the element + // itself comprises. + for _, witItem := range t { + n += VarIntSerializeSize(uint64(len(witItem))) + n += len(witItem) + } + + return n +} + +// TxOut defines a bitcoin transaction output. +type TxOut struct { + Value int64 + PkScript []byte +} + +// SerializeSize returns the number of bytes it would take to serialize the +// the transaction output. +func (t *TxOut) SerializeSize() int { + // Value 8 bytes + serialized varint size for the length of PkScript + + // PkScript bytes. + return 8 + VarIntSerializeSize(uint64(len(t.PkScript))) + len(t.PkScript) +} + +// NewTxOut returns a new bitcoin transaction output with the provided +// transaction value and public key script. +func NewTxOut(value int64, pkScript []byte) *TxOut { + return &TxOut{ + Value: value, + PkScript: pkScript, + } +} + +// MsgTx implements the Message interface and represents a bitcoin tx message. +// It is used to deliver transaction information in response to a getdata +// message (MsgGetData) for a given transaction. +// +// Use the AddTxIn and AddTxOut functions to build up the list of transaction +// inputs and outputs. +type MsgTx struct { + Version int32 + TxIn []*TxIn + TxOut []*TxOut + LockTime uint32 +} + +// AddTxIn adds a transaction input to the message. +func (msg *MsgTx) AddTxIn(ti *TxIn) { + msg.TxIn = append(msg.TxIn, ti) +} + +// AddTxOut adds a transaction output to the message. +func (msg *MsgTx) AddTxOut(to *TxOut) { + msg.TxOut = append(msg.TxOut, to) +} + +// TxHash generates the Hash for the transaction. +func (msg *MsgTx) TxHash() chainhash.Hash { + // Encode the transaction and calculate double sha256 on the result. + // Ignore the error returns since the only way the encode could fail + // is being out of memory or due to nil pointers, both of which would + // cause a run-time panic. + buf := bytes.NewBuffer(make([]byte, 0, msg.SerializeSizeStripped())) + _ = msg.SerializeNoWitness(buf) + return chainhash.DoubleHashH(buf.Bytes()) +} + +// WitnessHash generates the hash of the transaction serialized according to +// the new witness serialization defined in BIP0141 and BIP0144. The final +// output is used within the Segregated Witness commitment of all the witnesses +// within a block. If a transaction has no witness data, then the witness hash, +// is the same as its txid. +func (msg *MsgTx) WitnessHash() chainhash.Hash { + if msg.HasWitness() { + buf := bytes.NewBuffer(make([]byte, 0, msg.SerializeSize())) + _ = msg.Serialize(buf) + return chainhash.DoubleHashH(buf.Bytes()) + } + + return msg.TxHash() +} + +// Copy creates a deep copy of a transaction so that the original does not get +// modified when the copy is manipulated. +func (msg *MsgTx) Copy() *MsgTx { + // Create new tx and start by copying primitive values and making space + // for the transaction inputs and outputs. + newTx := MsgTx{ + Version: msg.Version, + TxIn: make([]*TxIn, 0, len(msg.TxIn)), + TxOut: make([]*TxOut, 0, len(msg.TxOut)), + LockTime: msg.LockTime, + } + + // Deep copy the old TxIn data. + for _, oldTxIn := range msg.TxIn { + // Deep copy the old previous outpoint. + oldOutPoint := oldTxIn.PreviousOutPoint + newOutPoint := OutPoint{} + newOutPoint.Hash.SetBytes(oldOutPoint.Hash[:]) + newOutPoint.Index = oldOutPoint.Index + + // Deep copy the old signature script. + var newScript []byte + oldScript := oldTxIn.SignatureScript + oldScriptLen := len(oldScript) + if oldScriptLen > 0 { + newScript = make([]byte, oldScriptLen) + copy(newScript, oldScript[:oldScriptLen]) + } + + // Create new txIn with the deep copied data. + newTxIn := TxIn{ + PreviousOutPoint: newOutPoint, + SignatureScript: newScript, + Sequence: oldTxIn.Sequence, + } + + // If the transaction is witnessy, then also copy the + // witnesses. + if len(oldTxIn.Witness) != 0 { + // Deep copy the old witness data. + newTxIn.Witness = make([][]byte, len(oldTxIn.Witness)) + for i, oldItem := range oldTxIn.Witness { + newItem := make([]byte, len(oldItem)) + copy(newItem, oldItem) + newTxIn.Witness[i] = newItem + } + } + + // Finally, append this fully copied txin. + newTx.TxIn = append(newTx.TxIn, &newTxIn) + } + + // Deep copy the old TxOut data. + for _, oldTxOut := range msg.TxOut { + // Deep copy the old PkScript + var newScript []byte + oldScript := oldTxOut.PkScript + oldScriptLen := len(oldScript) + if oldScriptLen > 0 { + newScript = make([]byte, oldScriptLen) + copy(newScript, oldScript[:oldScriptLen]) + } + + // Create new txOut with the deep copied data and append it to + // new Tx. + newTxOut := TxOut{ + Value: oldTxOut.Value, + PkScript: newScript, + } + newTx.TxOut = append(newTx.TxOut, &newTxOut) + } + + return &newTx +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +// See Deserialize for decoding transactions stored to disk, such as in a +// database, as opposed to decoding transactions from the wire. +func (msg *MsgTx) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + version, err := binarySerializer.Uint32(r, littleEndian) + if err != nil { + return err + } + msg.Version = int32(version) + + count, err := ReadVarInt(r, pver) + if err != nil { + return err + } + + // A count of zero (meaning no TxIn's to the uninitiated) indicates + // this is a transaction with witness data. + var flag [1]byte + if count == 0 && enc == WitnessEncoding { + // Next, we need to read the flag, which is a single byte. + if _, err = io.ReadFull(r, flag[:]); err != nil { + return err + } + + // At the moment, the flag MUST be 0x01. In the future other + // flag types may be supported. + if flag[0] != 0x01 { + str := fmt.Sprintf("witness tx but flag byte is %x", flag) + return messageError("MsgTx.BtcDecode", str) + } + + // With the Segregated Witness specific fields decoded, we can + // now read in the actual txin count. + count, err = ReadVarInt(r, pver) + if err != nil { + return err + } + } + + // Prevent more input transactions than could possibly fit into a + // message. It would be possible to cause memory exhaustion and panics + // without a sane upper bound on this count. + if count > uint64(maxTxInPerMessage) { + str := fmt.Sprintf("too many input transactions to fit into "+ + "max message size [count %d, max %d]", count, + maxTxInPerMessage) + return messageError("MsgTx.BtcDecode", str) + } + + // returnScriptBuffers is a closure that returns any script buffers that + // were borrowed from the pool when there are any deserialization + // errors. This is only valid to call before the final step which + // replaces the scripts with the location in a contiguous buffer and + // returns them. + returnScriptBuffers := func() { + for _, txIn := range msg.TxIn { + if txIn == nil { + continue + } + + if txIn.SignatureScript != nil { + scriptPool.Return(txIn.SignatureScript) + } + + for _, witnessElem := range txIn.Witness { + if witnessElem != nil { + scriptPool.Return(witnessElem) + } + } + } + for _, txOut := range msg.TxOut { + if txOut == nil || txOut.PkScript == nil { + continue + } + scriptPool.Return(txOut.PkScript) + } + } + + // Deserialize the inputs. + var totalScriptSize uint64 + txIns := make([]TxIn, count) + msg.TxIn = make([]*TxIn, count) + for i := uint64(0); i < count; i++ { + // The pointer is set now in case a script buffer is borrowed + // and needs to be returned to the pool on error. + ti := &txIns[i] + msg.TxIn[i] = ti + err = readTxIn(r, pver, msg.Version, ti) + if err != nil { + returnScriptBuffers() + return err + } + totalScriptSize += uint64(len(ti.SignatureScript)) + } + + count, err = ReadVarInt(r, pver) + if err != nil { + returnScriptBuffers() + return err + } + + // Prevent more output transactions than could possibly fit into a + // message. It would be possible to cause memory exhaustion and panics + // without a sane upper bound on this count. + if count > uint64(maxTxOutPerMessage) { + returnScriptBuffers() + str := fmt.Sprintf("too many output transactions to fit into "+ + "max message size [count %d, max %d]", count, + maxTxOutPerMessage) + return messageError("MsgTx.BtcDecode", str) + } + + // Deserialize the outputs. + txOuts := make([]TxOut, count) + msg.TxOut = make([]*TxOut, count) + for i := uint64(0); i < count; i++ { + // The pointer is set now in case a script buffer is borrowed + // and needs to be returned to the pool on error. + to := &txOuts[i] + msg.TxOut[i] = to + err = readTxOut(r, pver, msg.Version, to) + if err != nil { + returnScriptBuffers() + return err + } + totalScriptSize += uint64(len(to.PkScript)) + } + + // If the transaction's flag byte isn't 0x00 at this point, then one or + // more of its inputs has accompanying witness data. + if flag[0] != 0 && enc == WitnessEncoding { + for _, txin := range msg.TxIn { + // For each input, the witness is encoded as a stack + // with one or more items. Therefore, we first read a + // varint which encodes the number of stack items. + witCount, err := ReadVarInt(r, pver) + if err != nil { + returnScriptBuffers() + return err + } + + // Prevent a possible memory exhaustion attack by + // limiting the witCount value to a sane upper bound. + if witCount > maxWitnessItemsPerInput { + returnScriptBuffers() + str := fmt.Sprintf("too many witness items to fit "+ + "into max message size [count %d, max %d]", + witCount, maxWitnessItemsPerInput) + return messageError("MsgTx.BtcDecode", str) + } + + // Then for witCount number of stack items, each item + // has a varint length prefix, followed by the witness + // item itself. + txin.Witness = make([][]byte, witCount) + for j := uint64(0); j < witCount; j++ { + txin.Witness[j], err = readScript(r, pver, + maxWitnessItemSize, "script witness item") + if err != nil { + returnScriptBuffers() + return err + } + totalScriptSize += uint64(len(txin.Witness[j])) + } + } + } + + msg.LockTime, err = binarySerializer.Uint32(r, littleEndian) + if err != nil { + returnScriptBuffers() + return err + } + + // Create a single allocation to house all of the scripts and set each + // input signature script and output public key script to the + // appropriate subslice of the overall contiguous buffer. Then, return + // each individual script buffer back to the pool so they can be reused + // for future deserializations. This is done because it significantly + // reduces the number of allocations the garbage collector needs to + // track, which in turn improves performance and drastically reduces the + // amount of runtime overhead that would otherwise be needed to keep + // track of millions of small allocations. + // + // NOTE: It is no longer valid to call the returnScriptBuffers closure + // after these blocks of code run because it is already done and the + // scripts in the transaction inputs and outputs no longer point to the + // buffers. + var offset uint64 + scripts := make([]byte, totalScriptSize) + for i := 0; i < len(msg.TxIn); i++ { + // Copy the signature script into the contiguous buffer at the + // appropriate offset. + signatureScript := msg.TxIn[i].SignatureScript + copy(scripts[offset:], signatureScript) + + // Reset the signature script of the transaction input to the + // slice of the contiguous buffer where the script lives. + scriptSize := uint64(len(signatureScript)) + end := offset + scriptSize + msg.TxIn[i].SignatureScript = scripts[offset:end:end] + offset += scriptSize + + // Return the temporary script buffer to the pool. + scriptPool.Return(signatureScript) + + for j := 0; j < len(msg.TxIn[i].Witness); j++ { + // Copy each item within the witness stack for this + // input into the contiguous buffer at the appropriate + // offset. + witnessElem := msg.TxIn[i].Witness[j] + copy(scripts[offset:], witnessElem) + + // Reset the witness item within the stack to the slice + // of the contiguous buffer where the witness lives. + witnessElemSize := uint64(len(witnessElem)) + end := offset + witnessElemSize + msg.TxIn[i].Witness[j] = scripts[offset:end:end] + offset += witnessElemSize + + // Return the temporary buffer used for the witness stack + // item to the pool. + scriptPool.Return(witnessElem) + } + } + for i := 0; i < len(msg.TxOut); i++ { + // Copy the public key script into the contiguous buffer at the + // appropriate offset. + pkScript := msg.TxOut[i].PkScript + copy(scripts[offset:], pkScript) + + // Reset the public key script of the transaction output to the + // slice of the contiguous buffer where the script lives. + scriptSize := uint64(len(pkScript)) + end := offset + scriptSize + msg.TxOut[i].PkScript = scripts[offset:end:end] + offset += scriptSize + + // Return the temporary script buffer to the pool. + scriptPool.Return(pkScript) + } + + return nil +} + +// Deserialize decodes a transaction from r into the receiver using a format +// that is suitable for long-term storage such as a database while respecting +// the Version field in the transaction. This function differs from BtcDecode +// in that BtcDecode decodes from the bitcoin wire protocol as it was sent +// across the network. The wire encoding can technically differ depending on +// the protocol version and doesn't even really need to match the format of a +// stored transaction at all. As of the time this comment was written, the +// encoded transaction is the same in both instances, but there is a distinct +// difference and separating the two allows the API to be flexible enough to +// deal with changes. +func (msg *MsgTx) Deserialize(r io.Reader) error { + // At the current time, there is no difference between the wire encoding + // at protocol version 0 and the stable long-term storage format. As + // a result, make use of BtcDecode. + return msg.BtcDecode(r, 0, WitnessEncoding) +} + +// DeserializeNoWitness decodes a transaction from r into the receiver, where +// the transaction encoding format within r MUST NOT utilize the new +// serialization format created to encode transaction bearing witness data +// within inputs. +func (msg *MsgTx) DeserializeNoWitness(r io.Reader) error { + return msg.BtcDecode(r, 0, BaseEncoding) +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +// See Serialize for encoding transactions to be stored to disk, such as in a +// database, as opposed to encoding transactions for the wire. +func (msg *MsgTx) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + err := binarySerializer.PutUint32(w, littleEndian, uint32(msg.Version)) + if err != nil { + return err + } + + // If the encoding version is set to WitnessEncoding, and the Flags + // field for the MsgTx aren't 0x00, then this indicates the transaction + // is to be encoded using the new witness inclusionary structure + // defined in BIP0144. + doWitness := enc == WitnessEncoding && msg.HasWitness() + if doWitness { + // After the txn's Version field, we include two additional + // bytes specific to the witness encoding. The first byte is an + // always 0x00 marker byte, which allows decoders to + // distinguish a serialized transaction with witnesses from a + // regular (legacy) one. The second byte is the Flag field, + // which at the moment is always 0x01, but may be extended in + // the future to accommodate auxiliary non-committed fields. + if _, err := w.Write(witessMarkerBytes); err != nil { + return err + } + } + + count := uint64(len(msg.TxIn)) + err = WriteVarInt(w, pver, count) + if err != nil { + return err + } + + for _, ti := range msg.TxIn { + err = writeTxIn(w, pver, msg.Version, ti) + if err != nil { + return err + } + } + + count = uint64(len(msg.TxOut)) + err = WriteVarInt(w, pver, count) + if err != nil { + return err + } + + for _, to := range msg.TxOut { + err = WriteTxOut(w, pver, msg.Version, to) + if err != nil { + return err + } + } + + // If this transaction is a witness transaction, and the witness + // encoded is desired, then encode the witness for each of the inputs + // within the transaction. + if doWitness { + for _, ti := range msg.TxIn { + err = writeTxWitness(w, pver, msg.Version, ti.Witness) + if err != nil { + return err + } + } + } + + return binarySerializer.PutUint32(w, littleEndian, msg.LockTime) +} + +// HasWitness returns false if none of the inputs within the transaction +// contain witness data, true false otherwise. +func (msg *MsgTx) HasWitness() bool { + for _, txIn := range msg.TxIn { + if len(txIn.Witness) != 0 { + return true + } + } + + return false +} + +// Serialize encodes the transaction to w using a format that suitable for +// long-term storage such as a database while respecting the Version field in +// the transaction. This function differs from BtcEncode in that BtcEncode +// encodes the transaction to the bitcoin wire protocol in order to be sent +// across the network. The wire encoding can technically differ depending on +// the protocol version and doesn't even really need to match the format of a +// stored transaction at all. As of the time this comment was written, the +// encoded transaction is the same in both instances, but there is a distinct +// difference and separating the two allows the API to be flexible enough to +// deal with changes. +func (msg *MsgTx) Serialize(w io.Writer) error { + // At the current time, there is no difference between the wire encoding + // at protocol version 0 and the stable long-term storage format. As + // a result, make use of BtcEncode. + // + // Passing a encoding type of WitnessEncoding to BtcEncode for MsgTx + // indicates that the transaction's witnesses (if any) should be + // serialized according to the new serialization structure defined in + // BIP0144. + return msg.BtcEncode(w, 0, WitnessEncoding) +} + +// SerializeNoWitness encodes the transaction to w in an identical manner to +// Serialize, however even if the source transaction has inputs with witness +// data, the old serialization format will still be used. +func (msg *MsgTx) SerializeNoWitness(w io.Writer) error { + return msg.BtcEncode(w, 0, BaseEncoding) +} + +// baseSize returns the serialized size of the transaction without accounting +// for any witness data. +func (msg *MsgTx) baseSize() int { + // Version 4 bytes + LockTime 4 bytes + Serialized varint size for the + // number of transaction inputs and outputs. + n := 8 + VarIntSerializeSize(uint64(len(msg.TxIn))) + + VarIntSerializeSize(uint64(len(msg.TxOut))) + + for _, txIn := range msg.TxIn { + n += txIn.SerializeSize() + } + + for _, txOut := range msg.TxOut { + n += txOut.SerializeSize() + } + + return n +} + +// SerializeSize returns the number of bytes it would take to serialize the +// the transaction. +func (msg *MsgTx) SerializeSize() int { + n := msg.baseSize() + + if msg.HasWitness() { + // The marker, and flag fields take up two additional bytes. + n += 2 + + // Additionally, factor in the serialized size of each of the + // witnesses for each txin. + for _, txin := range msg.TxIn { + n += txin.Witness.SerializeSize() + } + } + + return n +} + +// SerializeSizeStripped returns the number of bytes it would take to serialize +// the transaction, excluding any included witness data. +func (msg *MsgTx) SerializeSizeStripped() int { + return msg.baseSize() +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgTx) Command() string { + return CmdTx +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgTx) MaxPayloadLength(pver uint32) uint32 { + return MaxBlockPayload +} + +// PkScriptLocs returns a slice containing the start of each public key script +// within the raw serialized transaction. The caller can easily obtain the +// length of each script by using len on the script available via the +// appropriate transaction output entry. +func (msg *MsgTx) PkScriptLocs() []int { + numTxOut := len(msg.TxOut) + if numTxOut == 0 { + return nil + } + + // The starting offset in the serialized transaction of the first + // transaction output is: + // + // Version 4 bytes + serialized varint size for the number of + // transaction inputs and outputs + serialized size of each transaction + // input. + n := 4 + VarIntSerializeSize(uint64(len(msg.TxIn))) + + VarIntSerializeSize(uint64(numTxOut)) + + // If this transaction has a witness input, the an additional two bytes + // for the marker, and flag byte need to be taken into account. + if len(msg.TxIn) > 0 && msg.TxIn[0].Witness != nil { + n += 2 + } + + for _, txIn := range msg.TxIn { + n += txIn.SerializeSize() + } + + // Calculate and set the appropriate offset for each public key script. + pkScriptLocs := make([]int, numTxOut) + for i, txOut := range msg.TxOut { + // The offset of the script in the transaction output is: + // + // Value 8 bytes + serialized varint size for the length of + // PkScript. + n += 8 + VarIntSerializeSize(uint64(len(txOut.PkScript))) + pkScriptLocs[i] = n + n += len(txOut.PkScript) + } + + return pkScriptLocs +} + +// NewMsgTx returns a new bitcoin tx message that conforms to the Message +// interface. The return instance has a default version of TxVersion and there +// are no transaction inputs or outputs. Also, the lock time is set to zero +// to indicate the transaction is valid immediately as opposed to some time in +// future. +func NewMsgTx(version int32) *MsgTx { + return &MsgTx{ + Version: version, + TxIn: make([]*TxIn, 0, defaultTxInOutAlloc), + TxOut: make([]*TxOut, 0, defaultTxInOutAlloc), + } +} + +// readOutPoint reads the next sequence of bytes from r as an OutPoint. +func readOutPoint(r io.Reader, pver uint32, version int32, op *OutPoint) error { + _, err := io.ReadFull(r, op.Hash[:]) + if err != nil { + return err + } + + op.Index, err = binarySerializer.Uint32(r, littleEndian) + return err +} + +// writeOutPoint encodes op to the bitcoin protocol encoding for an OutPoint +// to w. +func writeOutPoint(w io.Writer, pver uint32, version int32, op *OutPoint) error { + _, err := w.Write(op.Hash[:]) + if err != nil { + return err + } + + return binarySerializer.PutUint32(w, littleEndian, op.Index) +} + +// readScript reads a variable length byte array that represents a transaction +// script. It is encoded as a varInt containing the length of the array +// followed by the bytes themselves. An error is returned if the length is +// greater than the passed maxAllowed parameter which helps protect against +// memory exhuastion attacks and forced panics thorugh malformed messages. The +// fieldName parameter is only used for the error message so it provides more +// context in the error. +func readScript(r io.Reader, pver uint32, maxAllowed uint32, fieldName string) ([]byte, error) { + count, err := ReadVarInt(r, pver) + if err != nil { + return nil, err + } + + // Prevent byte array larger than the max message size. It would + // be possible to cause memory exhaustion and panics without a sane + // upper bound on this count. + if count > uint64(maxAllowed) { + str := fmt.Sprintf("%s is larger than the max allowed size "+ + "[count %d, max %d]", fieldName, count, maxAllowed) + return nil, messageError("readScript", str) + } + + b := scriptPool.Borrow(count) + _, err = io.ReadFull(r, b) + if err != nil { + scriptPool.Return(b) + return nil, err + } + return b, nil +} + +// readTxIn reads the next sequence of bytes from r as a transaction input +// (TxIn). +func readTxIn(r io.Reader, pver uint32, version int32, ti *TxIn) error { + err := readOutPoint(r, pver, version, &ti.PreviousOutPoint) + if err != nil { + return err + } + + ti.SignatureScript, err = readScript(r, pver, MaxMessagePayload, + "transaction input signature script") + if err != nil { + return err + } + + return readElement(r, &ti.Sequence) +} + +// writeTxIn encodes ti to the bitcoin protocol encoding for a transaction +// input (TxIn) to w. +func writeTxIn(w io.Writer, pver uint32, version int32, ti *TxIn) error { + err := writeOutPoint(w, pver, version, &ti.PreviousOutPoint) + if err != nil { + return err + } + + err = WriteVarBytes(w, pver, ti.SignatureScript) + if err != nil { + return err + } + + return binarySerializer.PutUint32(w, littleEndian, ti.Sequence) +} + +// readTxOut reads the next sequence of bytes from r as a transaction output +// (TxOut). +func readTxOut(r io.Reader, pver uint32, version int32, to *TxOut) error { + err := readElement(r, &to.Value) + if err != nil { + return err + } + + to.PkScript, err = readScript(r, pver, MaxMessagePayload, + "transaction output public key script") + return err +} + +// WriteTxOut encodes to into the bitcoin protocol encoding for a transaction +// output (TxOut) to w. +// +// NOTE: This function is exported in order to allow txscript to compute the +// new sighashes for witness transactions (BIP0143). +func WriteTxOut(w io.Writer, pver uint32, version int32, to *TxOut) error { + err := binarySerializer.PutUint64(w, littleEndian, uint64(to.Value)) + if err != nil { + return err + } + + return WriteVarBytes(w, pver, to.PkScript) +} + +// writeTxWitness encodes the bitcoin protocol encoding for a transaction +// input's witness into to w. +func writeTxWitness(w io.Writer, pver uint32, version int32, wit [][]byte) error { + err := WriteVarInt(w, pver, uint64(len(wit))) + if err != nil { + return err + } + for _, item := range wit { + err = WriteVarBytes(w, pver, item) + if err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgverack.go b/vendor/github.com/ltcsuite/ltcd/wire/msgverack.go new file mode 100644 index 0000000000..60342b5642 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgverack.go @@ -0,0 +1,46 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "io" +) + +// MsgVerAck defines a bitcoin verack message which is used for a peer to +// acknowledge a version message (MsgVersion) after it has used the information +// to negotiate parameters. It implements the Message interface. +// +// This message has no payload. +type MsgVerAck struct{} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// This is part of the Message interface implementation. +func (msg *MsgVerAck) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgVerAck) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgVerAck) Command() string { + return CmdVerAck +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgVerAck) MaxPayloadLength(pver uint32) uint32 { + return 0 +} + +// NewMsgVerAck returns a new bitcoin verack message that conforms to the +// Message interface. +func NewMsgVerAck() *MsgVerAck { + return &MsgVerAck{} +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/msgversion.go b/vendor/github.com/ltcsuite/ltcd/wire/msgversion.go new file mode 100644 index 0000000000..3077f12760 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/msgversion.go @@ -0,0 +1,269 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "bytes" + "fmt" + "io" + "strings" + "time" +) + +// MaxUserAgentLen is the maximum allowed length for the user agent field in a +// version message (MsgVersion). +const MaxUserAgentLen = 256 + +// DefaultUserAgent for wire in the stack +const DefaultUserAgent = "/btcwire:0.5.0/" + +// MsgVersion implements the Message interface and represents a bitcoin version +// message. It is used for a peer to advertise itself as soon as an outbound +// connection is made. The remote peer then uses this information along with +// its own to negotiate. The remote peer must then respond with a version +// message of its own containing the negotiated values followed by a verack +// message (MsgVerAck). This exchange must take place before any further +// communication is allowed to proceed. +type MsgVersion struct { + // Version of the protocol the node is using. + ProtocolVersion int32 + + // Bitfield which identifies the enabled services. + Services ServiceFlag + + // Time the message was generated. This is encoded as an int64 on the wire. + Timestamp time.Time + + // Address of the remote peer. + AddrYou NetAddress + + // Address of the local peer. + AddrMe NetAddress + + // Unique value associated with message that is used to detect self + // connections. + Nonce uint64 + + // The user agent that generated messsage. This is a encoded as a varString + // on the wire. This has a max length of MaxUserAgentLen. + UserAgent string + + // Last block seen by the generator of the version message. + LastBlock int32 + + // Don't announce transactions to peer. + DisableRelayTx bool +} + +// HasService returns whether the specified service is supported by the peer +// that generated the message. +func (msg *MsgVersion) HasService(service ServiceFlag) bool { + return msg.Services&service == service +} + +// AddService adds service as a supported service by the peer generating the +// message. +func (msg *MsgVersion) AddService(service ServiceFlag) { + msg.Services |= service +} + +// BtcDecode decodes r using the bitcoin protocol encoding into the receiver. +// The version message is special in that the protocol version hasn't been +// negotiated yet. As a result, the pver field is ignored and any fields which +// are added in new versions are optional. This also mean that r must be a +// *bytes.Buffer so the number of remaining bytes can be ascertained. +// +// This is part of the Message interface implementation. +func (msg *MsgVersion) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { + buf, ok := r.(*bytes.Buffer) + if !ok { + return fmt.Errorf("MsgVersion.BtcDecode reader is not a " + + "*bytes.Buffer") + } + + err := readElements(buf, &msg.ProtocolVersion, &msg.Services, + (*int64Time)(&msg.Timestamp)) + if err != nil { + return err + } + + err = readNetAddress(buf, pver, &msg.AddrYou, false) + if err != nil { + return err + } + + // Protocol versions >= 106 added a from address, nonce, and user agent + // field and they are only considered present if there are bytes + // remaining in the message. + if buf.Len() > 0 { + err = readNetAddress(buf, pver, &msg.AddrMe, false) + if err != nil { + return err + } + } + if buf.Len() > 0 { + err = readElement(buf, &msg.Nonce) + if err != nil { + return err + } + } + if buf.Len() > 0 { + userAgent, err := ReadVarString(buf, pver) + if err != nil { + return err + } + err = validateUserAgent(userAgent) + if err != nil { + return err + } + msg.UserAgent = userAgent + } + + // Protocol versions >= 209 added a last known block field. It is only + // considered present if there are bytes remaining in the message. + if buf.Len() > 0 { + err = readElement(buf, &msg.LastBlock) + if err != nil { + return err + } + } + + // There was no relay transactions field before BIP0037Version, but + // the default behavior prior to the addition of the field was to always + // relay transactions. + if buf.Len() > 0 { + // It's safe to ignore the error here since the buffer has at + // least one byte and that byte will result in a boolean value + // regardless of its value. Also, the wire encoding for the + // field is true when transactions should be relayed, so reverse + // it for the DisableRelayTx field. + var relayTx bool + readElement(r, &relayTx) + msg.DisableRelayTx = !relayTx + } + + return nil +} + +// BtcEncode encodes the receiver to w using the bitcoin protocol encoding. +// This is part of the Message interface implementation. +func (msg *MsgVersion) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { + err := validateUserAgent(msg.UserAgent) + if err != nil { + return err + } + + err = writeElements(w, msg.ProtocolVersion, msg.Services, + msg.Timestamp.Unix()) + if err != nil { + return err + } + + err = writeNetAddress(w, pver, &msg.AddrYou, false) + if err != nil { + return err + } + + err = writeNetAddress(w, pver, &msg.AddrMe, false) + if err != nil { + return err + } + + err = writeElement(w, msg.Nonce) + if err != nil { + return err + } + + err = WriteVarString(w, pver, msg.UserAgent) + if err != nil { + return err + } + + err = writeElement(w, msg.LastBlock) + if err != nil { + return err + } + + // There was no relay transactions field before BIP0037Version. Also, + // the wire encoding for the field is true when transactions should be + // relayed, so reverse it from the DisableRelayTx field. + if pver >= BIP0037Version { + err = writeElement(w, !msg.DisableRelayTx) + if err != nil { + return err + } + } + return nil +} + +// Command returns the protocol command string for the message. This is part +// of the Message interface implementation. +func (msg *MsgVersion) Command() string { + return CmdVersion +} + +// MaxPayloadLength returns the maximum length the payload can be for the +// receiver. This is part of the Message interface implementation. +func (msg *MsgVersion) MaxPayloadLength(pver uint32) uint32 { + // XXX: <= 106 different + + // Protocol version 4 bytes + services 8 bytes + timestamp 8 bytes + + // remote and local net addresses + nonce 8 bytes + length of user + // agent (varInt) + max allowed useragent length + last block 4 bytes + + // relay transactions flag 1 byte. + return 33 + (maxNetAddressPayload(pver) * 2) + MaxVarIntPayload + + MaxUserAgentLen +} + +// NewMsgVersion returns a new bitcoin version message that conforms to the +// Message interface using the passed parameters and defaults for the remaining +// fields. +func NewMsgVersion(me *NetAddress, you *NetAddress, nonce uint64, + lastBlock int32) *MsgVersion { + + // Limit the timestamp to one second precision since the protocol + // doesn't support better. + return &MsgVersion{ + ProtocolVersion: int32(ProtocolVersion), + Services: 0, + Timestamp: time.Unix(time.Now().Unix(), 0), + AddrYou: *you, + AddrMe: *me, + Nonce: nonce, + UserAgent: DefaultUserAgent, + LastBlock: lastBlock, + DisableRelayTx: false, + } +} + +// validateUserAgent checks userAgent length against MaxUserAgentLen +func validateUserAgent(userAgent string) error { + if len(userAgent) > MaxUserAgentLen { + str := fmt.Sprintf("user agent too long [len %v, max %v]", + len(userAgent), MaxUserAgentLen) + return messageError("MsgVersion", str) + } + return nil +} + +// AddUserAgent adds a user agent to the user agent string for the version +// message. The version string is not defined to any strict format, although +// it is recommended to use the form "major.minor.revision" e.g. "2.6.41". +func (msg *MsgVersion) AddUserAgent(name string, version string, + comments ...string) error { + + newUserAgent := fmt.Sprintf("%s:%s", name, version) + if len(comments) != 0 { + newUserAgent = fmt.Sprintf("%s(%s)", newUserAgent, + strings.Join(comments, "; ")) + } + newUserAgent = fmt.Sprintf("%s%s/", msg.UserAgent, newUserAgent) + err := validateUserAgent(newUserAgent) + if err != nil { + return err + } + msg.UserAgent = newUserAgent + return nil +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/netaddress.go b/vendor/github.com/ltcsuite/ltcd/wire/netaddress.go new file mode 100644 index 0000000000..5a2610bccc --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/netaddress.go @@ -0,0 +1,149 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "encoding/binary" + "io" + "net" + "time" +) + +// maxNetAddressPayload returns the max payload size for a bitcoin NetAddress +// based on the protocol version. +func maxNetAddressPayload(pver uint32) uint32 { + // Services 8 bytes + ip 16 bytes + port 2 bytes. + plen := uint32(26) + + // NetAddressTimeVersion added a timestamp field. + if pver >= NetAddressTimeVersion { + // Timestamp 4 bytes. + plen += 4 + } + + return plen +} + +// NetAddress defines information about a peer on the network including the time +// it was last seen, the services it supports, its IP address, and port. +type NetAddress struct { + // Last time the address was seen. This is, unfortunately, encoded as a + // uint32 on the wire and therefore is limited to 2106. This field is + // not present in the bitcoin version message (MsgVersion) nor was it + // added until protocol version >= NetAddressTimeVersion. + Timestamp time.Time + + // Bitfield which identifies the services supported by the address. + Services ServiceFlag + + // IP address of the peer. + IP net.IP + + // Port the peer is using. This is encoded in big endian on the wire + // which differs from most everything else. + Port uint16 +} + +// HasService returns whether the specified service is supported by the address. +func (na *NetAddress) HasService(service ServiceFlag) bool { + return na.Services&service == service +} + +// AddService adds service as a supported service by the peer generating the +// message. +func (na *NetAddress) AddService(service ServiceFlag) { + na.Services |= service +} + +// NewNetAddressIPPort returns a new NetAddress using the provided IP, port, and +// supported services with defaults for the remaining fields. +func NewNetAddressIPPort(ip net.IP, port uint16, services ServiceFlag) *NetAddress { + return NewNetAddressTimestamp(time.Now(), services, ip, port) +} + +// NewNetAddressTimestamp returns a new NetAddress using the provided +// timestamp, IP, port, and supported services. The timestamp is rounded to +// single second precision. +func NewNetAddressTimestamp( + timestamp time.Time, services ServiceFlag, ip net.IP, port uint16) *NetAddress { + // Limit the timestamp to one second precision since the protocol + // doesn't support better. + na := NetAddress{ + Timestamp: time.Unix(timestamp.Unix(), 0), + Services: services, + IP: ip, + Port: port, + } + return &na +} + +// NewNetAddress returns a new NetAddress using the provided TCP address and +// supported services with defaults for the remaining fields. +func NewNetAddress(addr *net.TCPAddr, services ServiceFlag) *NetAddress { + return NewNetAddressIPPort(addr.IP, uint16(addr.Port), services) +} + +// readNetAddress reads an encoded NetAddress from r depending on the protocol +// version and whether or not the timestamp is included per ts. Some messages +// like version do not include the timestamp. +func readNetAddress(r io.Reader, pver uint32, na *NetAddress, ts bool) error { + var ip [16]byte + + // NOTE: The bitcoin protocol uses a uint32 for the timestamp so it will + // stop working somewhere around 2106. Also timestamp wasn't added until + // protocol version >= NetAddressTimeVersion + if ts && pver >= NetAddressTimeVersion { + err := readElement(r, (*uint32Time)(&na.Timestamp)) + if err != nil { + return err + } + } + + err := readElements(r, &na.Services, &ip) + if err != nil { + return err + } + // Sigh. Bitcoin protocol mixes little and big endian. + port, err := binarySerializer.Uint16(r, bigEndian) + if err != nil { + return err + } + + *na = NetAddress{ + Timestamp: na.Timestamp, + Services: na.Services, + IP: net.IP(ip[:]), + Port: port, + } + return nil +} + +// writeNetAddress serializes a NetAddress to w depending on the protocol +// version and whether or not the timestamp is included per ts. Some messages +// like version do not include the timestamp. +func writeNetAddress(w io.Writer, pver uint32, na *NetAddress, ts bool) error { + // NOTE: The bitcoin protocol uses a uint32 for the timestamp so it will + // stop working somewhere around 2106. Also timestamp wasn't added until + // until protocol version >= NetAddressTimeVersion. + if ts && pver >= NetAddressTimeVersion { + err := writeElement(w, uint32(na.Timestamp.Unix())) + if err != nil { + return err + } + } + + // Ensure to always write 16 bytes even if the ip is nil. + var ip [16]byte + if na.IP != nil { + copy(ip[:], na.IP.To16()) + } + err := writeElements(w, na.Services, ip) + if err != nil { + return err + } + + // Sigh. Bitcoin protocol mixes little and big endian. + return binary.Write(w, bigEndian, na.Port) +} diff --git a/vendor/github.com/ltcsuite/ltcd/wire/protocol.go b/vendor/github.com/ltcsuite/ltcd/wire/protocol.go new file mode 100644 index 0000000000..8fe1f163ac --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcd/wire/protocol.go @@ -0,0 +1,165 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package wire + +import ( + "fmt" + "strconv" + "strings" +) + +// XXX pedro: we will probably need to bump this. +const ( + // ProtocolVersion is the latest protocol version this package supports. + ProtocolVersion uint32 = 70015 + + // MultipleAddressVersion is the protocol version which added multiple + // addresses per message (pver >= MultipleAddressVersion). + MultipleAddressVersion uint32 = 209 + + // NetAddressTimeVersion is the protocol version which added the + // timestamp field (pver >= NetAddressTimeVersion). + NetAddressTimeVersion uint32 = 31402 + + // BIP0031Version is the protocol version AFTER which a pong message + // and nonce field in ping were added (pver > BIP0031Version). + BIP0031Version uint32 = 60000 + + // BIP0035Version is the protocol version which added the mempool + // message (pver >= BIP0035Version). + BIP0035Version uint32 = 60002 + + // BIP0037Version is the protocol version which added new connection + // bloom filtering related messages and extended the version message + // with a relay flag (pver >= BIP0037Version). + BIP0037Version uint32 = 70001 + + // RejectVersion is the protocol version which added a new reject + // message. + RejectVersion uint32 = 70002 + + // BIP0111Version is the protocol version which added the SFNodeBloom + // service flag. + BIP0111Version uint32 = 70011 + + // SendHeadersVersion is the protocol version which added a new + // sendheaders message. + SendHeadersVersion uint32 = 70012 + + // FeeFilterVersion is the protocol version which added a new + // feefilter message. + FeeFilterVersion uint32 = 70013 +) + +// ServiceFlag identifies services supported by a bitcoin peer. +type ServiceFlag uint64 + +const ( + // SFNodeNetwork is a flag used to indicate a peer is a full node. + SFNodeNetwork ServiceFlag = 1 << iota + + // SFNodeGetUTXO is a flag used to indicate a peer supports the + // getutxos and utxos commands (BIP0064). + SFNodeGetUTXO + + // SFNodeBloom is a flag used to indicate a peer supports bloom + // filtering. + SFNodeBloom + + // SFNodeWitness is a flag used to indicate a peer supports blocks + // and transactions including witness data (BIP0144). + SFNodeWitness + + // SFNodeCF is a flag used to indicate a peer supports committed + // filters (CFs). + SFNodeCF +) + +// Map of service flags back to their constant names for pretty printing. +var sfStrings = map[ServiceFlag]string{ + SFNodeNetwork: "SFNodeNetwork", + SFNodeGetUTXO: "SFNodeGetUTXO", + SFNodeBloom: "SFNodeBloom", + SFNodeWitness: "SFNodeWitness", + SFNodeCF: "SFNodeCF", +} + +// orderedSFStrings is an ordered list of service flags from highest to +// lowest. +var orderedSFStrings = []ServiceFlag{ + SFNodeNetwork, + SFNodeGetUTXO, + SFNodeBloom, + SFNodeWitness, + SFNodeCF, +} + +// String returns the ServiceFlag in human-readable form. +func (f ServiceFlag) String() string { + // No flags are set. + if f == 0 { + return "0x0" + } + + // Add individual bit flags. + s := "" + for _, flag := range orderedSFStrings { + if f&flag == flag { + s += sfStrings[flag] + "|" + f -= flag + } + } + + // Add any remaining flags which aren't accounted for as hex. + s = strings.TrimRight(s, "|") + if f != 0 { + s += "|0x" + strconv.FormatUint(uint64(f), 16) + } + s = strings.TrimLeft(s, "|") + return s +} + +// BitcoinNet represents which bitcoin network a message belongs to. +type BitcoinNet uint32 + +// Constants used to indicate the message bitcoin network. They can also be +// used to seek to the next message when a stream's state is unknown, but +// this package does not provide that functionality since it's generally a +// better idea to simply disconnect clients that are misbehaving over TCP. +const ( + // MainNet represents the main litecoin network. + MainNet BitcoinNet = 0xdbb6c0fb + + // TestNet represents the regression test network. + TestNet BitcoinNet = 0xdab5bffa + + // TestNet3 represents the test network (version 3). + TestNet3 BitcoinNet = 0x0709110b + + // TestNet4 represents the test network (version 4). + TestNet4 BitcoinNet = 0xf1c8d2fd + + // SimNet represents the simulation test network. + SimNet BitcoinNet = 0x12141c16 +) + +// bnStrings is a map of bitcoin networks back to their constant names for +// pretty printing. +var bnStrings = map[BitcoinNet]string{ + MainNet: "MainNet", + TestNet: "TestNet", + TestNet3: "TestNet3", + TestNet4: "TestNet4", + SimNet: "SimNet", +} + +// String returns the BitcoinNet in human-readable form. +func (n BitcoinNet) String() string { + if s, ok := bnStrings[n]; ok { + return s + } + + return fmt.Sprintf("Unknown BitcoinNet (%d)", uint32(n)) +} diff --git a/vendor/github.com/ltcsuite/ltcutil/LICENSE b/vendor/github.com/ltcsuite/ltcutil/LICENSE new file mode 100644 index 0000000000..3e7b16791f --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcutil/LICENSE @@ -0,0 +1,16 @@ +ISC License + +Copyright (c) 2013-2017 The btcsuite developers +Copyright (c) 2016-2017 The Lightning Network Developers + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/vendor/github.com/ltcsuite/ltcutil/base58/README.md b/vendor/github.com/ltcsuite/ltcutil/base58/README.md new file mode 100644 index 0000000000..7ed5e4c4d1 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcutil/base58/README.md @@ -0,0 +1,40 @@ +base58 +========== + +[![Build Status](http://img.shields.io/travis/ltcsuite/ltcutil.svg)] +(https://travis-ci.org/ltcsuite/ltcutil) [![ISC License] +(http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) +[![GoDoc](https://godoc.org/github.com/ltcsuite/ltcutil/base58?status.png)] +(http://godoc.org/github.com/ltcsuite/ltcutil/base58) + +Package base58 provides an API for encoding and decoding to and from the +modified base58 encoding. It also provides an API to do Base58Check encoding, +as described [here](https://en.litecoin.it/wiki/Base58Check_encoding). + +A comprehensive suite of tests is provided to ensure proper functionality. + +## Installation and Updating + +```bash +$ go get -u github.com/ltcsuite/ltcutil/base58 +``` + +## Examples + +* [Decode Example] + (http://godoc.org/github.com/ltcsuite/ltcutil/base58#example-Decode) + Demonstrates how to decode modified base58 encoded data. +* [Encode Example] + (http://godoc.org/github.com/ltcsuite/ltcutil/base58#example-Encode) + Demonstrates how to encode data using the modified base58 encoding scheme. +* [CheckDecode Example] + (http://godoc.org/github.com/ltcsuite/ltcutil/base58#example-CheckDecode) + Demonstrates how to decode Base58Check encoded data. +* [CheckEncode Example] + (http://godoc.org/github.com/ltcsuite/ltcutil/base58#example-CheckEncode) + Demonstrates how to encode data using the Base58Check encoding scheme. + +## License + +Package base58 is licensed under the [copyfree](http://copyfree.org) ISC +License. diff --git a/vendor/github.com/ltcsuite/ltcutil/base58/alphabet.go b/vendor/github.com/ltcsuite/ltcutil/base58/alphabet.go new file mode 100644 index 0000000000..6bb39fef11 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcutil/base58/alphabet.go @@ -0,0 +1,49 @@ +// Copyright (c) 2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +// AUTOGENERATED by genalphabet.go; do not edit. + +package base58 + +const ( + // alphabet is the modified base58 alphabet used by Bitcoin. + alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" + + alphabetIdx0 = '1' +) + +var b58 = [256]byte{ + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 0, 1, 2, 3, 4, 5, 6, + 7, 8, 255, 255, 255, 255, 255, 255, + 255, 9, 10, 11, 12, 13, 14, 15, + 16, 255, 17, 18, 19, 20, 21, 255, + 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 255, 255, 255, 255, 255, + 255, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 255, 44, 45, 46, + 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, +} diff --git a/vendor/github.com/ltcsuite/ltcutil/base58/base58.go b/vendor/github.com/ltcsuite/ltcutil/base58/base58.go new file mode 100644 index 0000000000..19a72de2c4 --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcutil/base58/base58.go @@ -0,0 +1,75 @@ +// Copyright (c) 2013-2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package base58 + +import ( + "math/big" +) + +//go:generate go run genalphabet.go + +var bigRadix = big.NewInt(58) +var bigZero = big.NewInt(0) + +// Decode decodes a modified base58 string to a byte slice. +func Decode(b string) []byte { + answer := big.NewInt(0) + j := big.NewInt(1) + + scratch := new(big.Int) + for i := len(b) - 1; i >= 0; i-- { + tmp := b58[b[i]] + if tmp == 255 { + return []byte("") + } + scratch.SetInt64(int64(tmp)) + scratch.Mul(j, scratch) + answer.Add(answer, scratch) + j.Mul(j, bigRadix) + } + + tmpval := answer.Bytes() + + var numZeros int + for numZeros = 0; numZeros < len(b); numZeros++ { + if b[numZeros] != alphabetIdx0 { + break + } + } + flen := numZeros + len(tmpval) + val := make([]byte, flen) + copy(val[numZeros:], tmpval) + + return val +} + +// Encode encodes a byte slice to a modified base58 string. +func Encode(b []byte) string { + x := new(big.Int) + x.SetBytes(b) + + answer := make([]byte, 0, len(b)*136/100) + for x.Cmp(bigZero) > 0 { + mod := new(big.Int) + x.DivMod(x, bigRadix, mod) + answer = append(answer, alphabet[mod.Int64()]) + } + + // leading zero bytes + for _, i := range b { + if i != 0 { + break + } + answer = append(answer, alphabetIdx0) + } + + // reverse + alen := len(answer) + for i := 0; i < alen/2; i++ { + answer[i], answer[alen-1-i] = answer[alen-1-i], answer[i] + } + + return string(answer) +} diff --git a/vendor/github.com/ltcsuite/ltcutil/base58/base58check.go b/vendor/github.com/ltcsuite/ltcutil/base58/base58check.go new file mode 100644 index 0000000000..7cdafeeece --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcutil/base58/base58check.go @@ -0,0 +1,52 @@ +// Copyright (c) 2013-2014 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package base58 + +import ( + "crypto/sha256" + "errors" +) + +// ErrChecksum indicates that the checksum of a check-encoded string does not verify against +// the checksum. +var ErrChecksum = errors.New("checksum error") + +// ErrInvalidFormat indicates that the check-encoded string has an invalid format. +var ErrInvalidFormat = errors.New("invalid format: version and/or checksum bytes missing") + +// checksum: first four bytes of sha256^2 +func checksum(input []byte) (cksum [4]byte) { + h := sha256.Sum256(input) + h2 := sha256.Sum256(h[:]) + copy(cksum[:], h2[:4]) + return +} + +// CheckEncode prepends a version byte and appends a four byte checksum. +func CheckEncode(input []byte, version byte) string { + b := make([]byte, 0, 1+len(input)+4) + b = append(b, version) + b = append(b, input[:]...) + cksum := checksum(b) + b = append(b, cksum[:]...) + return Encode(b) +} + +// CheckDecode decodes a string that was encoded with CheckEncode and verifies the checksum. +func CheckDecode(input string) (result []byte, version byte, err error) { + decoded := Decode(input) + if len(decoded) < 5 { + return nil, 0, ErrInvalidFormat + } + version = decoded[0] + var cksum [4]byte + copy(cksum[:], decoded[len(decoded)-4:]) + if checksum(decoded[:len(decoded)-4]) != cksum { + return nil, 0, ErrChecksum + } + payload := decoded[1 : len(decoded)-4] + result = append(result, payload...) + return +} diff --git a/vendor/github.com/ltcsuite/ltcutil/base58/cov_report.sh b/vendor/github.com/ltcsuite/ltcutil/base58/cov_report.sh new file mode 100644 index 0000000000..307f05b76c --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcutil/base58/cov_report.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# This script uses gocov to generate a test coverage report. +# The gocov tool my be obtained with the following command: +# go get github.com/axw/gocov/gocov +# +# It will be installed to $GOPATH/bin, so ensure that location is in your $PATH. + +# Check for gocov. +type gocov >/dev/null 2>&1 +if [ $? -ne 0 ]; then + echo >&2 "This script requires the gocov tool." + echo >&2 "You may obtain it with the following command:" + echo >&2 "go get github.com/axw/gocov/gocov" + exit 1 +fi +gocov test | gocov report diff --git a/vendor/github.com/ltcsuite/ltcutil/base58/doc.go b/vendor/github.com/ltcsuite/ltcutil/base58/doc.go new file mode 100644 index 0000000000..9a2c0e6e3d --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcutil/base58/doc.go @@ -0,0 +1,29 @@ +// Copyright (c) 2014 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* +Package base58 provides an API for working with modified base58 and Base58Check +encodings. + +Modified Base58 Encoding + +Standard base58 encoding is similar to standard base64 encoding except, as the +name implies, it uses a 58 character alphabet which results in an alphanumeric +string and allows some characters which are problematic for humans to be +excluded. Due to this, there can be various base58 alphabets. + +The modified base58 alphabet used by Bitcoin, and hence this package, omits the +0, O, I, and l characters that look the same in many fonts and are therefore +hard to humans to distinguish. + +Base58Check Encoding Scheme + +The Base58Check encoding scheme is primarily used for Bitcoin addresses at the +time of this writing, however it can be used to generically encode arbitrary +byte arrays into human-readable strings along with a version byte that can be +used to differentiate the same payload. For Bitcoin addresses, the extra +version is used to differentiate the network of otherwise identical public keys +which helps prevent using an address intended for one network on another. +*/ +package base58 diff --git a/vendor/github.com/ltcsuite/ltcutil/base58/genalphabet.go b/vendor/github.com/ltcsuite/ltcutil/base58/genalphabet.go new file mode 100644 index 0000000000..010cbee39e --- /dev/null +++ b/vendor/github.com/ltcsuite/ltcutil/base58/genalphabet.go @@ -0,0 +1,79 @@ +// Copyright (c) 2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +//+build ignore + +package main + +import ( + "bytes" + "io" + "log" + "os" + "strconv" +) + +var ( + start = []byte(`// Copyright (c) 2015 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +// AUTOGENERATED by genalphabet.go; do not edit. + +package base58 + +const ( + // alphabet is the modified base58 alphabet used by Bitcoin. + alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" + + alphabetIdx0 = '1' +) + +var b58 = [256]byte{`) + + end = []byte(`}`) + + alphabet = []byte("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz") + tab = []byte("\t") + invalid = []byte("255") + comma = []byte(",") + space = []byte(" ") + nl = []byte("\n") +) + +func write(w io.Writer, b []byte) { + _, err := w.Write(b) + if err != nil { + log.Fatal(err) + } +} + +func main() { + fi, err := os.Create("alphabet.go") + if err != nil { + log.Fatal(err) + } + defer fi.Close() + + write(fi, start) + write(fi, nl) + for i := byte(0); i < 32; i++ { + write(fi, tab) + for j := byte(0); j < 8; j++ { + idx := bytes.IndexByte(alphabet, i*8+j) + if idx == -1 { + write(fi, invalid) + } else { + write(fi, strconv.AppendInt(nil, int64(idx), 10)) + } + write(fi, comma) + if j != 7 { + write(fi, space) + } + } + write(fi, nl) + } + write(fi, end) + write(fi, nl) +} diff --git a/vendor/github.com/mutecomm/go-sqlcipher/.travis.yml b/vendor/github.com/mutecomm/go-sqlcipher/.travis.yml index 9e820def78..01609e085d 100644 --- a/vendor/github.com/mutecomm/go-sqlcipher/.travis.yml +++ b/vendor/github.com/mutecomm/go-sqlcipher/.travis.yml @@ -1,5 +1,5 @@ language: go -go: 1.8 +go: 1.5 before_install: - go get github.com/frankbraun/gocheck script: diff --git a/vendor/github.com/mutecomm/go-sqlcipher/MAINTENANCE b/vendor/github.com/mutecomm/go-sqlcipher/MAINTENANCE index 972ce0a7e5..f320de5631 100644 --- a/vendor/github.com/mutecomm/go-sqlcipher/MAINTENANCE +++ b/vendor/github.com/mutecomm/go-sqlcipher/MAINTENANCE @@ -9,13 +9,13 @@ tracked for changes (maintenance details below): Update code from https://github.com/mattn/go-sqlite3 ---------------------------------------------------- -Current HEAD: cf7286f069c3ef596efcc87781a4653a2e7607bd +Current HEAD: 5651a9d9d49ec25811d08220ee972080378d52ea Track files: error.go error_test.go -sqlite3.go (dead code and extension loading removed, sqlcipher pragmas added) +sqlite3.go (dead code and extension loading coding removed) sqlite3_other.go sqlite3_test.go (adjust path) sqlite3_windows.go @@ -26,7 +26,7 @@ sqlite3_test/sqltest.go Update code from https://github.com/sqlcipher/sqlcipher ------------------------------------------------------- -Current release: v3.4.1 +Current release: v3.3.1 Execute: ./configure diff --git a/vendor/github.com/mutecomm/go-sqlcipher/README.md b/vendor/github.com/mutecomm/go-sqlcipher/README.md index e34ccfabfe..6d1ebf5c35 100644 --- a/vendor/github.com/mutecomm/go-sqlcipher/README.md +++ b/vendor/github.com/mutecomm/go-sqlcipher/README.md @@ -36,10 +36,6 @@ you want a different value than the default 1024 bytes). API documentation can be found here: http://godoc.org/github.com/mutecomm/go-sqlcipher -Use the function -[sqlite3.IsEncrypted()](https://godoc.org/github.com/mutecomm/go-sqlcipher#IsEncrypted) -to check whether a database file is encrypted or not. - Examples can be found under the `./_example` directory diff --git a/vendor/github.com/mutecomm/go-sqlcipher/_example/simple/simple.go b/vendor/github.com/mutecomm/go-sqlcipher/_example/simple/simple.go deleted file mode 100644 index f456e12537..0000000000 --- a/vendor/github.com/mutecomm/go-sqlcipher/_example/simple/simple.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "database/sql" - "fmt" - "log" - "os" - - _ "github.com/mattn/go-sqlite3" -) - -func main() { - os.Remove("./foo.db") - - db, err := sql.Open("sqlite3", "./foo.db") - if err != nil { - log.Fatal(err) - } - defer db.Close() - - sqlStmt := ` - create table foo (id integer not null primary key, name text); - delete from foo; - ` - _, err = db.Exec(sqlStmt) - if err != nil { - log.Printf("%q: %s\n", err, sqlStmt) - return - } - - tx, err := db.Begin() - if err != nil { - log.Fatal(err) - } - stmt, err := tx.Prepare("insert into foo(id, name) values(?, ?)") - if err != nil { - log.Fatal(err) - } - defer stmt.Close() - for i := 0; i < 100; i++ { - _, err = stmt.Exec(i, fmt.Sprintf("こんにちわ世界%03d", i)) - if err != nil { - log.Fatal(err) - } - } - tx.Commit() - - rows, err := db.Query("select id, name from foo") - if err != nil { - log.Fatal(err) - } - defer rows.Close() - for rows.Next() { - var id int - var name string - err = rows.Scan(&id, &name) - if err != nil { - log.Fatal(err) - } - fmt.Println(id, name) - } - err = rows.Err() - if err != nil { - log.Fatal(err) - } - - stmt, err = db.Prepare("select name from foo where id = ?") - if err != nil { - log.Fatal(err) - } - defer stmt.Close() - var name string - err = stmt.QueryRow("3").Scan(&name) - if err != nil { - log.Fatal(err) - } - fmt.Println(name) - - _, err = db.Exec("delete from foo") - if err != nil { - log.Fatal(err) - } - - _, err = db.Exec("insert into foo(id, name) values(1, 'foo'), (2, 'bar'), (3, 'baz')") - if err != nil { - log.Fatal(err) - } - - rows, err = db.Query("select id, name from foo") - if err != nil { - log.Fatal(err) - } - defer rows.Close() - for rows.Next() { - var id int - var name string - err = rows.Scan(&id, &name) - if err != nil { - log.Fatal(err) - } - fmt.Println(id, name) - } - err = rows.Err() - if err != nil { - log.Fatal(err) - } -} diff --git a/vendor/github.com/mutecomm/go-sqlcipher/error.go b/vendor/github.com/mutecomm/go-sqlcipher/error.go index 49ab890366..b91010835c 100644 --- a/vendor/github.com/mutecomm/go-sqlcipher/error.go +++ b/vendor/github.com/mutecomm/go-sqlcipher/error.go @@ -7,16 +7,12 @@ package sqlite3 import "C" -// ErrNo inherit errno. type ErrNo int -// ErrNoMask is mask code. const ErrNoMask C.int = 0xff -// ErrNoExtended is extended errno. type ErrNoExtended int -// Error implement sqlite error code. type Error struct { Code ErrNo /* The error code returned by SQLite */ ExtendedCode ErrNoExtended /* The extended error code returned by SQLite */ @@ -56,17 +52,14 @@ var ( ErrWarning = ErrNo(28) /* Warnings from sqlite3_log() */ ) -// Error return error message from errno. func (err ErrNo) Error() string { return Error{Code: err}.Error() } -// Extend return extended errno. func (err ErrNo) Extend(by int) ErrNoExtended { return ErrNoExtended(int(err) | (by << 8)) } -// Error return error message that is extended code. func (err ErrNoExtended) Error() string { return Error{Code: ErrNo(C.int(err) & ErrNoMask), ExtendedCode: err}.Error() } @@ -128,7 +121,7 @@ var ( ErrConstraintTrigger = ErrConstraint.Extend(7) ErrConstraintUnique = ErrConstraint.Extend(8) ErrConstraintVTab = ErrConstraint.Extend(9) - ErrConstraintRowID = ErrConstraint.Extend(10) + ErrConstraintRowId = ErrConstraint.Extend(10) ErrNoticeRecoverWAL = ErrNotice.Extend(1) ErrNoticeRecoverRollback = ErrNotice.Extend(2) ErrWarningAutoIndex = ErrWarning.Extend(1) diff --git a/vendor/github.com/mutecomm/go-sqlcipher/error_test.go b/vendor/github.com/mutecomm/go-sqlcipher/error_test.go deleted file mode 100644 index 1ccbe5bf85..0000000000 --- a/vendor/github.com/mutecomm/go-sqlcipher/error_test.go +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright (C) 2014 Yasuhiro Matsumoto . -// -// Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. - -package sqlite3 - -import ( - "database/sql" - "io/ioutil" - "os" - "path" - "testing" -) - -func TestSimpleError(t *testing.T) { - e := ErrError.Error() - if e != "SQL logic error or missing database" { - t.Error("wrong error code:" + e) - } -} - -func TestCorruptDbErrors(t *testing.T) { - dirName, err := ioutil.TempDir("", "sqlite3") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dirName) - - dbFileName := path.Join(dirName, "test.db") - f, err := os.Create(dbFileName) - if err != nil { - t.Error(err) - } - f.Write([]byte{1, 2, 3, 4, 5}) - f.Close() - - db, err := sql.Open("sqlite3", dbFileName) - if err == nil { - _, err = db.Exec("drop table foo") - } - - sqliteErr := err.(Error) - if sqliteErr.Code != ErrNotADB { - t.Error("wrong error code for corrupted DB") - } - if err.Error() == "" { - t.Error("wrong error string for corrupted DB") - } - db.Close() -} - -func TestSqlLogicErrors(t *testing.T) { - dirName, err := ioutil.TempDir("", "sqlite3") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dirName) - - dbFileName := path.Join(dirName, "test.db") - db, err := sql.Open("sqlite3", dbFileName) - if err != nil { - t.Error(err) - } - defer db.Close() - - _, err = db.Exec("CREATE TABLE Foo (id INTEGER PRIMARY KEY)") - if err != nil { - t.Error(err) - } - - const expectedErr = "table Foo already exists" - _, err = db.Exec("CREATE TABLE Foo (id INTEGER PRIMARY KEY)") - if err.Error() != expectedErr { - t.Errorf("Unexpected error: %s, expected %s", err.Error(), expectedErr) - } - -} - -func TestExtendedErrorCodes_ForeignKey(t *testing.T) { - dirName, err := ioutil.TempDir("", "sqlite3-err") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dirName) - - dbFileName := path.Join(dirName, "test.db") - db, err := sql.Open("sqlite3", dbFileName) - if err != nil { - t.Error(err) - } - defer db.Close() - - _, err = db.Exec("PRAGMA foreign_keys=ON;") - if err != nil { - t.Errorf("PRAGMA foreign_keys=ON: %v", err) - } - - _, err = db.Exec(`CREATE TABLE Foo ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - value INTEGER NOT NULL, - ref INTEGER NULL REFERENCES Foo (id), - UNIQUE(value) - );`) - if err != nil { - t.Error(err) - } - - _, err = db.Exec("INSERT INTO Foo (ref, value) VALUES (100, 100);") - if err == nil { - t.Error("No error!") - } else { - sqliteErr := err.(Error) - if sqliteErr.Code != ErrConstraint { - t.Errorf("Wrong basic error code: %d != %d", - sqliteErr.Code, ErrConstraint) - } - if sqliteErr.ExtendedCode != ErrConstraintForeignKey { - t.Errorf("Wrong extended error code: %d != %d", - sqliteErr.ExtendedCode, ErrConstraintForeignKey) - } - } - -} - -func TestExtendedErrorCodes_NotNull(t *testing.T) { - dirName, err := ioutil.TempDir("", "sqlite3-err") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dirName) - - dbFileName := path.Join(dirName, "test.db") - db, err := sql.Open("sqlite3", dbFileName) - if err != nil { - t.Error(err) - } - defer db.Close() - - _, err = db.Exec("PRAGMA foreign_keys=ON;") - if err != nil { - t.Errorf("PRAGMA foreign_keys=ON: %v", err) - } - - _, err = db.Exec(`CREATE TABLE Foo ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - value INTEGER NOT NULL, - ref INTEGER NULL REFERENCES Foo (id), - UNIQUE(value) - );`) - if err != nil { - t.Error(err) - } - - res, err := db.Exec("INSERT INTO Foo (value) VALUES (100);") - if err != nil { - t.Fatalf("Creating first row: %v", err) - } - - id, err := res.LastInsertId() - if err != nil { - t.Fatalf("Retrieving last insert id: %v", err) - } - - _, err = db.Exec("INSERT INTO Foo (ref) VALUES (?);", id) - if err == nil { - t.Error("No error!") - } else { - sqliteErr := err.(Error) - if sqliteErr.Code != ErrConstraint { - t.Errorf("Wrong basic error code: %d != %d", - sqliteErr.Code, ErrConstraint) - } - if sqliteErr.ExtendedCode != ErrConstraintNotNull { - t.Errorf("Wrong extended error code: %d != %d", - sqliteErr.ExtendedCode, ErrConstraintNotNull) - } - } - -} - -func TestExtendedErrorCodes_Unique(t *testing.T) { - dirName, err := ioutil.TempDir("", "sqlite3-err") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dirName) - - dbFileName := path.Join(dirName, "test.db") - db, err := sql.Open("sqlite3", dbFileName) - if err != nil { - t.Error(err) - } - defer db.Close() - - _, err = db.Exec("PRAGMA foreign_keys=ON;") - if err != nil { - t.Errorf("PRAGMA foreign_keys=ON: %v", err) - } - - _, err = db.Exec(`CREATE TABLE Foo ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - value INTEGER NOT NULL, - ref INTEGER NULL REFERENCES Foo (id), - UNIQUE(value) - );`) - if err != nil { - t.Error(err) - } - - res, err := db.Exec("INSERT INTO Foo (value) VALUES (100);") - if err != nil { - t.Fatalf("Creating first row: %v", err) - } - - id, err := res.LastInsertId() - if err != nil { - t.Fatalf("Retrieving last insert id: %v", err) - } - - _, err = db.Exec("INSERT INTO Foo (ref, value) VALUES (?, 100);", id) - if err == nil { - t.Error("No error!") - } else { - sqliteErr := err.(Error) - if sqliteErr.Code != ErrConstraint { - t.Errorf("Wrong basic error code: %d != %d", - sqliteErr.Code, ErrConstraint) - } - if sqliteErr.ExtendedCode != ErrConstraintUnique { - t.Errorf("Wrong extended error code: %d != %d", - sqliteErr.ExtendedCode, ErrConstraintUnique) - } - extended := sqliteErr.Code.Extend(3).Error() - expected := "constraint failed" - if extended != expected { - t.Errorf("Wrong basic error code: %q != %q", - extended, expected) - } - } - -} diff --git a/vendor/github.com/mutecomm/go-sqlcipher/flags.go b/vendor/github.com/mutecomm/go-sqlcipher/flags.go index 66ef6eaefd..fec2a1612e 100644 --- a/vendor/github.com/mutecomm/go-sqlcipher/flags.go +++ b/vendor/github.com/mutecomm/go-sqlcipher/flags.go @@ -13,6 +13,9 @@ package sqlite3 // disable loadable extensions in sqlite #cgo CFLAGS: -DSQLITE_OMIT_LOAD_EXTENSION=1 +// make libtomcrypt thread-safe +#cgo CFLAGS: -DLTC_PTHREAD + // disable anything "not portable" in libtomcrypt #cgo CFLAGS: -DLTC_NO_ASM diff --git a/vendor/github.com/mutecomm/go-sqlcipher/sqlcipher.go b/vendor/github.com/mutecomm/go-sqlcipher/sqlcipher.go deleted file mode 100644 index c0e5e5f151..0000000000 --- a/vendor/github.com/mutecomm/go-sqlcipher/sqlcipher.go +++ /dev/null @@ -1,35 +0,0 @@ -package sqlite3 - -import ( - "bytes" - "errors" - "os" -) - -// sqlite3Header defines the header string used by SQLite 3. -var sqlite3Header = []byte("SQLite format 3\000") - -// IsEncrypted returns true, if the database with the given filename is -// encrypted, and false otherwise. -// If the database header cannot be read properly an error is returned. -func IsEncrypted(filename string) (bool, error) { - // open file - db, err := os.Open(filename) - if err != nil { - return false, err - } - defer db.Close() - // read header - var header [16]byte - n, err := db.Read(header[:]) - if err != nil { - return false, err - } - if n != len(header) { - return false, errors.New("go-sqlcipher: could not read full header") - } - // SQLCipher encrypts also the header, the file is encrypted if the read - // header does not equal the header string used by SQLite 3. - encrypted := !bytes.Equal(header[:], sqlite3Header) - return encrypted, nil -} diff --git a/vendor/github.com/mutecomm/go-sqlcipher/sqlcipher_test.go b/vendor/github.com/mutecomm/go-sqlcipher/sqlcipher_test.go deleted file mode 100644 index 49c8281dfc..0000000000 --- a/vendor/github.com/mutecomm/go-sqlcipher/sqlcipher_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package sqlite3_test - -import ( - "crypto/rand" - "database/sql" - "encoding/hex" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "path/filepath" - "testing" - - "github.com/mutecomm/go-sqlcipher" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -var ( - db *sql.DB - testDir = "go-sqlcipher_test" - tables = ` -CREATE TABLE KeyValueStore ( - KeyEntry TEXT NOT NULL UNIQUE, - ValueEntry TEXT NOT NULL -);` -) - -func init() { - // create DB - key := "passphrase" - tmpdir, err := ioutil.TempDir("", testDir) - if err != nil { - panic(err) - } - dbname := filepath.Join(tmpdir, "sqlcipher_test") - dbnameWithDSN := dbname + fmt.Sprintf("?_pragma_key=%s&_pragma_cipher_page_size=4096", key) - db, err = sql.Open("sqlite3", dbnameWithDSN) - if err != nil { - panic(err) - } - _, err = db.Exec(tables) - if err != nil { - panic(err) - } - db.Close() - // make sure DB is encrypted - encrypted, err := sqlite3.IsEncrypted(dbname) - if err != nil { - panic(err) - } - if !encrypted { - panic(errors.New("go-sqlcipher: DB not encrypted")) - } - // open DB for testing - db, err = sql.Open("sqlite3", dbnameWithDSN) - if err != nil { - panic(err) - } - _, err = db.Exec("SELECT count(*) FROM sqlite_master;") - if err != nil { - panic(err) - } -} - -var mapping = map[string]string{ - "foo": "one", - "bar": "two", - "baz": "three", -} - -func TestSQLCipherParallelInsert(t *testing.T) { - t.Parallel() - insertValueQuery, err := db.Prepare("INSERT INTO KeyValueStore (KeyEntry, ValueEntry) VALUES (?, ?);") - require.NoError(t, err) - for key, value := range mapping { - _, err := insertValueQuery.Exec(key, value) - assert.NoError(t, err) - } -} - -func TestSQLCipherParallelSelect(t *testing.T) { - t.Parallel() - getValueQuery, err := db.Prepare("SELECT ValueEntry FROM KeyValueStore WHERE KeyEntry=?;") - if err != nil { - t.Fatal(err) - } - for key, value := range mapping { - var val string - err := getValueQuery.QueryRow(key).Scan(&val) - if err != sql.ErrNoRows { - if assert.NoError(t, err) { - assert.Equal(t, value, val) - } - } - } -} - -func TestSQLCipherIsEncryptedFalse(t *testing.T) { - tmpdir, err := ioutil.TempDir("", testDir) - require.NoError(t, err) - defer os.RemoveAll(tmpdir) - dbname := filepath.Join(tmpdir, "unencrypted.sqlite") - db, err := sql.Open("sqlite3", dbname) - require.NoError(t, err) - defer db.Close() - _, err = db.Exec(tables) - require.NoError(t, err) - encrypted, err := sqlite3.IsEncrypted(dbname) - if assert.NoError(t, err) { - assert.False(t, encrypted) - } -} - -func TestSQLCipherIsEncryptedTrue(t *testing.T) { - tmpdir, err := ioutil.TempDir("", testDir) - require.NoError(t, err) - defer os.RemoveAll(tmpdir) - dbname := filepath.Join(tmpdir, "encrypted.sqlite") - var key [32]byte - _, err = io.ReadFull(rand.Reader, key[:]) - require.NoError(t, err) - dbnameWithDSN := dbname + fmt.Sprintf("?_pragma_key=x'%s'", - hex.EncodeToString(key[:])) - db, err := sql.Open("sqlite3", dbnameWithDSN) - require.NoError(t, err) - defer db.Close() - _, err = db.Exec(tables) - require.NoError(t, err) - encrypted, err := sqlite3.IsEncrypted(dbname) - if assert.NoError(t, err) { - assert.True(t, encrypted) - } -} - -func ExampleIsEncrypted() { - // create random key - var key [32]byte - _, err := io.ReadFull(rand.Reader, key[:]) - if err != nil { - log.Fatal(err) - } - // set DB name - dbname := "go-sqlcipher.sqlite" - dbnameWithDSN := dbname + fmt.Sprintf("?_pragma_key=x'%s'", - hex.EncodeToString(key[:])) - // create encrypted DB file - db, err := sql.Open("sqlite3", dbnameWithDSN) - if err != nil { - log.Fatal(err) - } - defer os.Remove(dbname) - defer db.Close() - // create table - _, err = db.Exec("CREATE TABLE t(x INTEGER);") - if err != nil { - log.Fatal(err) - } - // make sure database is encrypted - encrypted, err := sqlite3.IsEncrypted(dbname) - if err != nil { - log.Fatal(err) - } - if encrypted { - fmt.Println("DB is encrypted") - } else { - fmt.Println("DB is unencrypted") - } - // Output: - // DB is encrypted -} diff --git a/vendor/github.com/mutecomm/go-sqlcipher/sqlite3.c b/vendor/github.com/mutecomm/go-sqlcipher/sqlite3.c index 3aee1fb5a0..e9774dd529 100644 --- a/vendor/github.com/mutecomm/go-sqlcipher/sqlite3.c +++ b/vendor/github.com/mutecomm/go-sqlcipher/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.15.2. By combining all the individual C code files into this +** version 3.8.10.2. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -9,7 +9,7 @@ ** ** This file is all you need to compile SQLite. To use SQLite in other ** programs, you need this file and the "sqlite3.h" header file that defines -** the programming interface to the SQLite library. (If you do not have +** the programming interface to the SQLite library. (If you do not have ** the "sqlite3.h" header file at hand, you will find a copy embedded within ** the text of this file. Search for "Begin file sqlite3.h" to find the start ** of the embedded sqlite3.h header file.) Additional code files may be needed @@ -37,51 +37,8 @@ ** Internal interface definitions for SQLite. ** */ -#ifndef SQLITEINT_H -#define SQLITEINT_H - -/* Special Comments: -** -** Some comments have special meaning to the tools that measure test -** coverage: -** -** NO_TEST - The branches on this line are not -** measured by branch coverage. This is -** used on lines of code that actually -** implement parts of coverage testing. -** -** OPTIMIZATION-IF-TRUE - This branch is allowed to alway be false -** and the correct answer is still obtained, -** though perhaps more slowly. -** -** OPTIMIZATION-IF-FALSE - This branch is allowed to alway be true -** and the correct answer is still obtained, -** though perhaps more slowly. -** -** PREVENTS-HARMLESS-OVERREAD - This branch prevents a buffer overread -** that would be harmless and undetectable -** if it did occur. -** -** In all cases, the special comment must be enclosed in the usual -** slash-asterisk...asterisk-slash comment marks, with no spaces between the -** asterisks and the comment text. -*/ - -/* -** Make sure the Tcl calling convention macro is defined. This macro is -** only used by test code and Tcl integration code. -*/ -#ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -#endif - -/* -** Make sure that rand_s() is available on Windows systems with MSVC 2005 -** or higher. -*/ -#if defined(_MSC_VER) && _MSC_VER>=1400 -# define _CRT_RAND_S -#endif +#ifndef _SQLITEINT_H_ +#define _SQLITEINT_H_ /* ** Include the header file used to customize the compiler options for MSVC. @@ -105,8 +62,8 @@ ** ** This file contains code that is specific to MSVC. */ -#ifndef SQLITE_MSVC_H -#define SQLITE_MSVC_H +#ifndef _MSVC_H_ +#define _MSVC_H_ #if defined(_MSC_VER) #pragma warning(disable : 4054) @@ -126,7 +83,7 @@ #pragma warning(disable : 4706) #endif /* defined(_MSC_VER) */ -#endif /* SQLITE_MSVC_H */ +#endif /* _MSVC_H_ */ /************** End of msvc.h ************************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -164,9 +121,6 @@ #else /* This is not VxWorks. */ #define OS_VXWORKS 0 -#define HAVE_FCHOWN 1 -#define HAVE_READLINK 1 -#define HAVE_LSTAT 1 #endif /* defined(_WRS_KERNEL) */ /************** End of vxworks.h *********************************************/ @@ -204,13 +158,6 @@ # define _LARGEFILE_SOURCE 1 #endif -/* What version of GCC is being used. 0 means GCC is not being used */ -#ifdef __GNUC__ -# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) -#else -# define GCC_VERSION 0 -#endif - /* Needed for various definitions... */ #if defined(__GNUC__) && !defined(_GNU_SOURCE) # define _GNU_SOURCE @@ -283,15 +230,15 @@ ** ** The official C-language API documentation for SQLite is derived ** from comments in this file. This file is the authoritative source -** on how SQLite interfaces are supposed to operate. +** on how SQLite interfaces are suppose to operate. ** ** The name of this file under configuration management is "sqlite.h.in". ** The makefile makes some minor changes to this file (such as inserting ** the version number) and changes its name to "sqlite3.h" as ** part of the build process. */ -#ifndef SQLITE3_H -#define SQLITE3_H +#ifndef _SQLITE3_H_ +#define _SQLITE3_H_ #include /* Needed for the definition of va_list */ /* @@ -314,17 +261,8 @@ extern "C" { #ifndef SQLITE_CDECL # define SQLITE_CDECL #endif -#ifndef SQLITE_APICALL -# define SQLITE_APICALL -#endif #ifndef SQLITE_STDCALL -# define SQLITE_STDCALL SQLITE_APICALL -#endif -#ifndef SQLITE_CALLBACK -# define SQLITE_CALLBACK -#endif -#ifndef SQLITE_SYSAPI -# define SQLITE_SYSAPI +# define SQLITE_STDCALL #endif /* @@ -368,8 +306,7 @@ extern "C" { ** be held constant and Z will be incremented or else Y will be incremented ** and Z will be reset to zero. ** -** Since [version 3.6.18] ([dateof:3.6.18]), -** SQLite source code has been stored in the +** Since version 3.6.18, SQLite source code has been stored in the ** Fossil configuration management ** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite @@ -381,9 +318,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.15.2" -#define SQLITE_VERSION_NUMBER 3015002 -#define SQLITE_SOURCE_ID "2016-11-28 19:13:37 bbd85d235f7037c6a033a9690534391ffeacecc8" +#define SQLITE_VERSION "3.8.10.2" +#define SQLITE_VERSION_NUMBER 3008010 +#define SQLITE_SOURCE_ID "2015-05-20 18:17:19 2ef4f3a5b1d1d0c4338f8243d40a2452cc1f7fe4" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -394,7 +331,7 @@ extern "C" { ** but are associated with the library instead of the header file. ^(Cautious ** programmers might include assert() statements in their application to ** verify that values returned by these interfaces match the macros in -** the header, and thus ensure that the application is +** the header, and thus insure that the application is ** compiled with matching library and header files. ** **
@@ -416,9 +353,9 @@ extern "C" {
 ** See also: [sqlite_version()] and [sqlite_source_id()].
 */
 SQLITE_API const char sqlite3_version[] = SQLITE_VERSION;
-SQLITE_API const char *sqlite3_libversion(void);
-SQLITE_API const char *sqlite3_sourceid(void);
-SQLITE_API int sqlite3_libversion_number(void);
+SQLITE_API const char *SQLITE_STDCALL sqlite3_libversion(void);
+SQLITE_API const char *SQLITE_STDCALL sqlite3_sourceid(void);
+SQLITE_API int SQLITE_STDCALL sqlite3_libversion_number(void);
 
 /*
 ** CAPI3REF: Run-Time Library Compilation Options Diagnostics
@@ -443,8 +380,8 @@ SQLITE_API int sqlite3_libversion_number(void);
 ** [sqlite_compileoption_get()] and the [compile_options pragma].
 */
 #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
-SQLITE_API int sqlite3_compileoption_used(const char *zOptName);
-SQLITE_API const char *sqlite3_compileoption_get(int N);
+SQLITE_API int SQLITE_STDCALL sqlite3_compileoption_used(const char *zOptName);
+SQLITE_API const char *SQLITE_STDCALL sqlite3_compileoption_get(int N);
 #endif
 
 /*
@@ -483,7 +420,7 @@ SQLITE_API const char *sqlite3_compileoption_get(int N);
 **
 ** See the [threading mode] documentation for additional information.
 */
-SQLITE_API int sqlite3_threadsafe(void);
+SQLITE_API int SQLITE_STDCALL sqlite3_threadsafe(void);
 
 /*
 ** CAPI3REF: Database Connection Handle
@@ -580,8 +517,8 @@ typedef sqlite_uint64 sqlite3_uint64;
 ** ^Calling sqlite3_close() or sqlite3_close_v2() with a NULL pointer
 ** argument is a harmless no-op.
 */
-SQLITE_API int sqlite3_close(sqlite3*);
-SQLITE_API int sqlite3_close_v2(sqlite3*);
+SQLITE_API int SQLITE_STDCALL sqlite3_close(sqlite3*);
+SQLITE_API int SQLITE_STDCALL sqlite3_close_v2(sqlite3*);
 
 /*
 ** The type for a callback function.
@@ -617,7 +554,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** from [sqlite3_malloc()] and passed back through the 5th parameter.
 ** To avoid memory leaks, the application should invoke [sqlite3_free()]
 ** on error message strings returned through the 5th parameter of
-** sqlite3_exec() after the error message string is no longer needed.
+** of sqlite3_exec() after the error message string is no longer needed.
 ** ^If the 5th parameter to sqlite3_exec() is not NULL and no errors
 ** occur, then sqlite3_exec() sets the pointer in its 5th parameter to
 ** NULL before returning.
@@ -644,7 +581,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** Restrictions:
 **
 ** 
    -**
  • The application must ensure that the 1st parameter to sqlite3_exec() +**
  • The application must insure that the 1st parameter to sqlite3_exec() ** is a valid and open [database connection]. **
  • The application must not close the [database connection] specified by ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. @@ -652,7 +589,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. **
*/ -SQLITE_API int sqlite3_exec( +SQLITE_API int SQLITE_STDCALL sqlite3_exec( sqlite3*, /* An open database */ const char *sql, /* SQL to be evaluated */ int (*callback)(void*,int,char**,char**), /* Callback function */ @@ -713,8 +650,7 @@ SQLITE_API int sqlite3_exec( ** [result codes]. However, experience has shown that many of ** these result codes are too coarse-grained. They do not provide as ** much information about problems as programmers might like. In an effort to -** address this, newer versions of SQLite (version 3.3.8 [dateof:3.3.8] -** and later) include +** address this, newer versions of SQLite (version 3.3.8 and later) include ** support for additional result codes that provide more detailed information ** about errors. These [extended result codes] are enabled or disabled ** on a per database connection basis using the @@ -748,8 +684,6 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) #define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) -#define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8)) -#define SQLITE_IOERR_AUTH (SQLITE_IOERR | (28<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) @@ -777,7 +711,6 @@ SQLITE_API int sqlite3_exec( #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) -#define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -1066,13 +999,8 @@ struct sqlite3_io_methods { **
  • [[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database -** connection. See also [SQLITE_FCNTL_JOURNAL_POINTER]. -** -**
  • [[SQLITE_FCNTL_JOURNAL_POINTER]] -** The [SQLITE_FCNTL_JOURNAL_POINTER] opcode is used to obtain a pointer -** to the [sqlite3_file] object associated with the journal file (either -** the [rollback journal] or the [write-ahead log]) for a particular database -** connection. See also [SQLITE_FCNTL_FILE_POINTER]. +** connection. See the [sqlite3_file_control()] documentation for +** additional information. ** **
  • [[SQLITE_FCNTL_SYNC_OMITTED]] ** No longer in use. @@ -1159,15 +1087,6 @@ struct sqlite3_io_methods { ** pointer in case this file-control is not implemented. This file-control ** is intended for diagnostic use only. ** -**
  • [[SQLITE_FCNTL_VFS_POINTER]] -** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level -** [VFSes] currently in use. ^(The argument X in -** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be -** of type "[sqlite3_vfs] **". This opcodes will set *X -** to a pointer to the top-level VFS.)^ -** ^When there are multiple VFS shims in the stack, this opcode finds the -** upper-most shim only. -** **
  • [[SQLITE_FCNTL_PRAGMA]] ** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] ** file control is sent to the open [sqlite3_file] object corresponding @@ -1238,12 +1157,6 @@ struct sqlite3_io_methods { ** on whether or not the file has been renamed, moved, or deleted since it ** was first opened. ** -**
  • [[SQLITE_FCNTL_WIN32_GET_HANDLE]] -** The [SQLITE_FCNTL_WIN32_GET_HANDLE] opcode can be used to obtain the -** underlying native file handle associated with a file handle. This file -** control interprets its argument as a pointer to a native file handle and -** writes the resulting value there. -** **
  • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] ** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This ** opcode causes the xFileControl method to swap the file handle with the one @@ -1257,14 +1170,6 @@ struct sqlite3_io_methods { ** circumstances in order to fix a problem with priority inversion. ** Applications should not use this file-control. ** -**
  • [[SQLITE_FCNTL_ZIPVFS]] -** The [SQLITE_FCNTL_ZIPVFS] opcode is implemented by zipvfs only. All other -** VFS should return SQLITE_NOTFOUND for this opcode. -** -**
  • [[SQLITE_FCNTL_RBU]] -** The [SQLITE_FCNTL_RBU] opcode is implemented by the special VFS used by -** the RBU extension only. All other VFS should return SQLITE_NOTFOUND for -** this opcode. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1290,11 +1195,6 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_COMMIT_PHASETWO 22 #define SQLITE_FCNTL_WIN32_SET_HANDLE 23 #define SQLITE_FCNTL_WAL_BLOCK 24 -#define SQLITE_FCNTL_ZIPVFS 25 -#define SQLITE_FCNTL_RBU 26 -#define SQLITE_FCNTL_VFS_POINTER 27 -#define SQLITE_FCNTL_JOURNAL_POINTER 28 -#define SQLITE_FCNTL_WIN32_GET_HANDLE 29 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1314,16 +1214,6 @@ struct sqlite3_io_methods { */ typedef struct sqlite3_mutex sqlite3_mutex; -/* -** CAPI3REF: Loadable Extension Thunk -** -** A pointer to the opaque sqlite3_api_routines structure is passed as -** the third parameter to entry points of [loadable extensions]. This -** structure must be typedefed in order to work around compiler warnings -** on some platforms. -*/ -typedef struct sqlite3_api_routines sqlite3_api_routines; - /* ** CAPI3REF: OS Interface Object ** @@ -1517,7 +1407,7 @@ struct sqlite3_vfs { const char *(*xNextSystemCall)(sqlite3_vfs*, const char *zName); /* ** The methods above are in versions 1 through 3 of the sqlite_vfs object. - ** New fields may be appended in future versions. The iVersion + ** New fields may be appended in figure versions. The iVersion ** value will increment whenever this happens. */ }; @@ -1659,10 +1549,10 @@ struct sqlite3_vfs { ** must return [SQLITE_OK] on success and some other [error code] upon ** failure. */ -SQLITE_API int sqlite3_initialize(void); -SQLITE_API int sqlite3_shutdown(void); -SQLITE_API int sqlite3_os_init(void); -SQLITE_API int sqlite3_os_end(void); +SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void); +SQLITE_API int SQLITE_STDCALL sqlite3_shutdown(void); +SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void); +SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void); /* ** CAPI3REF: Configuring The SQLite Library @@ -1673,11 +1563,9 @@ SQLITE_API int sqlite3_os_end(void); ** applications and so this routine is usually not necessary. It is ** provided to support rare applications with unusual needs. ** -** The sqlite3_config() interface is not threadsafe. The application -** must ensure that no other SQLite interfaces are invoked by other -** threads while sqlite3_config() is running. -** -** The sqlite3_config() interface +** The sqlite3_config() interface is not threadsafe. The application +** must insure that no other SQLite interfaces are invoked by other +** threads while sqlite3_config() is running. Furthermore, sqlite3_config() ** may only be invoked prior to library initialization using ** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before @@ -1695,7 +1583,7 @@ SQLITE_API int sqlite3_os_end(void); ** ^If the option is unknown or SQLite is unable to set the option ** then this routine returns a non-zero [error code]. */ -SQLITE_API int sqlite3_config(int, ...); +SQLITE_API int SQLITE_CDECL sqlite3_config(int, ...); /* ** CAPI3REF: Configure database connections @@ -1714,7 +1602,7 @@ SQLITE_API int sqlite3_config(int, ...); ** ^Calls to sqlite3_db_config() return SQLITE_OK if and only if ** the call is considered successful. */ -SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); +SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3*, int op, ...); /* ** CAPI3REF: Memory Allocation Routines @@ -1904,34 +1792,29 @@ struct sqlite3_mem_methods { ** ** ** [[SQLITE_CONFIG_PAGECACHE]]
    SQLITE_CONFIG_PAGECACHE
    -**
    ^The SQLITE_CONFIG_PAGECACHE option specifies a memory pool +**
    ^The SQLITE_CONFIG_PAGECACHE option specifies a static memory buffer ** that SQLite can use for the database page cache with the default page ** cache implementation. -** This configuration option is a no-op if an application-define page -** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2]. +** This configuration should not be used if an application-define page +** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] +** configuration option. ** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to -** 8-byte aligned memory (pMem), the size of each page cache line (sz), -** and the number of cache lines (N). +** 8-byte aligned +** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page ** (a power of two between 512 and 65536) plus some extra bytes for each ** page header. ^The number of extra bytes needed by the page header -** can be determined using [SQLITE_CONFIG_PCACHE_HDRSZ]. +** can be determined using the [SQLITE_CONFIG_PCACHE_HDRSZ] option +** to [sqlite3_config()]. ** ^It is harmless, apart from the wasted memory, -** for the sz parameter to be larger than necessary. The pMem -** argument must be either a NULL pointer or a pointer to an 8-byte -** aligned block of memory of at least sz*N bytes, otherwise -** subsequent behavior is undefined. -** ^When pMem is not NULL, SQLite will strive to use the memory provided -** to satisfy page cache needs, falling back to [sqlite3_malloc()] if -** a page cache line is larger than sz bytes or if all of the pMem buffer -** is exhausted. -** ^If pMem is NULL and N is non-zero, then each database connection -** does an initial bulk allocation for page cache memory -** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or -** of -1024*N bytes if N is negative, . ^If additional -** page cache memory is needed beyond what is provided by the initial -** allocation, then SQLite goes to [sqlite3_malloc()] separately for each -** additional cache line.
    +** for the sz parameter to be larger than necessary. The first +** argument should pointer to an 8-byte aligned block of memory that +** is at least sz*N bytes of memory, otherwise subsequent behavior is +** undefined. +** ^SQLite will use the memory provided by the first argument to satisfy its +** memory needs for the first N pages that it adds to cache. ^If additional +** page cache memory is needed beyond what is provided by this option, then +** SQLite goes to [sqlite3_malloc()] for the additional storage space. ** ** [[SQLITE_CONFIG_HEAP]]
    SQLITE_CONFIG_HEAP
    **
    ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer @@ -2109,20 +1992,6 @@ struct sqlite3_mem_methods { ** is enabled (using the [PRAGMA threads] command) and the amount of content ** to be sorted exceeds the page size times the minimum of the ** [PRAGMA cache_size] setting and this value. -** -** [[SQLITE_CONFIG_STMTJRNL_SPILL]] -**
    SQLITE_CONFIG_STMTJRNL_SPILL -**
    ^The SQLITE_CONFIG_STMTJRNL_SPILL option takes a single parameter which -** becomes the [statement journal] spill-to-disk threshold. -** [Statement journals] are held in memory until their size (in bytes) -** exceeds this threshold, at which point they are written to disk. -** Or if the threshold is -1, statement journals are always held -** exclusively in memory. -** Since many statement journals never become large, setting the spill -** threshold to a value such as 64KiB can greatly reduce the amount of -** I/O required to support statement rollback. -** The default value for this setting is controlled by the -** [SQLITE_STMTJRNL_SPILL] compile-time option. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -2150,7 +2019,6 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ #define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ -#define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2208,53 +2076,11 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the trigger setting is not reported back.
    ** -**
    SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
    -**
    ^This option is used to enable or disable the two-argument -** version of the [fts3_tokenizer()] function which is part of the -** [FTS3] full-text search engine extension. -** There should be two additional arguments. -** The first argument is an integer which is 0 to disable fts3_tokenizer() or -** positive to enable fts3_tokenizer() or negative to leave the setting -** unchanged. -** The second parameter is a pointer to an integer into which -** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled -** following this call. The second parameter may be a NULL pointer, in -** which case the new setting is not reported back.
    -** -**
    SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
    -**
    ^This option is used to enable or disable the [sqlite3_load_extension()] -** interface independently of the [load_extension()] SQL function. -** The [sqlite3_enable_load_extension()] API enables or disables both the -** C-API [sqlite3_load_extension()] and the SQL function [load_extension()]. -** There should be two additional arguments. -** When the first argument to this interface is 1, then only the C-API is -** enabled and the SQL function remains disabled. If the first argument to -** this interface is 0, then both the C-API and the SQL function are disabled. -** If the first argument is -1, then no changes are made to state of either the -** C-API or the SQL function. -** The second parameter is a pointer to an integer into which -** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface -** is disabled or enabled following this call. The second parameter may -** be a NULL pointer, in which case the new setting is not reported back. -**
    -** -**
    SQLITE_DBCONFIG_MAINDBNAME
    -**
    ^This option is used to change the name of the "main" database -** schema. ^The sole argument is a pointer to a constant UTF8 string -** which will become the new schema name in place of "main". ^SQLite -** does not make a copy of the new main schema name string, so the application -** must ensure that the argument passed into this DBCONFIG option is unchanged -** until after the database connection closes. -**
    -** ** */ -#define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ -#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ -#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ -#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ -#define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */ -#define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */ +#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ +#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ /* @@ -2265,7 +2091,7 @@ struct sqlite3_mem_methods { ** [extended result codes] feature of SQLite. ^The extended result ** codes are disabled by default for historical compatibility. */ -SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); +SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3*, int onoff); /* ** CAPI3REF: Last Insert Rowid @@ -2317,7 +2143,7 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); ** unpredictable and might not equal either the old or the new ** last insert [rowid]. */ -SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_last_insert_rowid(sqlite3*); /* ** CAPI3REF: Count The Number Of Rows Modified @@ -2370,7 +2196,7 @@ SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); ** while [sqlite3_changes()] is running then the value returned ** is unpredictable and not meaningful. */ -SQLITE_API int sqlite3_changes(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_changes(sqlite3*); /* ** CAPI3REF: Total Number Of Rows Modified @@ -2394,7 +2220,7 @@ SQLITE_API int sqlite3_changes(sqlite3*); ** while [sqlite3_total_changes()] is running then the value ** returned is unpredictable and not meaningful. */ -SQLITE_API int sqlite3_total_changes(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3*); /* ** CAPI3REF: Interrupt A Long-Running Query @@ -2434,7 +2260,7 @@ SQLITE_API int sqlite3_total_changes(sqlite3*); ** If the database connection closes while [sqlite3_interrupt()] ** is running then bad things will likely happen. */ -SQLITE_API void sqlite3_interrupt(sqlite3*); +SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3*); /* ** CAPI3REF: Determine If An SQL Statement Is Complete @@ -2469,8 +2295,8 @@ SQLITE_API void sqlite3_interrupt(sqlite3*); ** The input to [sqlite3_complete16()] must be a zero-terminated ** UTF-16 string in native byte order. */ -SQLITE_API int sqlite3_complete(const char *sql); -SQLITE_API int sqlite3_complete16(const void *sql); +SQLITE_API int SQLITE_STDCALL sqlite3_complete(const char *sql); +SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors @@ -2531,7 +2357,7 @@ SQLITE_API int sqlite3_complete16(const void *sql); ** A busy handler must not close the database connection ** or [prepared statement] that invoked the busy handler. */ -SQLITE_API int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*); +SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); /* ** CAPI3REF: Set A Busy Timeout @@ -2554,7 +2380,7 @@ SQLITE_API int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*); ** ** See also: [PRAGMA busy_timeout] */ -SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); +SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3*, int ms); /* ** CAPI3REF: Convenience Routines For Running Queries @@ -2629,7 +2455,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** reflected in subsequent calls to [sqlite3_errcode()] or ** [sqlite3_errmsg()]. */ -SQLITE_API int sqlite3_get_table( +SQLITE_API int SQLITE_STDCALL sqlite3_get_table( sqlite3 *db, /* An open database */ const char *zSql, /* SQL to be evaluated */ char ***pazResult, /* Results of the query */ @@ -2637,7 +2463,7 @@ SQLITE_API int sqlite3_get_table( int *pnColumn, /* Number of result columns written here */ char **pzErrmsg /* Error msg written here */ ); -SQLITE_API void sqlite3_free_table(char **result); +SQLITE_API void SQLITE_STDCALL sqlite3_free_table(char **result); /* ** CAPI3REF: Formatted String Printing Functions @@ -2743,10 +2569,10 @@ SQLITE_API void sqlite3_free_table(char **result); ** addition that after the string has been read and copied into ** the result, [sqlite3_free()] is called on the input string.)^ */ -SQLITE_API char *sqlite3_mprintf(const char*,...); -SQLITE_API char *sqlite3_vmprintf(const char*, va_list); -SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); -SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); +SQLITE_API char *SQLITE_CDECL sqlite3_mprintf(const char*,...); +SQLITE_API char *SQLITE_STDCALL sqlite3_vmprintf(const char*, va_list); +SQLITE_API char *SQLITE_CDECL sqlite3_snprintf(int,char*,const char*, ...); +SQLITE_API char *SQLITE_STDCALL sqlite3_vsnprintf(int,char*,const char*, va_list); /* ** CAPI3REF: Memory Allocation Subsystem @@ -2836,12 +2662,12 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** a block of memory after it has been released using ** [sqlite3_free()] or [sqlite3_realloc()]. */ -SQLITE_API void *sqlite3_malloc(int); -SQLITE_API void *sqlite3_malloc64(sqlite3_uint64); -SQLITE_API void *sqlite3_realloc(void*, int); -SQLITE_API void *sqlite3_realloc64(void*, sqlite3_uint64); -SQLITE_API void sqlite3_free(void*); -SQLITE_API sqlite3_uint64 sqlite3_msize(void*); +SQLITE_API void *SQLITE_STDCALL sqlite3_malloc(int); +SQLITE_API void *SQLITE_STDCALL sqlite3_malloc64(sqlite3_uint64); +SQLITE_API void *SQLITE_STDCALL sqlite3_realloc(void*, int); +SQLITE_API void *SQLITE_STDCALL sqlite3_realloc64(void*, sqlite3_uint64); +SQLITE_API void SQLITE_STDCALL sqlite3_free(void*); +SQLITE_API sqlite3_uint64 SQLITE_STDCALL sqlite3_msize(void*); /* ** CAPI3REF: Memory Allocator Statistics @@ -2866,8 +2692,8 @@ SQLITE_API sqlite3_uint64 sqlite3_msize(void*); ** by [sqlite3_memory_highwater(1)] is the high-water mark ** prior to the reset. */ -SQLITE_API sqlite3_int64 sqlite3_memory_used(void); -SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_used(void); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_highwater(int resetFlag); /* ** CAPI3REF: Pseudo-Random Number Generator @@ -2890,7 +2716,7 @@ SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ -SQLITE_API void sqlite3_randomness(int N, void *P); +SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *P); /* ** CAPI3REF: Compile-Time Authorization Callbacks @@ -2973,7 +2799,7 @@ SQLITE_API void sqlite3_randomness(int N, void *P); ** as stated in the previous paragraph, sqlite3_step() invokes ** sqlite3_prepare_v2() to reprepare a statement after a schema change. */ -SQLITE_API int sqlite3_set_authorizer( +SQLITE_API int SQLITE_STDCALL sqlite3_set_authorizer( sqlite3*, int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pUserData @@ -3053,9 +2879,6 @@ SQLITE_API int sqlite3_set_authorizer( ** CAPI3REF: Tracing And Profiling Functions ** METHOD: sqlite3 ** -** These routines are deprecated. Use the [sqlite3_trace_v2()] interface -** instead of the routines described here. -** ** These routines register callback functions that can be used for ** tracing and profiling the execution of SQL statements. ** @@ -3081,104 +2904,10 @@ SQLITE_API int sqlite3_set_authorizer( ** sqlite3_profile() function is considered experimental and is ** subject to change in future versions of SQLite. */ -SQLITE_API SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*, - void(*xTrace)(void*,const char*), void*); -SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, +SQLITE_API void *SQLITE_STDCALL sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); +SQLITE_API SQLITE_EXPERIMENTAL void *SQLITE_STDCALL sqlite3_profile(sqlite3*, void(*xProfile)(void*,const char*,sqlite3_uint64), void*); -/* -** CAPI3REF: SQL Trace Event Codes -** KEYWORDS: SQLITE_TRACE -** -** These constants identify classes of events that can be monitored -** using the [sqlite3_trace_v2()] tracing logic. The third argument -** to [sqlite3_trace_v2()] is an OR-ed combination of one or more of -** the following constants. ^The first argument to the trace callback -** is one of the following constants. -** -** New tracing constants may be added in future releases. -** -** ^A trace callback has four arguments: xCallback(T,C,P,X). -** ^The T argument is one of the integer type codes above. -** ^The C argument is a copy of the context pointer passed in as the -** fourth argument to [sqlite3_trace_v2()]. -** The P and X arguments are pointers whose meanings depend on T. -** -**
    -** [[SQLITE_TRACE_STMT]]
    SQLITE_TRACE_STMT
    -**
    ^An SQLITE_TRACE_STMT callback is invoked when a prepared statement -** first begins running and possibly at other times during the -** execution of the prepared statement, such as at the start of each -** trigger subprogram. ^The P argument is a pointer to the -** [prepared statement]. ^The X argument is a pointer to a string which -** is the unexpanded SQL text of the prepared statement or an SQL comment -** that indicates the invocation of a trigger. ^The callback can compute -** the same text that would have been returned by the legacy [sqlite3_trace()] -** interface by using the X argument when X begins with "--" and invoking -** [sqlite3_expanded_sql(P)] otherwise. -** -** [[SQLITE_TRACE_PROFILE]]
    SQLITE_TRACE_PROFILE
    -**
    ^An SQLITE_TRACE_PROFILE callback provides approximately the same -** information as is provided by the [sqlite3_profile()] callback. -** ^The P argument is a pointer to the [prepared statement] and the -** X argument points to a 64-bit integer which is the estimated of -** the number of nanosecond that the prepared statement took to run. -** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes. -** -** [[SQLITE_TRACE_ROW]]
    SQLITE_TRACE_ROW
    -**
    ^An SQLITE_TRACE_ROW callback is invoked whenever a prepared -** statement generates a single row of result. -** ^The P argument is a pointer to the [prepared statement] and the -** X argument is unused. -** -** [[SQLITE_TRACE_CLOSE]]
    SQLITE_TRACE_CLOSE
    -**
    ^An SQLITE_TRACE_CLOSE callback is invoked when a database -** connection closes. -** ^The P argument is a pointer to the [database connection] object -** and the X argument is unused. -**
    -*/ -#define SQLITE_TRACE_STMT 0x01 -#define SQLITE_TRACE_PROFILE 0x02 -#define SQLITE_TRACE_ROW 0x04 -#define SQLITE_TRACE_CLOSE 0x08 - -/* -** CAPI3REF: SQL Trace Hook -** METHOD: sqlite3 -** -** ^The sqlite3_trace_v2(D,M,X,P) interface registers a trace callback -** function X against [database connection] D, using property mask M -** and context pointer P. ^If the X callback is -** NULL or if the M mask is zero, then tracing is disabled. The -** M argument should be the bitwise OR-ed combination of -** zero or more [SQLITE_TRACE] constants. -** -** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides -** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2(). -** -** ^The X callback is invoked whenever any of the events identified by -** mask M occur. ^The integer return value from the callback is currently -** ignored, though this may change in future releases. Callback -** implementations should return zero to ensure future compatibility. -** -** ^A trace callback is invoked with four arguments: callback(T,C,P,X). -** ^The T argument is one of the [SQLITE_TRACE] -** constants to indicate why the callback was invoked. -** ^The C argument is a copy of the context pointer. -** The P and X arguments are pointers whose meanings depend on T. -** -** The sqlite3_trace_v2() interface is intended to replace the legacy -** interfaces [sqlite3_trace()] and [sqlite3_profile()], both of which -** are deprecated. -*/ -SQLITE_API int sqlite3_trace_v2( - sqlite3*, - unsigned uMask, - int(*xCallback)(unsigned,void*,void*,void*), - void *pCtx -); - /* ** CAPI3REF: Query Progress Callbacks ** METHOD: sqlite3 @@ -3211,7 +2940,7 @@ SQLITE_API int sqlite3_trace_v2( ** database connections for the meaning of "modify" in this paragraph. ** */ -SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); +SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); /* ** CAPI3REF: Opening A New Database Connection @@ -3440,15 +3169,15 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ** See also: [sqlite3_temp_directory] */ -SQLITE_API int sqlite3_open( +SQLITE_API int SQLITE_STDCALL sqlite3_open( const char *filename, /* Database filename (UTF-8) */ sqlite3 **ppDb /* OUT: SQLite db handle */ ); -SQLITE_API int sqlite3_open16( +SQLITE_API int SQLITE_STDCALL sqlite3_open16( const void *filename, /* Database filename (UTF-16) */ sqlite3 **ppDb /* OUT: SQLite db handle */ ); -SQLITE_API int sqlite3_open_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_open_v2( const char *filename, /* Database filename (UTF-8) */ sqlite3 **ppDb, /* OUT: SQLite db handle */ int flags, /* Flags */ @@ -3494,9 +3223,9 @@ SQLITE_API int sqlite3_open_v2( ** VFS method, then the behavior of this routine is undefined and probably ** undesirable. */ -SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); -SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); -SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); +SQLITE_API const char *SQLITE_STDCALL sqlite3_uri_parameter(const char *zFilename, const char *zParam); +SQLITE_API int SQLITE_STDCALL sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_uri_int64(const char*, const char*, sqlite3_int64); /* @@ -3540,11 +3269,11 @@ SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int ** was invoked incorrectly by the application. In that case, the ** error code and message may or may not be set. */ -SQLITE_API int sqlite3_errcode(sqlite3 *db); -SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); -SQLITE_API const char *sqlite3_errmsg(sqlite3*); -SQLITE_API const void *sqlite3_errmsg16(sqlite3*); -SQLITE_API const char *sqlite3_errstr(int); +SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db); +SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db); +SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int); /* ** CAPI3REF: Prepared Statement Object @@ -3612,7 +3341,7 @@ typedef struct sqlite3_stmt sqlite3_stmt; ** ** New run-time limit categories may be added in future releases. */ -SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); +SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3*, int id, int newVal); /* ** CAPI3REF: Run-Time Limit Categories @@ -3764,28 +3493,28 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** */ -SQLITE_API int sqlite3_prepare( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare( sqlite3 *db, /* Database handle */ const char *zSql, /* SQL statement, UTF-8 encoded */ int nByte, /* Maximum length of zSql in bytes. */ sqlite3_stmt **ppStmt, /* OUT: Statement handle */ const char **pzTail /* OUT: Pointer to unused portion of zSql */ ); -SQLITE_API int sqlite3_prepare_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare_v2( sqlite3 *db, /* Database handle */ const char *zSql, /* SQL statement, UTF-8 encoded */ int nByte, /* Maximum length of zSql in bytes. */ sqlite3_stmt **ppStmt, /* OUT: Statement handle */ const char **pzTail /* OUT: Pointer to unused portion of zSql */ ); -SQLITE_API int sqlite3_prepare16( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare16( sqlite3 *db, /* Database handle */ const void *zSql, /* SQL statement, UTF-16 encoded */ int nByte, /* Maximum length of zSql in bytes. */ sqlite3_stmt **ppStmt, /* OUT: Statement handle */ const void **pzTail /* OUT: Pointer to unused portion of zSql */ ); -SQLITE_API int sqlite3_prepare16_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare16_v2( sqlite3 *db, /* Database handle */ const void *zSql, /* SQL statement, UTF-16 encoded */ int nByte, /* Maximum length of zSql in bytes. */ @@ -3797,35 +3526,11 @@ SQLITE_API int sqlite3_prepare16_v2( ** CAPI3REF: Retrieving Statement SQL ** METHOD: sqlite3_stmt ** -** ^The sqlite3_sql(P) interface returns a pointer to a copy of the UTF-8 -** SQL text used to create [prepared statement] P if P was -** created by either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. -** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8 -** string containing the SQL text of prepared statement P with -** [bound parameters] expanded. -** -** ^(For example, if a prepared statement is created using the SQL -** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345 -** and parameter :xyz is unbound, then sqlite3_sql() will return -** the original string, "SELECT $abc,:xyz" but sqlite3_expanded_sql() -** will return "SELECT 2345,NULL".)^ -** -** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory -** is available to hold the result, or if the result would exceed the -** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. -** -** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of -** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time -** option causes sqlite3_expanded_sql() to always return NULL. -** -** ^The string returned by sqlite3_sql(P) is managed by SQLite and is -** automatically freed when the prepared statement is finalized. -** ^The string returned by sqlite3_expanded_sql(P), on the other hand, -** is obtained from [sqlite3_malloc()] and must be free by the application -** by passing it to [sqlite3_free()]. +** ^This interface can be used to retrieve a saved copy of the original +** SQL text used to create a [prepared statement] if that statement was +** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. */ -SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); -SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt); +SQLITE_API const char *SQLITE_STDCALL sqlite3_sql(sqlite3_stmt *pStmt); /* ** CAPI3REF: Determine If An SQL Statement Writes The Database @@ -3857,7 +3562,7 @@ SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt); ** change the configuration of a database connection, they do not make ** changes to the content of the database files on disk. */ -SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_readonly(sqlite3_stmt *pStmt); /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset @@ -3865,8 +3570,7 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); ** ** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the ** [prepared statement] S has been stepped at least once using -** [sqlite3_step(S)] but has neither run to completion (returned -** [SQLITE_DONE] from [sqlite3_step(S)]) nor +** [sqlite3_step(S)] but has not run to completion and/or has not ** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) ** interface returns false if S is a NULL pointer. If S is not a ** NULL pointer and is not a pointer to a valid [prepared statement] @@ -3878,7 +3582,7 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); ** for example, in diagnostic routines to search for prepared ** statements that are holding a transaction open. */ -SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt*); /* ** CAPI3REF: Dynamically Typed Value Object @@ -3893,9 +3597,7 @@ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); ** Some interfaces require a protected sqlite3_value. Other interfaces ** will accept either a protected or an unprotected sqlite3_value. ** Every interface that accepts sqlite3_value arguments specifies -** whether or not it requires a protected sqlite3_value. The -** [sqlite3_value_dup()] interface can be used to construct a new -** protected sqlite3_value from an unprotected sqlite3_value. +** whether or not it requires a protected sqlite3_value. ** ** The terms "protected" and "unprotected" refer to whether or not ** a mutex is held. An internal mutex is held for a protected @@ -4042,20 +3744,19 @@ typedef struct sqlite3_context sqlite3_context; ** See also: [sqlite3_bind_parameter_count()], ** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. */ -SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); -SQLITE_API int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, +SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, void(*)(void*)); -SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double); -SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int); -SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); -SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int); -SQLITE_API int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); -SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); -SQLITE_API int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, +SQLITE_API int SQLITE_STDCALL sqlite3_bind_double(sqlite3_stmt*, int, double); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_int(sqlite3_stmt*, int, int); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_null(sqlite3_stmt*, int); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, void(*)(void*), unsigned char encoding); -SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); -SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); -SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); /* ** CAPI3REF: Number Of SQL Parameters @@ -4076,7 +3777,7 @@ SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64); ** [sqlite3_bind_parameter_name()], and ** [sqlite3_bind_parameter_index()]. */ -SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_count(sqlite3_stmt*); /* ** CAPI3REF: Name Of A Host Parameter @@ -4104,7 +3805,7 @@ SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*); ** [sqlite3_bind_parameter_count()], and ** [sqlite3_bind_parameter_index()]. */ -SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_bind_parameter_name(sqlite3_stmt*, int); /* ** CAPI3REF: Index Of A Parameter With A Given Name @@ -4119,9 +3820,9 @@ SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); ** ** See also: [sqlite3_bind_blob|sqlite3_bind()], ** [sqlite3_bind_parameter_count()], and -** [sqlite3_bind_parameter_name()]. +** [sqlite3_bind_parameter_index()]. */ -SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); +SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); /* ** CAPI3REF: Reset All Bindings On A Prepared Statement @@ -4131,7 +3832,7 @@ SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); ** the [sqlite3_bind_blob | bindings] on a [prepared statement]. ** ^Use this routine to reset all host parameters to NULL. */ -SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_clear_bindings(sqlite3_stmt*); /* ** CAPI3REF: Number Of Columns In A Result Set @@ -4143,7 +3844,7 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); ** ** See also: [sqlite3_data_count()] */ -SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_column_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Column Names In A Result Set @@ -4172,8 +3873,8 @@ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); ** then the name of the column is unspecified and may change from ** one release of SQLite to the next. */ -SQLITE_API const char *sqlite3_column_name(sqlite3_stmt*, int N); -SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_name(sqlite3_stmt*, int N); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_name16(sqlite3_stmt*, int N); /* ** CAPI3REF: Source Of Data In A Query Result @@ -4221,12 +3922,12 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); ** for the same [prepared statement] and result column ** at the same time then the results are undefined. */ -SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt*,int); -SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt*,int); -SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_database_name(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_database_name16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_table_name(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_table_name16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_origin_name(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_origin_name16(sqlite3_stmt*,int); /* ** CAPI3REF: Declared Datatype Of A Query Result @@ -4258,8 +3959,8 @@ SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); ** is associated with individual values, not with the containers ** used to hold those values. */ -SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt*,int); -SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_decltype(sqlite3_stmt*,int); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_decltype16(sqlite3_stmt*,int); /* ** CAPI3REF: Evaluate An SQL Statement @@ -4320,8 +4021,7 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** other than [SQLITE_ROW] before any subsequent invocation of ** sqlite3_step(). Failure to reset the prepared statement using ** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from -** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1], -** sqlite3_step() began +** sqlite3_step(). But after version 3.6.23.1, sqlite3_step() began ** calling [sqlite3_reset()] automatically in this circumstance rather ** than returning [SQLITE_MISUSE]. This is not considered a compatibility ** break because any application that ever receives an SQLITE_MISUSE error @@ -4340,7 +4040,7 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** then the more specific [error codes] are returned directly ** by sqlite3_step(). The use of the "v2" interface is recommended. */ -SQLITE_API int sqlite3_step(sqlite3_stmt*); +SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt*); /* ** CAPI3REF: Number of columns in a result set @@ -4361,7 +4061,7 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); ** ** See also: [sqlite3_column_count()] */ -SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Fundamental Datatypes @@ -4400,6 +4100,8 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** KEYWORDS: {column access functions} ** METHOD: sqlite3_stmt ** +** These routines form the "result set" interface. +** ** ^These routines return information about a single column of the current ** result row of a query. ^In every case the first argument is a pointer ** to the [prepared statement] that is being evaluated (the [sqlite3_stmt*] @@ -4459,14 +4161,13 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** even empty strings, are always zero-terminated. ^The return ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** -** Warning: ^The object returned by [sqlite3_column_value()] is an -** [unprotected sqlite3_value] object. In a multithreaded environment, -** an unprotected sqlite3_value object may only be used safely with -** [sqlite3_bind_value()] and [sqlite3_result_value()]. +** ^The object returned by [sqlite3_column_value()] is an +** [unprotected sqlite3_value] object. An unprotected sqlite3_value object +** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()]. ** If the [unprotected sqlite3_value] object returned by ** [sqlite3_column_value()] is used in any other way, including calls ** to routines like [sqlite3_value_int()], [sqlite3_value_text()], -** or [sqlite3_value_bytes()], the behavior is not threadsafe. +** or [sqlite3_value_bytes()], then the behavior is undefined. ** ** These routines attempt to convert the value where appropriate. ^For ** example, if the internal representation is FLOAT and a text result @@ -4497,6 +4198,12 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** **
    )^ ** +** The table above makes reference to standard C library functions atoi() +** and atof(). SQLite does not really use these functions. It has its +** own equivalent internal routines. The atoi() and atof() names are +** used in the table for brevity and because they are familiar to most +** C programmers. +** ** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. @@ -4521,7 +4228,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** The safest policy is to invoke these routines +** The safest and easiest to remember policy is to invoke these routines ** in one of the following ways: ** **
      @@ -4541,7 +4248,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** ^The pointers returned are valid until a type conversion occurs as ** described above, or until [sqlite3_step()] or [sqlite3_reset()] or ** [sqlite3_finalize()] is called. ^The memory space used to hold strings -** and BLOBs is freed automatically. Do not pass the pointers returned +** and BLOBs is freed automatically. Do not pass the pointers returned ** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** @@ -4551,16 +4258,16 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** pointer. Subsequent calls to [sqlite3_errcode()] will return ** [SQLITE_NOMEM].)^ */ -SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_bytes(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt*, int iCol); -SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_int(sqlite3_stmt*, int iCol); -SQLITE_API sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); -SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); -SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt*, int iCol); -SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); -SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_blob(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes16(sqlite3_stmt*, int iCol); +SQLITE_API double SQLITE_STDCALL sqlite3_column_double(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_int(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_column_int64(sqlite3_stmt*, int iCol); +SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_column_text(sqlite3_stmt*, int iCol); +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_text16(sqlite3_stmt*, int iCol); +SQLITE_API int SQLITE_STDCALL sqlite3_column_type(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_column_value(sqlite3_stmt*, int iCol); /* ** CAPI3REF: Destroy A Prepared Statement Object @@ -4588,7 +4295,7 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); ** statement after it has been finalized can result in undefined and ** undesirable behavior such as segfaults and heap corruption. */ -SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_finalize(sqlite3_stmt *pStmt); /* ** CAPI3REF: Reset A Prepared Statement Object @@ -4615,7 +4322,7 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); ** ^The [sqlite3_reset(S)] interface does not change the values ** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. */ -SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); +SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt); /* ** CAPI3REF: Create Or Redefine SQL Functions @@ -4715,7 +4422,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** close the database connection nor finalize or reset the prepared ** statement in which the function is running. */ -SQLITE_API int sqlite3_create_function( +SQLITE_API int SQLITE_STDCALL sqlite3_create_function( sqlite3 *db, const char *zFunctionName, int nArg, @@ -4725,7 +4432,7 @@ SQLITE_API int sqlite3_create_function( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); -SQLITE_API int sqlite3_create_function16( +SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( sqlite3 *db, const void *zFunctionName, int nArg, @@ -4735,7 +4442,7 @@ SQLITE_API int sqlite3_create_function16( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); -SQLITE_API int sqlite3_create_function_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( sqlite3 *db, const char *zFunctionName, int nArg, @@ -4781,22 +4488,22 @@ SQLITE_API int sqlite3_create_function_v2( ** these functions, we will not explain what they do. */ #ifndef SQLITE_OMIT_DEPRECATED -SQLITE_API SQLITE_DEPRECATED int sqlite3_aggregate_count(sqlite3_context*); -SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); -SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); -SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); -SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); -SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_aggregate_count(sqlite3_context*); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_expired(sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_global_recover(void); +SQLITE_API SQLITE_DEPRECATED void SQLITE_STDCALL sqlite3_thread_cleanup(void); +SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), void*,sqlite3_int64); #endif /* -** CAPI3REF: Obtaining SQL Values +** CAPI3REF: Obtaining SQL Function Parameter Values ** METHOD: sqlite3_value ** ** The C-language implementation of SQL functions and aggregates uses ** this set of interface routines to access the parameter values on -** the function or aggregate. +** the function or aggregate. ** ** The xFunc (for scalar functions) or xStep (for aggregates) parameters ** to [sqlite3_create_function()] and [sqlite3_create_function16()] @@ -4836,51 +4543,18 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** These routines must be called from the same thread as ** the SQL function that supplied the [sqlite3_value*] parameters. */ -SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); -SQLITE_API int sqlite3_value_bytes(sqlite3_value*); -SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); -SQLITE_API double sqlite3_value_double(sqlite3_value*); -SQLITE_API int sqlite3_value_int(sqlite3_value*); -SQLITE_API sqlite3_int64 sqlite3_value_int64(sqlite3_value*); -SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value*); -SQLITE_API const void *sqlite3_value_text16(sqlite3_value*); -SQLITE_API const void *sqlite3_value_text16le(sqlite3_value*); -SQLITE_API const void *sqlite3_value_text16be(sqlite3_value*); -SQLITE_API int sqlite3_value_type(sqlite3_value*); -SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); - -/* -** CAPI3REF: Finding The Subtype Of SQL Values -** METHOD: sqlite3_value -** -** The sqlite3_value_subtype(V) function returns the subtype for -** an [application-defined SQL function] argument V. The subtype -** information can be used to pass a limited amount of context from -** one SQL function to another. Use the [sqlite3_result_subtype()] -** routine to set the subtype for the return value of an SQL function. -** -** SQLite makes no use of subtype itself. It merely passes the subtype -** from the result of one [application-defined SQL function] into the -** input of another. -*/ -SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); - -/* -** CAPI3REF: Copy And Free SQL Values -** METHOD: sqlite3_value -** -** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value] -** object D and returns a pointer to that copy. ^The [sqlite3_value] returned -** is a [protected sqlite3_value] object even if the input is not. -** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a -** memory allocation fails. -** -** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object -** previously obtained from [sqlite3_value_dup()]. ^If V is a NULL pointer -** then sqlite3_value_free(V) is a harmless no-op. -*/ -SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value*); -SQLITE_API void sqlite3_value_free(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_blob(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes16(sqlite3_value*); +SQLITE_API double SQLITE_STDCALL sqlite3_value_double(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_int(sqlite3_value*); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_value_int64(sqlite3_value*); +SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_value_text(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16le(sqlite3_value*); +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16be(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_type(sqlite3_value*); +SQLITE_API int SQLITE_STDCALL sqlite3_value_numeric_type(sqlite3_value*); /* ** CAPI3REF: Obtain Aggregate Function Context @@ -4925,7 +4599,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*); ** This routine must be called from the same thread in which ** the aggregate SQL function is running. */ -SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); +SQLITE_API void *SQLITE_STDCALL sqlite3_aggregate_context(sqlite3_context*, int nBytes); /* ** CAPI3REF: User Data For Functions @@ -4940,7 +4614,7 @@ SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); ** This routine must be called from the same thread in which ** the application-defined function is running. */ -SQLITE_API void *sqlite3_user_data(sqlite3_context*); +SQLITE_API void *SQLITE_STDCALL sqlite3_user_data(sqlite3_context*); /* ** CAPI3REF: Database Connection For Functions @@ -4952,7 +4626,7 @@ SQLITE_API void *sqlite3_user_data(sqlite3_context*); ** and [sqlite3_create_function16()] routines that originally ** registered the application defined function. */ -SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_context_db_handle(sqlite3_context*); /* ** CAPI3REF: Function Auxiliary Data @@ -4984,13 +4658,12 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** SQLite will invoke the destructor function X with parameter P exactly ** once, when the metadata is discarded. ** SQLite is free to discard the metadata at any time, including:
        -**
      • ^(when the corresponding function parameter changes)^, or -**
      • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the -** SQL statement)^, or -**
      • ^(when sqlite3_set_auxdata() is invoked again on the same -** parameter)^, or -**
      • ^(during the original sqlite3_set_auxdata() call when a memory -** allocation error occurs.)^
      +**
    • when the corresponding function parameter changes, or +**
    • when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement, or +**
    • when sqlite3_set_auxdata() is invoked again on the same parameter, or +**
    • during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.
    )^ ** ** Note the last bullet in particular. The destructor X in ** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the @@ -5006,8 +4679,8 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** These routines must be called from the same thread in which ** the SQL function is running. */ -SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); -SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); +SQLITE_API void *SQLITE_STDCALL sqlite3_get_auxdata(sqlite3_context*, int N); +SQLITE_API void SQLITE_STDCALL sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); /* @@ -5046,9 +4719,9 @@ typedef void (*sqlite3_destructor_type)(void*); ** to by the second parameter and which is N bytes long where N is the ** third parameter. ** -** ^The sqlite3_result_zeroblob(C,N) and sqlite3_result_zeroblob64(C,N) -** interfaces set the result of the application-defined function to be -** a BLOB containing all zero bytes and N bytes in size. +** ^The sqlite3_result_zeroblob() interfaces set the result of +** the application-defined function to be a BLOB containing all zero +** bytes and N bytes in size, where N is the value of the 2nd parameter. ** ** ^The sqlite3_result_double() interface sets the result from ** an application-defined function to be a floating point value specified @@ -5130,7 +4803,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** from [sqlite3_malloc()] before it returns. ** ** ^The sqlite3_result_value() interface sets the result of -** the application-defined function to be a copy of the +** the application-defined function to be a copy the ** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The ** sqlite3_result_value() interface makes a copy of the [sqlite3_value] ** so that the [sqlite3_value] specified in the parameter may change or @@ -5143,42 +4816,26 @@ typedef void (*sqlite3_destructor_type)(void*); ** than the one containing the application-defined function that received ** the [sqlite3_context] pointer, the results are undefined. */ -SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); -SQLITE_API void sqlite3_result_blob64(sqlite3_context*,const void*, +SQLITE_API void SQLITE_STDCALL sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_blob64(sqlite3_context*,const void*, sqlite3_uint64,void(*)(void*)); -SQLITE_API void sqlite3_result_double(sqlite3_context*, double); -SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); -SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); -SQLITE_API void sqlite3_result_error_toobig(sqlite3_context*); -SQLITE_API void sqlite3_result_error_nomem(sqlite3_context*); -SQLITE_API void sqlite3_result_error_code(sqlite3_context*, int); -SQLITE_API void sqlite3_result_int(sqlite3_context*, int); -SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); -SQLITE_API void sqlite3_result_null(sqlite3_context*); -SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); -SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, +SQLITE_API void SQLITE_STDCALL sqlite3_result_double(sqlite3_context*, double); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error(sqlite3_context*, const char*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error16(sqlite3_context*, const void*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_toobig(sqlite3_context*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_nomem(sqlite3_context*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_code(sqlite3_context*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_int(sqlite3_context*, int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_int64(sqlite3_context*, sqlite3_int64); +SQLITE_API void SQLITE_STDCALL sqlite3_result_null(sqlite3_context*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, void(*)(void*), unsigned char encoding); -SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); -SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); -SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); -SQLITE_API void sqlite3_result_value(sqlite3_context*, sqlite3_value*); -SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); -SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); - - -/* -** CAPI3REF: Setting The Subtype Of An SQL Function -** METHOD: sqlite3_context -** -** The sqlite3_result_subtype(C,T) function causes the subtype of -** the result from the [application-defined SQL function] with -** [sqlite3_context] C to be the value T. Only the lower 8 bits -** of the subtype T are preserved in current versions of SQLite; -** higher order bits are discarded. -** The number of subtype bytes preserved by SQLite might increase -** in future releases of SQLite. -*/ -SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void SQLITE_STDCALL sqlite3_result_value(sqlite3_context*, sqlite3_value*); +SQLITE_API void SQLITE_STDCALL sqlite3_result_zeroblob(sqlite3_context*, int n); /* ** CAPI3REF: Define New Collating Sequences @@ -5260,14 +4917,14 @@ SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); ** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ -SQLITE_API int sqlite3_create_collation( +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation( sqlite3*, const char *zName, int eTextRep, void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); -SQLITE_API int sqlite3_create_collation_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation_v2( sqlite3*, const char *zName, int eTextRep, @@ -5275,7 +4932,7 @@ SQLITE_API int sqlite3_create_collation_v2( int(*xCompare)(void*,int,const void*,int,const void*), void(*xDestroy)(void*) ); -SQLITE_API int sqlite3_create_collation16( +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation16( sqlite3*, const void *zName, int eTextRep, @@ -5310,12 +4967,12 @@ SQLITE_API int sqlite3_create_collation16( ** [sqlite3_create_collation()], [sqlite3_create_collation16()], or ** [sqlite3_create_collation_v2()]. */ -SQLITE_API int sqlite3_collation_needed( +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed( sqlite3*, void*, void(*)(void*,sqlite3*,int eTextRep,const char*) ); -SQLITE_API int sqlite3_collation_needed16( +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed16( sqlite3*, void*, void(*)(void*,sqlite3*,int eTextRep,const void*) @@ -5329,11 +4986,11 @@ SQLITE_API int sqlite3_collation_needed16( ** The code to implement this API is not available in the public release ** of SQLite. */ -SQLITE_API int sqlite3_key( +SQLITE_API int SQLITE_STDCALL sqlite3_key( sqlite3 *db, /* Database to be rekeyed */ const void *pKey, int nKey /* The key */ ); -SQLITE_API int sqlite3_key_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_key_v2( sqlite3 *db, /* Database to be rekeyed */ const char *zDbName, /* Name of the database */ const void *pKey, int nKey /* The key */ @@ -5347,11 +5004,11 @@ SQLITE_API int sqlite3_key_v2( ** The code to implement this API is not available in the public release ** of SQLite. */ -SQLITE_API int sqlite3_rekey( +SQLITE_API int SQLITE_STDCALL sqlite3_rekey( sqlite3 *db, /* Database to be rekeyed */ const void *pKey, int nKey /* The new key */ ); -SQLITE_API int sqlite3_rekey_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_rekey_v2( sqlite3 *db, /* Database to be rekeyed */ const char *zDbName, /* Name of the database */ const void *pKey, int nKey /* The new key */ @@ -5361,7 +5018,7 @@ SQLITE_API int sqlite3_rekey_v2( ** Specify the activation key for a SEE database. Unless ** activated, none of the SEE routines will work. */ -SQLITE_API void sqlite3_activate_see( +SQLITE_API void SQLITE_STDCALL sqlite3_activate_see( const char *zPassPhrase /* Activation phrase */ ); #endif @@ -5371,7 +5028,7 @@ SQLITE_API void sqlite3_activate_see( ** Specify the activation key for a CEROD database. Unless ** activated, none of the CEROD routines will work. */ -SQLITE_API void sqlite3_activate_cerod( +SQLITE_API void SQLITE_STDCALL sqlite3_activate_cerod( const char *zPassPhrase /* Activation phrase */ ); #endif @@ -5393,7 +5050,7 @@ SQLITE_API void sqlite3_activate_cerod( ** all, then the behavior of sqlite3_sleep() may deviate from the description ** in the previous paragraphs. */ -SQLITE_API int sqlite3_sleep(int); +SQLITE_API int SQLITE_STDCALL sqlite3_sleep(int); /* ** CAPI3REF: Name Of The Folder Holding Temporary Files @@ -5512,7 +5169,7 @@ SQLITE_API char *sqlite3_data_directory; ** connection while this routine is running, then the return value ** is undefined. */ -SQLITE_API int sqlite3_get_autocommit(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3*); /* ** CAPI3REF: Find The Database Handle Of A Prepared Statement @@ -5525,7 +5182,7 @@ SQLITE_API int sqlite3_get_autocommit(sqlite3*); ** to the [sqlite3_prepare_v2()] call (or its variants) that was used to ** create the statement in the first place. */ -SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_db_handle(sqlite3_stmt*); /* ** CAPI3REF: Return The Filename For A Database Connection @@ -5542,7 +5199,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** will be an absolute pathname, even if the filename used ** to open the database originally was a URI or relative pathname. */ -SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); +SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Determine if a database is read-only @@ -5552,7 +5209,7 @@ SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); ** of connection D is read-only, 0 if it is read/write, or -1 if N is not ** the name of a database on connection D. */ -SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); +SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Find the next prepared statement @@ -5568,7 +5225,7 @@ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); ** [sqlite3_next_stmt(D,S)] must refer to an open database ** connection and in particular must not be a NULL pointer. */ -SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); +SQLITE_API sqlite3_stmt *SQLITE_STDCALL sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); /* ** CAPI3REF: Commit And Rollback Notification Callbacks @@ -5617,8 +5274,8 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); ** ** See also the [sqlite3_update_hook()] interface. */ -SQLITE_API void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); -SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); +SQLITE_API void *SQLITE_STDCALL sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); +SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); /* ** CAPI3REF: Data Change Notification Callbacks @@ -5627,7 +5284,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ^The sqlite3_update_hook() interface registers a callback function ** with the [database connection] identified by the first argument ** to be invoked whenever a row is updated, inserted or deleted in -** a [rowid table]. +** a rowid table. ** ^Any callback set by a previous call to this function ** for the same database connection is overridden. ** @@ -5666,10 +5323,10 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** on the same [database connection] D, or NULL for ** the first call on D. ** -** See also the [sqlite3_commit_hook()], [sqlite3_rollback_hook()], -** and [sqlite3_preupdate_hook()] interfaces. +** See also the [sqlite3_commit_hook()] and [sqlite3_rollback_hook()] +** interfaces. */ -SQLITE_API void *sqlite3_update_hook( +SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( sqlite3*, void(*)(void *,int ,char const *,char const *,sqlite3_int64), void* @@ -5684,8 +5341,7 @@ SQLITE_API void *sqlite3_update_hook( ** and disabled if the argument is false.)^ ** ** ^Cache sharing is enabled and disabled for an entire process. -** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). -** In prior versions of SQLite, +** This is a change as of SQLite version 3.5.0. In prior versions of SQLite, ** sharing was enabled or disabled for each thread separately. ** ** ^(The cache sharing mode set by this interface effects all subsequent @@ -5710,7 +5366,7 @@ SQLITE_API void *sqlite3_update_hook( ** ** See Also: [SQLite Shared-Cache Mode] */ -SQLITE_API int sqlite3_enable_shared_cache(int); +SQLITE_API int SQLITE_STDCALL sqlite3_enable_shared_cache(int); /* ** CAPI3REF: Attempt To Free Heap Memory @@ -5726,7 +5382,7 @@ SQLITE_API int sqlite3_enable_shared_cache(int); ** ** See also: [sqlite3_db_release_memory()] */ -SQLITE_API int sqlite3_release_memory(int); +SQLITE_API int SQLITE_STDCALL sqlite3_release_memory(int); /* ** CAPI3REF: Free Memory Used By A Database Connection @@ -5740,7 +5396,7 @@ SQLITE_API int sqlite3_release_memory(int); ** ** See also: [sqlite3_release_memory()] */ -SQLITE_API int sqlite3_db_release_memory(sqlite3*); +SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3*); /* ** CAPI3REF: Impose A Limit On Heap Size @@ -5779,8 +5435,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** from the heap. ** )^ ** -** Beginning with SQLite [version 3.7.3] ([dateof:3.7.3]), -** the soft heap limit is enforced +** Beginning with SQLite version 3.7.3, the soft heap limit is enforced ** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] ** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], ** the soft heap limit is enforced on every memory allocation. Without @@ -5793,7 +5448,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** The circumstances under which SQLite will enforce the soft heap limit may ** changes in future releases of SQLite. */ -SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_soft_heap_limit64(sqlite3_int64 N); /* ** CAPI3REF: Deprecated Soft Heap Limit Interface @@ -5804,7 +5459,7 @@ SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); ** only. All new applications should use the ** [sqlite3_soft_heap_limit64()] interface rather than this one. */ -SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); +SQLITE_API SQLITE_DEPRECATED void SQLITE_STDCALL sqlite3_soft_heap_limit(int N); /* @@ -5819,7 +5474,7 @@ SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** column exists. ^The sqlite3_table_column_metadata() interface returns ** SQLITE_ERROR and if the specified column does not exist. ** ^If the column-name parameter to sqlite3_table_column_metadata() is a -** NULL pointer, then this routine simply checks for the existence of the +** NULL pointer, then this routine simply checks for the existance of the ** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it ** does not. ** @@ -5874,7 +5529,7 @@ SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** parsed, if that has not already been done, and returns an error if ** any errors are encountered while loading the schema. */ -SQLITE_API int sqlite3_table_column_metadata( +SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ const char *zDbName, /* Database name or NULL */ const char *zTableName, /* Table name */ @@ -5916,21 +5571,12 @@ SQLITE_API int sqlite3_table_column_metadata( ** should free this memory by calling [sqlite3_free()]. ** ** ^Extension loading must be enabled using -** [sqlite3_enable_load_extension()] or -** [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],1,NULL) -** prior to calling this API, +** [sqlite3_enable_load_extension()] prior to calling this API, ** otherwise an error will be returned. ** -** Security warning: It is recommended that the -** [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method be used to enable only this -** interface. The use of the [sqlite3_enable_load_extension()] interface -** should be avoided. This will keep the SQL function [load_extension()] -** disabled and prevent SQL injections from giving attackers -** access to extension loading capabilities. -** ** See also the [load_extension() SQL function]. */ -SQLITE_API int sqlite3_load_extension( +SQLITE_API int SQLITE_STDCALL sqlite3_load_extension( sqlite3 *db, /* Load the extension into this database connection */ const char *zFile, /* Name of the shared library containing extension */ const char *zProc, /* Entry point. Derived from zFile if 0 */ @@ -5950,19 +5596,8 @@ SQLITE_API int sqlite3_load_extension( ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. -** -** ^This interface enables or disables both the C-API -** [sqlite3_load_extension()] and the SQL function [load_extension()]. -** ^(Use [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],..) -** to enable or disable only the C-API.)^ -** -** Security warning: It is recommended that extension loading -** be disabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method -** rather than this interface, so the [load_extension()] SQL function -** remains disabled. This will prevent SQL injections from giving attackers -** access to extension loading capabilities. */ -SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); +SQLITE_API int SQLITE_STDCALL sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* ** CAPI3REF: Automatically Load Statically Linked Extensions @@ -5974,7 +5609,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^(Even though the function prototype shows that xEntryPoint() takes ** no arguments and returns void, SQLite invokes xEntryPoint() with three -** arguments and expects an integer result as if the signature of the +** arguments and expects and integer result as if the signature of the ** entry point where as follows: ** **
    @@ -6000,7 +5635,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff);
     ** See also: [sqlite3_reset_auto_extension()]
     ** and [sqlite3_cancel_auto_extension()]
     */
    -SQLITE_API int sqlite3_auto_extension(void(*xEntryPoint)(void));
    +SQLITE_API int SQLITE_STDCALL sqlite3_auto_extension(void (*xEntryPoint)(void));
     
     /*
     ** CAPI3REF: Cancel Automatic Extension Loading
    @@ -6012,7 +5647,7 @@ SQLITE_API int sqlite3_auto_extension(void(*xEntryPoint)(void));
     ** unregistered and it returns 0 if X was not on the list of initialization
     ** routines.
     */
    -SQLITE_API int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void));
    +SQLITE_API int SQLITE_STDCALL sqlite3_cancel_auto_extension(void (*xEntryPoint)(void));
     
     /*
     ** CAPI3REF: Reset Automatic Extension Loading
    @@ -6020,7 +5655,7 @@ SQLITE_API int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void));
     ** ^This interface disables all automatic extensions previously
     ** registered using [sqlite3_auto_extension()].
     */
    -SQLITE_API void sqlite3_reset_auto_extension(void);
    +SQLITE_API void SQLITE_STDCALL sqlite3_reset_auto_extension(void);
     
     /*
     ** The interface to the virtual-table mechanism is currently considered
    @@ -6122,17 +5757,6 @@ struct sqlite3_module {
     ** ^Information about the ORDER BY clause is stored in aOrderBy[].
     ** ^Each term of aOrderBy records a column of the ORDER BY clause.
     **
    -** The colUsed field indicates which columns of the virtual table may be
    -** required by the current scan. Virtual table columns are numbered from
    -** zero in the order in which they appear within the CREATE TABLE statement
    -** passed to sqlite3_declare_vtab(). For the first 63 columns (columns 0-62),
    -** the corresponding bit is set within the colUsed mask if the column may be
    -** required by SQLite. If the table has at least 64 columns and any column
    -** to the right of the first 63 is required, then bit 63 of colUsed is also
    -** set. In other words, column iCol may be required if the expression
    -** (colUsed & ((sqlite3_uint64)1 << (iCol>=63 ? 63 : iCol))) evaluates to 
    -** non-zero.
    -**
     ** The [xBestIndex] method must fill aConstraintUsage[] with information
     ** about what parameters to pass to xFilter.  ^If argvIndex>0 then
     ** the right-hand side of the corresponding aConstraint[] is evaluated
    @@ -6158,39 +5782,19 @@ struct sqlite3_module {
     ** ^The estimatedRows value is an estimate of the number of rows that
     ** will be returned by the strategy.
     **
    -** The xBestIndex method may optionally populate the idxFlags field with a 
    -** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag -
    -** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite
    -** assumes that the strategy may visit at most one row. 
    -**
    -** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then
    -** SQLite also assumes that if a call to the xUpdate() method is made as
    -** part of the same statement to delete or update a virtual table row and the
    -** implementation returns SQLITE_CONSTRAINT, then there is no need to rollback
    -** any database changes. In other words, if the xUpdate() returns
    -** SQLITE_CONSTRAINT, the database contents must be exactly as they were
    -** before xUpdate was called. By contrast, if SQLITE_INDEX_SCAN_UNIQUE is not
    -** set and xUpdate returns SQLITE_CONSTRAINT, any database changes made by
    -** the xUpdate method are automatically rolled back by SQLite.
    -**
     ** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info
    -** structure for SQLite [version 3.8.2] ([dateof:3.8.2]). 
    -** If a virtual table extension is
    +** structure for SQLite version 3.8.2. If a virtual table extension is
     ** used with an SQLite version earlier than 3.8.2, the results of attempting 
     ** to read or write the estimatedRows field are undefined (but are likely 
     ** to included crashing the application). The estimatedRows field should
     ** therefore only be used if [sqlite3_libversion_number()] returns a
    -** value greater than or equal to 3008002. Similarly, the idxFlags field
    -** was added for [version 3.9.0] ([dateof:3.9.0]). 
    -** It may therefore only be used if
    -** sqlite3_libversion_number() returns a value greater than or equal to
    -** 3009000.
    +** value greater than or equal to 3008002.
     */
     struct sqlite3_index_info {
       /* Inputs */
       int nConstraint;           /* Number of entries in aConstraint */
       struct sqlite3_index_constraint {
    -     int iColumn;              /* Column constrained.  -1 for ROWID */
    +     int iColumn;              /* Column on left-hand side of constraint */
          unsigned char op;         /* Constraint operator */
          unsigned char usable;     /* True if this constraint is usable */
          int iTermOffset;          /* Used internally - xBestIndex should ignore */
    @@ -6212,17 +5816,8 @@ struct sqlite3_index_info {
       double estimatedCost;           /* Estimated cost of using this index */
       /* Fields below are only available in SQLite 3.8.2 and later */
       sqlite3_int64 estimatedRows;    /* Estimated number of rows returned */
    -  /* Fields below are only available in SQLite 3.9.0 and later */
    -  int idxFlags;              /* Mask of SQLITE_INDEX_SCAN_* flags */
    -  /* Fields below are only available in SQLite 3.10.0 and later */
    -  sqlite3_uint64 colUsed;    /* Input: Mask of columns used by statement */
     };
     
    -/*
    -** CAPI3REF: Virtual Table Scan Flags
    -*/
    -#define SQLITE_INDEX_SCAN_UNIQUE      1     /* Scan visits at most 1 row */
    -
     /*
     ** CAPI3REF: Virtual Table Constraint Operator Codes
     **
    @@ -6231,15 +5826,12 @@ struct sqlite3_index_info {
     ** an operator that is part of a constraint term in the wHERE clause of
     ** a query that uses a [virtual table].
     */
    -#define SQLITE_INDEX_CONSTRAINT_EQ      2
    -#define SQLITE_INDEX_CONSTRAINT_GT      4
    -#define SQLITE_INDEX_CONSTRAINT_LE      8
    -#define SQLITE_INDEX_CONSTRAINT_LT     16
    -#define SQLITE_INDEX_CONSTRAINT_GE     32
    -#define SQLITE_INDEX_CONSTRAINT_MATCH  64
    -#define SQLITE_INDEX_CONSTRAINT_LIKE   65
    -#define SQLITE_INDEX_CONSTRAINT_GLOB   66
    -#define SQLITE_INDEX_CONSTRAINT_REGEXP 67
    +#define SQLITE_INDEX_CONSTRAINT_EQ    2
    +#define SQLITE_INDEX_CONSTRAINT_GT    4
    +#define SQLITE_INDEX_CONSTRAINT_LE    8
    +#define SQLITE_INDEX_CONSTRAINT_LT    16
    +#define SQLITE_INDEX_CONSTRAINT_GE    32
    +#define SQLITE_INDEX_CONSTRAINT_MATCH 64
     
     /*
     ** CAPI3REF: Register A Virtual Table Implementation
    @@ -6267,13 +5859,13 @@ struct sqlite3_index_info {
     ** interface is equivalent to sqlite3_create_module_v2() with a NULL
     ** destructor.
     */
    -SQLITE_API int sqlite3_create_module(
    +SQLITE_API int SQLITE_STDCALL sqlite3_create_module(
       sqlite3 *db,               /* SQLite connection to register module with */
       const char *zName,         /* Name of the module */
       const sqlite3_module *p,   /* Methods for the module */
       void *pClientData          /* Client data for xCreate/xConnect */
     );
    -SQLITE_API int sqlite3_create_module_v2(
    +SQLITE_API int SQLITE_STDCALL sqlite3_create_module_v2(
       sqlite3 *db,               /* SQLite connection to register module with */
       const char *zName,         /* Name of the module */
       const sqlite3_module *p,   /* Methods for the module */
    @@ -6336,7 +5928,7 @@ struct sqlite3_vtab_cursor {
     ** to declare the format (the names and datatypes of the columns) of
     ** the virtual tables they implement.
     */
    -SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL);
    +SQLITE_API int SQLITE_STDCALL sqlite3_declare_vtab(sqlite3*, const char *zSQL);
     
     /*
     ** CAPI3REF: Overload A Function For A Virtual Table
    @@ -6355,7 +5947,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL);
     ** purpose is to be a placeholder function that can be overloaded
     ** by a [virtual table].
     */
    -SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg);
    +SQLITE_API int SQLITE_STDCALL sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg);
     
     /*
     ** The interface to the virtual-table mechanism defined above (back up
    @@ -6454,7 +6046,7 @@ typedef struct sqlite3_blob sqlite3_blob;
     ** To avoid a resource leak, every open [BLOB handle] should eventually
     ** be released by a call to [sqlite3_blob_close()].
     */
    -SQLITE_API int sqlite3_blob_open(
    +SQLITE_API int SQLITE_STDCALL sqlite3_blob_open(
       sqlite3*,
       const char *zDb,
       const char *zTable,
    @@ -6487,7 +6079,7 @@ SQLITE_API int sqlite3_blob_open(
     **
     ** ^This function sets the database handle error code and message.
     */
    -SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
    +SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
     
     /*
     ** CAPI3REF: Close A BLOB Handle
    @@ -6510,7 +6102,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
     ** is passed a valid open blob handle, the values returned by the 
     ** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning.
     */
    -SQLITE_API int sqlite3_blob_close(sqlite3_blob *);
    +SQLITE_API int SQLITE_STDCALL sqlite3_blob_close(sqlite3_blob *);
     
     /*
     ** CAPI3REF: Return The Size Of An Open BLOB
    @@ -6526,7 +6118,7 @@ SQLITE_API int sqlite3_blob_close(sqlite3_blob *);
     ** been closed by [sqlite3_blob_close()].  Passing any other pointer in
     ** to this routine results in undefined and probably undesirable behavior.
     */
    -SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *);
    +SQLITE_API int SQLITE_STDCALL sqlite3_blob_bytes(sqlite3_blob *);
     
     /*
     ** CAPI3REF: Read Data From A BLOB Incrementally
    @@ -6555,7 +6147,7 @@ SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *);
     **
     ** See also: [sqlite3_blob_write()].
     */
    -SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset);
    +SQLITE_API int SQLITE_STDCALL sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset);
     
     /*
     ** CAPI3REF: Write Data Into A BLOB Incrementally
    @@ -6597,7 +6189,7 @@ SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset);
     **
     ** See also: [sqlite3_blob_read()].
     */
    -SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset);
    +SQLITE_API int SQLITE_STDCALL sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset);
     
     /*
     ** CAPI3REF: Virtual File System Objects
    @@ -6628,9 +6220,9 @@ SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOff
     ** ^(If the default VFS is unregistered, another VFS is chosen as
     ** the default.  The choice for the new VFS is arbitrary.)^
     */
    -SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName);
    -SQLITE_API int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt);
    -SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*);
    +SQLITE_API sqlite3_vfs *SQLITE_STDCALL sqlite3_vfs_find(const char *zVfsName);
    +SQLITE_API int SQLITE_STDCALL sqlite3_vfs_register(sqlite3_vfs*, int makeDflt);
    +SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs*);
     
     /*
     ** CAPI3REF: Mutexes
    @@ -6683,9 +6275,6 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*);
     ** 
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 **
  • SQLITE_MUTEX_STATIC_APP3 -**
  • SQLITE_MUTEX_STATIC_VFS1 -**
  • SQLITE_MUTEX_STATIC_VFS2 -**
  • SQLITE_MUTEX_STATIC_VFS3 ** ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) @@ -6746,11 +6335,11 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. */ -SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int); -SQLITE_API void sqlite3_mutex_free(sqlite3_mutex*); -SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex*); -SQLITE_API int sqlite3_mutex_try(sqlite3_mutex*); -SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); +SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_mutex_alloc(int); +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex*); +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex*); +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex*); +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex*); /* ** CAPI3REF: Mutex Methods Object @@ -6860,8 +6449,8 @@ struct sqlite3_mutex_methods { ** interface should also return 1 when given a NULL pointer. */ #ifndef NDEBUG -SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); -SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_held(sqlite3_mutex*); +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex*); #endif /* @@ -6880,16 +6469,13 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_MEM 3 /* sqlite3_malloc() */ #define SQLITE_MUTEX_STATIC_MEM2 4 /* NOT USED */ #define SQLITE_MUTEX_STATIC_OPEN 4 /* sqlite3BtreeOpen() */ -#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_randomness() */ +#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_random() */ #define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ #define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ #define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ -#define SQLITE_MUTEX_STATIC_VFS1 11 /* For use by built-in VFS */ -#define SQLITE_MUTEX_STATIC_VFS2 12 /* For use by extension VFS */ -#define SQLITE_MUTEX_STATIC_VFS3 13 /* For use by application VFS */ /* ** CAPI3REF: Retrieve the mutex for a database connection @@ -6901,7 +6487,7 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); ** ^If the [threading mode] is Single-thread or Multi-thread then this ** routine returns a NULL pointer. */ -SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); +SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3*); /* ** CAPI3REF: Low-Level Control Of Database Files @@ -6936,7 +6522,7 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); ** ** See also: [SQLITE_FCNTL_LOCKSTATE] */ -SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); +SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); /* ** CAPI3REF: Testing Interface @@ -6955,7 +6541,7 @@ SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void* ** Unlike most of the SQLite API, this function is not guaranteed to ** operate consistently from one release to the next. */ -SQLITE_API int sqlite3_test_control(int op, ...); +SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...); /* ** CAPI3REF: Testing Interface Operation Codes @@ -6984,7 +6570,6 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 #define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ -#define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD 19 #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 #define SQLITE_TESTCTRL_VDBE_COVERAGE 21 #define SQLITE_TESTCTRL_BYTEORDER 22 @@ -7019,8 +6604,8 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** ** See also: [sqlite3_db_status()] */ -SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); -SQLITE_API int sqlite3_status64( +SQLITE_API int SQLITE_STDCALL sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); +SQLITE_API int SQLITE_STDCALL sqlite3_status64( int op, sqlite3_int64 *pCurrent, sqlite3_int64 *pHighwater, @@ -7104,8 +6689,7 @@ SQLITE_API int sqlite3_status64( ** The value written into the *pCurrent parameter is undefined.)^ ** ** [[SQLITE_STATUS_PARSER_STACK]] ^(
    SQLITE_STATUS_PARSER_STACK
    -**
    The *pHighwater parameter records the deepest parser stack. -** The *pCurrent value is undefined. The *pHighwater value is only +**
    This parameter records the deepest parser stack. It is only ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
    )^ ** ** @@ -7145,7 +6729,7 @@ SQLITE_API int sqlite3_status64( ** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ -SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); +SQLITE_API int SQLITE_STDCALL sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); /* ** CAPI3REF: Status Parameters for database connections @@ -7191,18 +6775,6 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** -** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] -** ^(
    SQLITE_DBSTATUS_CACHE_USED_SHARED
    -**
    This parameter is similar to DBSTATUS_CACHE_USED, except that if a -** pager cache is shared between two or more connections the bytes of heap -** memory used by that pager cache is divided evenly between the attached -** connections.)^ In other words, if none of the pager caches associated -** with the database connection are shared, this request returns the same -** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are -** shared, the value returned by this call will be smaller than that returned -** by DBSTATUS_CACHE_USED. ^The highwater mark associated with -** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. -** ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    **
    This parameter returns the approximate number of bytes of heap ** memory used to store the schema for all databases associated @@ -7260,8 +6832,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r #define SQLITE_DBSTATUS_CACHE_MISS 8 #define SQLITE_DBSTATUS_CACHE_WRITE 9 #define SQLITE_DBSTATUS_DEFERRED_FKS 10 -#define SQLITE_DBSTATUS_CACHE_USED_SHARED 11 -#define SQLITE_DBSTATUS_MAX 11 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_MAX 10 /* Largest defined DBSTATUS */ /* @@ -7288,7 +6859,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** ** See also: [sqlite3_status()] and [sqlite3_db_status()]. */ -SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); /* ** CAPI3REF: Status Parameters for prepared statements @@ -7615,7 +7186,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with ** an error. ** -** ^A call to sqlite3_backup_init() will fail, returning NULL, if +** ^A call to sqlite3_backup_init() will fail, returning SQLITE_ERROR, if ** there is already a read or read-write transaction open on the ** destination database. ** @@ -7757,16 +7328,16 @@ typedef struct sqlite3_backup sqlite3_backup; ** same time as another thread is invoking sqlite3_backup_step() it is ** possible that they return invalid values. */ -SQLITE_API sqlite3_backup *sqlite3_backup_init( +SQLITE_API sqlite3_backup *SQLITE_STDCALL sqlite3_backup_init( sqlite3 *pDest, /* Destination database handle */ const char *zDestName, /* Destination database name */ sqlite3 *pSource, /* Source database handle */ const char *zSourceName /* Source database name */ ); -SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage); -SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p); -SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p); -SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_finish(sqlite3_backup *p); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_remaining(sqlite3_backup *p); +SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p); /* ** CAPI3REF: Unlock Notification @@ -7883,7 +7454,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); ** the special "DROP TABLE/INDEX" case, the extended error code is just ** SQLITE_LOCKED.)^ */ -SQLITE_API int sqlite3_unlock_notify( +SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( sqlite3 *pBlocked, /* Waiting connection */ void (*xNotify)(void **apArg, int nArg), /* Callback function to invoke */ void *pNotifyArg /* Argument to pass to xNotify */ @@ -7898,48 +7469,23 @@ SQLITE_API int sqlite3_unlock_notify( ** strings in a case-independent fashion, using the same definition of "case ** independence" that SQLite uses internally when comparing identifiers. */ -SQLITE_API int sqlite3_stricmp(const char *, const char *); -SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); +SQLITE_API int SQLITE_STDCALL sqlite3_stricmp(const char *, const char *); +SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *, const char *, int); /* ** CAPI3REF: String Globbing * -** ^The [sqlite3_strglob(P,X)] interface returns zero if and only if -** string X matches the [GLOB] pattern P. -** ^The definition of [GLOB] pattern matching used in +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in ** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the -** SQL dialect understood by SQLite. ^The [sqlite3_strglob(P,X)] function -** is case sensitive. +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. ** ** Note that this routine returns zero on a match and non-zero if the strings ** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. -** -** See also: [sqlite3_strlike()]. */ -SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); - -/* -** CAPI3REF: String LIKE Matching -* -** ^The [sqlite3_strlike(P,X,E)] interface returns zero if and only if -** string X matches the [LIKE] pattern P with escape character E. -** ^The definition of [LIKE] pattern matching used in -** [sqlite3_strlike(P,X,E)] is the same as for the "X LIKE P ESCAPE E" -** operator in the SQL dialect understood by SQLite. ^For "X LIKE P" without -** the ESCAPE clause, set the E parameter of [sqlite3_strlike(P,X,E)] to 0. -** ^As with the LIKE operator, the [sqlite3_strlike(P,X,E)] function is case -** insensitive - equivalent upper and lower case ASCII characters match -** one another. -** -** ^The [sqlite3_strlike(P,X,E)] function matches Unicode characters, though -** only ASCII characters are case folded. -** -** Note that this routine returns zero on a match and non-zero if the strings -** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. -** -** See also: [sqlite3_strglob()]. -*/ -SQLITE_API int sqlite3_strlike(const char *zGlob, const char *zStr, unsigned int cEsc); +SQLITE_API int SQLITE_STDCALL sqlite3_strglob(const char *zGlob, const char *zStr); /* ** CAPI3REF: Error Logging Interface @@ -7962,7 +7508,7 @@ SQLITE_API int sqlite3_strlike(const char *zGlob, const char *zStr, unsigned int ** a few hundred characters, it will be truncated to the length of the ** buffer. */ -SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); +SQLITE_API void SQLITE_CDECL sqlite3_log(int iErrCode, const char *zFormat, ...); /* ** CAPI3REF: Write-Ahead Log Commit Hook @@ -7996,9 +7542,9 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** previously registered write-ahead log callback. ^Note that the ** [sqlite3_wal_autocheckpoint()] interface and the ** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will -** overwrite any prior [sqlite3_wal_hook()] settings. +** those overwrite any prior [sqlite3_wal_hook()] settings. */ -SQLITE_API void *sqlite3_wal_hook( +SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( sqlite3*, int(*)(void *,sqlite3*,const char*,int), void* @@ -8033,7 +7579,7 @@ SQLITE_API void *sqlite3_wal_hook( ** is only necessary if the default setting is found to be suboptimal ** for a particular application. */ -SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); +SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int N); /* ** CAPI3REF: Checkpoint a database @@ -8055,7 +7601,7 @@ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); ** start a callback but which do not need the full power (and corresponding ** complication) of [sqlite3_wal_checkpoint_v2()]. */ -SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Checkpoint a database @@ -8149,7 +7695,7 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface ** from SQL. */ -SQLITE_API int sqlite3_wal_checkpoint_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( sqlite3 *db, /* Database handle */ const char *zDb, /* Name of attached database (or NULL) */ int eMode, /* SQLITE_CHECKPOINT_* value */ @@ -8185,7 +7731,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options ** may be added in the future. */ -SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); +SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3*, int op, ...); /* ** CAPI3REF: Virtual Table Configuration Options @@ -8238,7 +7784,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** of the SQL statement that triggered the call to the [xUpdate] method of the ** [virtual table]. */ -SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); +SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *); /* ** CAPI3REF: Conflict resolution modes @@ -8343,7 +7889,7 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); ** ** See also: [sqlite3_stmt_scanstatus_reset()] */ -SQLITE_API int sqlite3_stmt_scanstatus( +SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ int idx, /* Index of loop to report on */ int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ @@ -8359,279 +7905,8 @@ SQLITE_API int sqlite3_stmt_scanstatus( ** This API is only available if the library is built with pre-processor ** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. */ -SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); - -/* -** CAPI3REF: Flush caches to disk mid-transaction -** -** ^If a write-transaction is open on [database connection] D when the -** [sqlite3_db_cacheflush(D)] interface invoked, any dirty -** pages in the pager-cache that are not currently in use are written out -** to disk. A dirty page may be in use if a database cursor created by an -** active SQL statement is reading from it, or if it is page 1 of a database -** file (page 1 is always "in use"). ^The [sqlite3_db_cacheflush(D)] -** interface flushes caches for all schemas - "main", "temp", and -** any [attached] databases. -** -** ^If this function needs to obtain extra database locks before dirty pages -** can be flushed to disk, it does so. ^If those locks cannot be obtained -** immediately and there is a busy-handler callback configured, it is invoked -** in the usual manner. ^If the required lock still cannot be obtained, then -** the database is skipped and an attempt made to flush any dirty pages -** belonging to the next (if any) database. ^If any databases are skipped -** because locks cannot be obtained, but no other error occurs, this -** function returns SQLITE_BUSY. -** -** ^If any other error occurs while flushing dirty pages to disk (for -** example an IO error or out-of-memory condition), then processing is -** abandoned and an SQLite [error code] is returned to the caller immediately. -** -** ^Otherwise, if no error occurs, [sqlite3_db_cacheflush()] returns SQLITE_OK. -** -** ^This function does not set the database handle error code or message -** returned by the [sqlite3_errcode()] and [sqlite3_errmsg()] functions. -*/ -SQLITE_API int sqlite3_db_cacheflush(sqlite3*); - -/* -** CAPI3REF: The pre-update hook. -** -** ^These interfaces are only available if SQLite is compiled using the -** [SQLITE_ENABLE_PREUPDATE_HOOK] compile-time option. -** -** ^The [sqlite3_preupdate_hook()] interface registers a callback function -** that is invoked prior to each [INSERT], [UPDATE], and [DELETE] operation -** on a [rowid table]. -** ^At most one preupdate hook may be registered at a time on a single -** [database connection]; each call to [sqlite3_preupdate_hook()] overrides -** the previous setting. -** ^The preupdate hook is disabled by invoking [sqlite3_preupdate_hook()] -** with a NULL pointer as the second parameter. -** ^The third parameter to [sqlite3_preupdate_hook()] is passed through as -** the first parameter to callbacks. -** -** ^The preupdate hook only fires for changes to [rowid tables]; the preupdate -** hook is not invoked for changes to [virtual tables] or [WITHOUT ROWID] -** tables. -** -** ^The second parameter to the preupdate callback is a pointer to -** the [database connection] that registered the preupdate hook. -** ^The third parameter to the preupdate callback is one of the constants -** [SQLITE_INSERT], [SQLITE_DELETE], or [SQLITE_UPDATE] to identify the -** kind of update operation that is about to occur. -** ^(The fourth parameter to the preupdate callback is the name of the -** database within the database connection that is being modified. This -** will be "main" for the main database or "temp" for TEMP tables or -** the name given after the AS keyword in the [ATTACH] statement for attached -** databases.)^ -** ^The fifth parameter to the preupdate callback is the name of the -** table that is being modified. -** ^The sixth parameter to the preupdate callback is the initial [rowid] of the -** row being changes for SQLITE_UPDATE and SQLITE_DELETE changes and is -** undefined for SQLITE_INSERT changes. -** ^The seventh parameter to the preupdate callback is the final [rowid] of -** the row being changed for SQLITE_UPDATE and SQLITE_INSERT changes and is -** undefined for SQLITE_DELETE changes. -** -** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()], -** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces -** provide additional information about a preupdate event. These routines -** may only be called from within a preupdate callback. Invoking any of -** these routines from outside of a preupdate callback or with a -** [database connection] pointer that is different from the one supplied -** to the preupdate callback results in undefined and probably undesirable -** behavior. -** -** ^The [sqlite3_preupdate_count(D)] interface returns the number of columns -** in the row that is being inserted, updated, or deleted. -** -** ^The [sqlite3_preupdate_old(D,N,P)] interface writes into P a pointer to -** a [protected sqlite3_value] that contains the value of the Nth column of -** the table row before it is updated. The N parameter must be between 0 -** and one less than the number of columns or the behavior will be -** undefined. This must only be used within SQLITE_UPDATE and SQLITE_DELETE -** preupdate callbacks; if it is used by an SQLITE_INSERT callback then the -** behavior is undefined. The [sqlite3_value] that P points to -** will be destroyed when the preupdate callback returns. -** -** ^The [sqlite3_preupdate_new(D,N,P)] interface writes into P a pointer to -** a [protected sqlite3_value] that contains the value of the Nth column of -** the table row after it is updated. The N parameter must be between 0 -** and one less than the number of columns or the behavior will be -** undefined. This must only be used within SQLITE_INSERT and SQLITE_UPDATE -** preupdate callbacks; if it is used by an SQLITE_DELETE callback then the -** behavior is undefined. The [sqlite3_value] that P points to -** will be destroyed when the preupdate callback returns. -** -** ^The [sqlite3_preupdate_depth(D)] interface returns 0 if the preupdate -** callback was invoked as a result of a direct insert, update, or delete -** operation; or 1 for inserts, updates, or deletes invoked by top-level -** triggers; or 2 for changes resulting from triggers called by top-level -** triggers; and so forth. -** -** See also: [sqlite3_update_hook()] -*/ -SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_preupdate_hook( - sqlite3 *db, - void(*xPreUpdate)( - void *pCtx, /* Copy of third arg to preupdate_hook() */ - sqlite3 *db, /* Database handle */ - int op, /* SQLITE_UPDATE, DELETE or INSERT */ - char const *zDb, /* Database name */ - char const *zName, /* Table name */ - sqlite3_int64 iKey1, /* Rowid of row about to be deleted/updated */ - sqlite3_int64 iKey2 /* New rowid value (for a rowid UPDATE) */ - ), - void* -); -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_preupdate_old(sqlite3 *, int, sqlite3_value **); -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_preupdate_count(sqlite3 *); -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_preupdate_depth(sqlite3 *); -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_preupdate_new(sqlite3 *, int, sqlite3_value **); - -/* -** CAPI3REF: Low-level system error code -** -** ^Attempt to return the underlying operating system error code or error -** number that caused the most recent I/O error or failure to open a file. -** The return value is OS-dependent. For example, on unix systems, after -** [sqlite3_open_v2()] returns [SQLITE_CANTOPEN], this interface could be -** called to get back the underlying "errno" that caused the problem, such -** as ENOSPC, EAUTH, EISDIR, and so forth. -*/ -SQLITE_API int sqlite3_system_errno(sqlite3*); - -/* -** CAPI3REF: Database Snapshot -** KEYWORDS: {snapshot} -** EXPERIMENTAL -** -** An instance of the snapshot object records the state of a [WAL mode] -** database for some specific point in history. -** -** In [WAL mode], multiple [database connections] that are open on the -** same database file can each be reading a different historical version -** of the database file. When a [database connection] begins a read -** transaction, that connection sees an unchanging copy of the database -** as it existed for the point in time when the transaction first started. -** Subsequent changes to the database from other connections are not seen -** by the reader until a new read transaction is started. -** -** The sqlite3_snapshot object records state information about an historical -** version of the database file so that it is possible to later open a new read -** transaction that sees that historical version of the database rather than -** the most recent version. -** -** The constructor for this object is [sqlite3_snapshot_get()]. The -** [sqlite3_snapshot_open()] method causes a fresh read transaction to refer -** to an historical snapshot (if possible). The destructor for -** sqlite3_snapshot objects is [sqlite3_snapshot_free()]. -*/ -typedef struct sqlite3_snapshot sqlite3_snapshot; - -/* -** CAPI3REF: Record A Database Snapshot -** EXPERIMENTAL -** -** ^The [sqlite3_snapshot_get(D,S,P)] interface attempts to make a -** new [sqlite3_snapshot] object that records the current state of -** schema S in database connection D. ^On success, the -** [sqlite3_snapshot_get(D,S,P)] interface writes a pointer to the newly -** created [sqlite3_snapshot] object into *P and returns SQLITE_OK. -** ^If schema S of [database connection] D is not a [WAL mode] database -** that is in a read transaction, then [sqlite3_snapshot_get(D,S,P)] -** leaves the *P value unchanged and returns an appropriate [error code]. -** -** The [sqlite3_snapshot] object returned from a successful call to -** [sqlite3_snapshot_get()] must be freed using [sqlite3_snapshot_free()] -** to avoid a memory leak. -** -** The [sqlite3_snapshot_get()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. -*/ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( - sqlite3 *db, - const char *zSchema, - sqlite3_snapshot **ppSnapshot -); +SQLITE_API SQLITE_EXPERIMENTAL void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); -/* -** CAPI3REF: Start a read transaction on an historical snapshot -** EXPERIMENTAL -** -** ^The [sqlite3_snapshot_open(D,S,P)] interface starts a -** read transaction for schema S of -** [database connection] D such that the read transaction -** refers to historical [snapshot] P, rather than the most -** recent change to the database. -** ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK on success -** or an appropriate [error code] if it fails. -** -** ^In order to succeed, a call to [sqlite3_snapshot_open(D,S,P)] must be -** the first operation following the [BEGIN] that takes the schema S -** out of [autocommit mode]. -** ^In other words, schema S must not currently be in -** a transaction for [sqlite3_snapshot_open(D,S,P)] to work, but the -** database connection D must be out of [autocommit mode]. -** ^A [snapshot] will fail to open if it has been overwritten by a -** [checkpoint]. -** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the -** database connection D does not know that the database file for -** schema S is in [WAL mode]. A database connection might not know -** that the database file is in [WAL mode] if there has been no prior -** I/O on that database connection, or if the database entered [WAL mode] -** after the most recent I/O on the database connection.)^ -** (Hint: Run "[PRAGMA application_id]" against a newly opened -** database connection in order to make it ready to use snapshots.) -** -** The [sqlite3_snapshot_open()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. -*/ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( - sqlite3 *db, - const char *zSchema, - sqlite3_snapshot *pSnapshot -); - -/* -** CAPI3REF: Destroy a snapshot -** EXPERIMENTAL -** -** ^The [sqlite3_snapshot_free(P)] interface destroys [sqlite3_snapshot] P. -** The application must eventually free every [sqlite3_snapshot] object -** using this routine to avoid a memory leak. -** -** The [sqlite3_snapshot_free()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. -*/ -SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); - -/* -** CAPI3REF: Compare the ages of two snapshot handles. -** EXPERIMENTAL -** -** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages -** of two valid snapshot handles. -** -** If the two snapshot handles are not associated with the same database -** file, the result of the comparison is undefined. -** -** Additionally, the result of the comparison is only valid if both of the -** snapshot handles were obtained by calling sqlite3_snapshot_get() since the -** last time the wal file was deleted. The wal file is deleted when the -** database is changed back to rollback mode or when the number of database -** clients drops to zero. If either snapshot handle was obtained before the -** wal file was last deleted, the value returned by this function -** is undefined. -** -** Otherwise, this API returns a negative value if P1 refers to an older -** snapshot than P2, zero if the two handles refer to the same database -** snapshot, and a positive value if P1 is a newer snapshot than P2. -*/ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( - sqlite3_snapshot *p1, - sqlite3_snapshot *p2 -); /* ** Undo the hack that converts floating point types to integer for @@ -8644,9 +7919,8 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( #if 0 } /* End of the 'extern "C"' block */ #endif -#endif /* SQLITE3_H */ +#endif /* _SQLITE3_H_ */ -/******** Begin file sqlite3rtree.h *********/ /* ** 2010 August 30 ** @@ -8686,7 +7960,7 @@ typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; ** ** SELECT ... FROM WHERE MATCH $zGeom(... params ...) */ -SQLITE_API int sqlite3_rtree_geometry_callback( +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), @@ -8712,7 +7986,7 @@ struct sqlite3_rtree_geometry { ** ** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) */ -SQLITE_API int sqlite3_rtree_query_callback( +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_query_callback( sqlite3 *db, const char *zQueryFunc, int (*xQueryFunc)(sqlite3_rtree_query_info*), @@ -8746,8 +8020,6 @@ struct sqlite3_rtree_query_info { int eParentWithin; /* Visibility of parent node */ int eWithin; /* OUT: Visiblity */ sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ - /* The following fields are only available in 3.8.11 and later */ - sqlite3_value **apSqlParam; /* Original SQL values of parameters */ }; /* @@ -8764,1871 +8036,6 @@ struct sqlite3_rtree_query_info { #endif /* ifndef _SQLITE3RTREE_H_ */ -/******** End of sqlite3rtree.h *********/ -/******** Begin file sqlite3session.h *********/ - -#if !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) -#define __SQLITESESSION_H_ 1 - -/* -** Make sure we can call this stuff from C++. -*/ -#if 0 -extern "C" { -#endif - - -/* -** CAPI3REF: Session Object Handle -*/ -typedef struct sqlite3_session sqlite3_session; - -/* -** CAPI3REF: Changeset Iterator Handle -*/ -typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; - -/* -** CAPI3REF: Create A New Session Object -** -** Create a new session object attached to database handle db. If successful, -** a pointer to the new object is written to *ppSession and SQLITE_OK is -** returned. If an error occurs, *ppSession is set to NULL and an SQLite -** error code (e.g. SQLITE_NOMEM) is returned. -** -** It is possible to create multiple session objects attached to a single -** database handle. -** -** Session objects created using this function should be deleted using the -** [sqlite3session_delete()] function before the database handle that they -** are attached to is itself closed. If the database handle is closed before -** the session object is deleted, then the results of calling any session -** module function, including [sqlite3session_delete()] on the session object -** are undefined. -** -** Because the session module uses the [sqlite3_preupdate_hook()] API, it -** is not possible for an application to register a pre-update hook on a -** database handle that has one or more session objects attached. Nor is -** it possible to create a session object attached to a database handle for -** which a pre-update hook is already defined. The results of attempting -** either of these things are undefined. -** -** The session object will be used to create changesets for tables in -** database zDb, where zDb is either "main", or "temp", or the name of an -** attached database. It is not an error if database zDb is not attached -** to the database when the session object is created. -*/ -int sqlite3session_create( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Name of db (e.g. "main") */ - sqlite3_session **ppSession /* OUT: New session object */ -); - -/* -** CAPI3REF: Delete A Session Object -** -** Delete a session object previously allocated using -** [sqlite3session_create()]. Once a session object has been deleted, the -** results of attempting to use pSession with any other session module -** function are undefined. -** -** Session objects must be deleted before the database handle to which they -** are attached is closed. Refer to the documentation for -** [sqlite3session_create()] for details. -*/ -void sqlite3session_delete(sqlite3_session *pSession); - - -/* -** CAPI3REF: Enable Or Disable A Session Object -** -** Enable or disable the recording of changes by a session object. When -** enabled, a session object records changes made to the database. When -** disabled - it does not. A newly created session object is enabled. -** Refer to the documentation for [sqlite3session_changeset()] for further -** details regarding how enabling and disabling a session object affects -** the eventual changesets. -** -** Passing zero to this function disables the session. Passing a value -** greater than zero enables it. Passing a value less than zero is a -** no-op, and may be used to query the current state of the session. -** -** The return value indicates the final state of the session object: 0 if -** the session is disabled, or 1 if it is enabled. -*/ -int sqlite3session_enable(sqlite3_session *pSession, int bEnable); - -/* -** CAPI3REF: Set Or Clear the Indirect Change Flag -** -** Each change recorded by a session object is marked as either direct or -** indirect. A change is marked as indirect if either: -** -**
      -**
    • The session object "indirect" flag is set when the change is -** made, or -**
    • The change is made by an SQL trigger or foreign key action -** instead of directly as a result of a users SQL statement. -**
    -** -** If a single row is affected by more than one operation within a session, -** then the change is considered indirect if all operations meet the criteria -** for an indirect change above, or direct otherwise. -** -** This function is used to set, clear or query the session object indirect -** flag. If the second argument passed to this function is zero, then the -** indirect flag is cleared. If it is greater than zero, the indirect flag -** is set. Passing a value less than zero does not modify the current value -** of the indirect flag, and may be used to query the current state of the -** indirect flag for the specified session object. -** -** The return value indicates the final state of the indirect flag: 0 if -** it is clear, or 1 if it is set. -*/ -int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); - -/* -** CAPI3REF: Attach A Table To A Session Object -** -** If argument zTab is not NULL, then it is the name of a table to attach -** to the session object passed as the first argument. All subsequent changes -** made to the table while the session object is enabled will be recorded. See -** documentation for [sqlite3session_changeset()] for further details. -** -** Or, if argument zTab is NULL, then changes are recorded for all tables -** in the database. If additional tables are added to the database (by -** executing "CREATE TABLE" statements) after this call is made, changes for -** the new tables are also recorded. -** -** Changes can only be recorded for tables that have a PRIMARY KEY explicitly -** defined as part of their CREATE TABLE statement. It does not matter if the -** PRIMARY KEY is an "INTEGER PRIMARY KEY" (rowid alias) or not. The PRIMARY -** KEY may consist of a single column, or may be a composite key. -** -** It is not an error if the named table does not exist in the database. Nor -** is it an error if the named table does not have a PRIMARY KEY. However, -** no changes will be recorded in either of these scenarios. -** -** Changes are not recorded for individual rows that have NULL values stored -** in one or more of their PRIMARY KEY columns. -** -** SQLITE_OK is returned if the call completes without error. Or, if an error -** occurs, an SQLite error code (e.g. SQLITE_NOMEM) is returned. -*/ -int sqlite3session_attach( - sqlite3_session *pSession, /* Session object */ - const char *zTab /* Table name */ -); - -/* -** CAPI3REF: Set a table filter on a Session Object. -** -** The second argument (xFilter) is the "filter callback". For changes to rows -** in tables that are not attached to the Session object, the filter is called -** to determine whether changes to the table's rows should be tracked or not. -** If xFilter returns 0, changes is not tracked. Note that once a table is -** attached, xFilter will not be called again. -*/ -void sqlite3session_table_filter( - sqlite3_session *pSession, /* Session object */ - int(*xFilter)( - void *pCtx, /* Copy of third arg to _filter_table() */ - const char *zTab /* Table name */ - ), - void *pCtx /* First argument passed to xFilter */ -); - -/* -** CAPI3REF: Generate A Changeset From A Session Object -** -** Obtain a changeset containing changes to the tables attached to the -** session object passed as the first argument. If successful, -** set *ppChangeset to point to a buffer containing the changeset -** and *pnChangeset to the size of the changeset in bytes before returning -** SQLITE_OK. If an error occurs, set both *ppChangeset and *pnChangeset to -** zero and return an SQLite error code. -** -** A changeset consists of zero or more INSERT, UPDATE and/or DELETE changes, -** each representing a change to a single row of an attached table. An INSERT -** change contains the values of each field of a new database row. A DELETE -** contains the original values of each field of a deleted database row. An -** UPDATE change contains the original values of each field of an updated -** database row along with the updated values for each updated non-primary-key -** column. It is not possible for an UPDATE change to represent a change that -** modifies the values of primary key columns. If such a change is made, it -** is represented in a changeset as a DELETE followed by an INSERT. -** -** Changes are not recorded for rows that have NULL values stored in one or -** more of their PRIMARY KEY columns. If such a row is inserted or deleted, -** no corresponding change is present in the changesets returned by this -** function. If an existing row with one or more NULL values stored in -** PRIMARY KEY columns is updated so that all PRIMARY KEY columns are non-NULL, -** only an INSERT is appears in the changeset. Similarly, if an existing row -** with non-NULL PRIMARY KEY values is updated so that one or more of its -** PRIMARY KEY columns are set to NULL, the resulting changeset contains a -** DELETE change only. -** -** The contents of a changeset may be traversed using an iterator created -** using the [sqlite3changeset_start()] API. A changeset may be applied to -** a database with a compatible schema using the [sqlite3changeset_apply()] -** API. -** -** Within a changeset generated by this function, all changes related to a -** single table are grouped together. In other words, when iterating through -** a changeset or when applying a changeset to a database, all changes related -** to a single table are processed before moving on to the next table. Tables -** are sorted in the same order in which they were attached (or auto-attached) -** to the sqlite3_session object. The order in which the changes related to -** a single table are stored is undefined. -** -** Following a successful call to this function, it is the responsibility of -** the caller to eventually free the buffer that *ppChangeset points to using -** [sqlite3_free()]. -** -**

    Changeset Generation

    -** -** Once a table has been attached to a session object, the session object -** records the primary key values of all new rows inserted into the table. -** It also records the original primary key and other column values of any -** deleted or updated rows. For each unique primary key value, data is only -** recorded once - the first time a row with said primary key is inserted, -** updated or deleted in the lifetime of the session. -** -** There is one exception to the previous paragraph: when a row is inserted, -** updated or deleted, if one or more of its primary key columns contain a -** NULL value, no record of the change is made. -** -** The session object therefore accumulates two types of records - those -** that consist of primary key values only (created when the user inserts -** a new record) and those that consist of the primary key values and the -** original values of other table columns (created when the users deletes -** or updates a record). -** -** When this function is called, the requested changeset is created using -** both the accumulated records and the current contents of the database -** file. Specifically: -** -**
      -**
    • For each record generated by an insert, the database is queried -** for a row with a matching primary key. If one is found, an INSERT -** change is added to the changeset. If no such row is found, no change -** is added to the changeset. -** -**
    • For each record generated by an update or delete, the database is -** queried for a row with a matching primary key. If such a row is -** found and one or more of the non-primary key fields have been -** modified from their original values, an UPDATE change is added to -** the changeset. Or, if no such row is found in the table, a DELETE -** change is added to the changeset. If there is a row with a matching -** primary key in the database, but all fields contain their original -** values, no change is added to the changeset. -**
    -** -** This means, amongst other things, that if a row is inserted and then later -** deleted while a session object is active, neither the insert nor the delete -** will be present in the changeset. Or if a row is deleted and then later a -** row with the same primary key values inserted while a session object is -** active, the resulting changeset will contain an UPDATE change instead of -** a DELETE and an INSERT. -** -** When a session object is disabled (see the [sqlite3session_enable()] API), -** it does not accumulate records when rows are inserted, updated or deleted. -** This may appear to have some counter-intuitive effects if a single row -** is written to more than once during a session. For example, if a row -** is inserted while a session object is enabled, then later deleted while -** the same session object is disabled, no INSERT record will appear in the -** changeset, even though the delete took place while the session was disabled. -** Or, if one field of a row is updated while a session is disabled, and -** another field of the same row is updated while the session is enabled, the -** resulting changeset will contain an UPDATE change that updates both fields. -*/ -int sqlite3session_changeset( - sqlite3_session *pSession, /* Session object */ - int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ - void **ppChangeset /* OUT: Buffer containing changeset */ -); - -/* -** CAPI3REF: Load The Difference Between Tables Into A Session -** -** If it is not already attached to the session object passed as the first -** argument, this function attaches table zTbl in the same manner as the -** [sqlite3session_attach()] function. If zTbl does not exist, or if it -** does not have a primary key, this function is a no-op (but does not return -** an error). -** -** Argument zFromDb must be the name of a database ("main", "temp" etc.) -** attached to the same database handle as the session object that contains -** a table compatible with the table attached to the session by this function. -** A table is considered compatible if it: -** -**
      -**
    • Has the same name, -**
    • Has the same set of columns declared in the same order, and -**
    • Has the same PRIMARY KEY definition. -**
    -** -** If the tables are not compatible, SQLITE_SCHEMA is returned. If the tables -** are compatible but do not have any PRIMARY KEY columns, it is not an error -** but no changes are added to the session object. As with other session -** APIs, tables without PRIMARY KEYs are simply ignored. -** -** This function adds a set of changes to the session object that could be -** used to update the table in database zFrom (call this the "from-table") -** so that its content is the same as the table attached to the session -** object (call this the "to-table"). Specifically: -** -**
      -**
    • For each row (primary key) that exists in the to-table but not in -** the from-table, an INSERT record is added to the session object. -** -**
    • For each row (primary key) that exists in the to-table but not in -** the from-table, a DELETE record is added to the session object. -** -**
    • For each row (primary key) that exists in both tables, but features -** different in each, an UPDATE record is added to the session. -**
    -** -** To clarify, if this function is called and then a changeset constructed -** using [sqlite3session_changeset()], then after applying that changeset to -** database zFrom the contents of the two compatible tables would be -** identical. -** -** It an error if database zFrom does not exist or does not contain the -** required compatible table. -** -** If the operation successful, SQLITE_OK is returned. Otherwise, an SQLite -** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg -** may be set to point to a buffer containing an English language error -** message. It is the responsibility of the caller to free this buffer using -** sqlite3_free(). -*/ -int sqlite3session_diff( - sqlite3_session *pSession, - const char *zFromDb, - const char *zTbl, - char **pzErrMsg -); - - -/* -** CAPI3REF: Generate A Patchset From A Session Object -** -** The differences between a patchset and a changeset are that: -** -**
      -**
    • DELETE records consist of the primary key fields only. The -** original values of other fields are omitted. -**
    • The original values of any modified fields are omitted from -** UPDATE records. -**
    -** -** A patchset blob may be used with up to date versions of all -** sqlite3changeset_xxx API functions except for sqlite3changeset_invert(), -** which returns SQLITE_CORRUPT if it is passed a patchset. Similarly, -** attempting to use a patchset blob with old versions of the -** sqlite3changeset_xxx APIs also provokes an SQLITE_CORRUPT error. -** -** Because the non-primary key "old.*" fields are omitted, no -** SQLITE_CHANGESET_DATA conflicts can be detected or reported if a patchset -** is passed to the sqlite3changeset_apply() API. Other conflict types work -** in the same way as for changesets. -** -** Changes within a patchset are ordered in the same way as for changesets -** generated by the sqlite3session_changeset() function (i.e. all changes for -** a single table are grouped together, tables appear in the order in which -** they were attached to the session object). -*/ -int sqlite3session_patchset( - sqlite3_session *pSession, /* Session object */ - int *pnPatchset, /* OUT: Size of buffer at *ppChangeset */ - void **ppPatchset /* OUT: Buffer containing changeset */ -); - -/* -** CAPI3REF: Test if a changeset has recorded any changes. -** -** Return non-zero if no changes to attached tables have been recorded by -** the session object passed as the first argument. Otherwise, if one or -** more changes have been recorded, return zero. -** -** Even if this function returns zero, it is possible that calling -** [sqlite3session_changeset()] on the session handle may still return a -** changeset that contains no changes. This can happen when a row in -** an attached table is modified and then later on the original values -** are restored. However, if this function returns non-zero, then it is -** guaranteed that a call to sqlite3session_changeset() will return a -** changeset containing zero changes. -*/ -int sqlite3session_isempty(sqlite3_session *pSession); - -/* -** CAPI3REF: Create An Iterator To Traverse A Changeset -** -** Create an iterator used to iterate through the contents of a changeset. -** If successful, *pp is set to point to the iterator handle and SQLITE_OK -** is returned. Otherwise, if an error occurs, *pp is set to zero and an -** SQLite error code is returned. -** -** The following functions can be used to advance and query a changeset -** iterator created by this function: -** -**
      -**
    • [sqlite3changeset_next()] -**
    • [sqlite3changeset_op()] -**
    • [sqlite3changeset_new()] -**
    • [sqlite3changeset_old()] -**
    -** -** It is the responsibility of the caller to eventually destroy the iterator -** by passing it to [sqlite3changeset_finalize()]. The buffer containing the -** changeset (pChangeset) must remain valid until after the iterator is -** destroyed. -** -** Assuming the changeset blob was created by one of the -** [sqlite3session_changeset()], [sqlite3changeset_concat()] or -** [sqlite3changeset_invert()] functions, all changes within the changeset -** that apply to a single table are grouped together. This means that when -** an application iterates through a changeset using an iterator created by -** this function, all changes that relate to a single table are visited -** consecutively. There is no chance that the iterator will visit a change -** the applies to table X, then one for table Y, and then later on visit -** another change for table X. -*/ -int sqlite3changeset_start( - sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ - int nChangeset, /* Size of changeset blob in bytes */ - void *pChangeset /* Pointer to blob containing changeset */ -); - - -/* -** CAPI3REF: Advance A Changeset Iterator -** -** This function may only be used with iterators created by function -** [sqlite3changeset_start()]. If it is called on an iterator passed to -** a conflict-handler callback by [sqlite3changeset_apply()], SQLITE_MISUSE -** is returned and the call has no effect. -** -** Immediately after an iterator is created by sqlite3changeset_start(), it -** does not point to any change in the changeset. Assuming the changeset -** is not empty, the first call to this function advances the iterator to -** point to the first change in the changeset. Each subsequent call advances -** the iterator to point to the next change in the changeset (if any). If -** no error occurs and the iterator points to a valid change after a call -** to sqlite3changeset_next() has advanced it, SQLITE_ROW is returned. -** Otherwise, if all changes in the changeset have already been visited, -** SQLITE_DONE is returned. -** -** If an error occurs, an SQLite error code is returned. Possible error -** codes include SQLITE_CORRUPT (if the changeset buffer is corrupt) or -** SQLITE_NOMEM. -*/ -int sqlite3changeset_next(sqlite3_changeset_iter *pIter); - -/* -** CAPI3REF: Obtain The Current Operation From A Changeset Iterator -** -** The pIter argument passed to this function may either be an iterator -** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator -** created by [sqlite3changeset_start()]. In the latter case, the most recent -** call to [sqlite3changeset_next()] must have returned [SQLITE_ROW]. If this -** is not the case, this function returns [SQLITE_MISUSE]. -** -** If argument pzTab is not NULL, then *pzTab is set to point to a -** nul-terminated utf-8 encoded string containing the name of the table -** affected by the current change. The buffer remains valid until either -** sqlite3changeset_next() is called on the iterator or until the -** conflict-handler function returns. If pnCol is not NULL, then *pnCol is -** set to the number of columns in the table affected by the change. If -** pbIncorrect is not NULL, then *pbIndirect is set to true (1) if the change -** is an indirect change, or false (0) otherwise. See the documentation for -** [sqlite3session_indirect()] for a description of direct and indirect -** changes. Finally, if pOp is not NULL, then *pOp is set to one of -** [SQLITE_INSERT], [SQLITE_DELETE] or [SQLITE_UPDATE], depending on the -** type of change that the iterator currently points to. -** -** If no error occurs, SQLITE_OK is returned. If an error does occur, an -** SQLite error code is returned. The values of the output variables may not -** be trusted in this case. -*/ -int sqlite3changeset_op( - sqlite3_changeset_iter *pIter, /* Iterator object */ - const char **pzTab, /* OUT: Pointer to table name */ - int *pnCol, /* OUT: Number of columns in table */ - int *pOp, /* OUT: SQLITE_INSERT, DELETE or UPDATE */ - int *pbIndirect /* OUT: True for an 'indirect' change */ -); - -/* -** CAPI3REF: Obtain The Primary Key Definition Of A Table -** -** For each modified table, a changeset includes the following: -** -**
      -**
    • The number of columns in the table, and -**
    • Which of those columns make up the tables PRIMARY KEY. -**
    -** -** This function is used to find which columns comprise the PRIMARY KEY of -** the table modified by the change that iterator pIter currently points to. -** If successful, *pabPK is set to point to an array of nCol entries, where -** nCol is the number of columns in the table. Elements of *pabPK are set to -** 0x01 if the corresponding column is part of the tables primary key, or -** 0x00 if it is not. -** -** If argument pnCol is not NULL, then *pnCol is set to the number of columns -** in the table. -** -** If this function is called when the iterator does not point to a valid -** entry, SQLITE_MISUSE is returned and the output variables zeroed. Otherwise, -** SQLITE_OK is returned and the output variables populated as described -** above. -*/ -int sqlite3changeset_pk( - sqlite3_changeset_iter *pIter, /* Iterator object */ - unsigned char **pabPK, /* OUT: Array of boolean - true for PK cols */ - int *pnCol /* OUT: Number of entries in output array */ -); - -/* -** CAPI3REF: Obtain old.* Values From A Changeset Iterator -** -** The pIter argument passed to this function may either be an iterator -** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator -** created by [sqlite3changeset_start()]. In the latter case, the most recent -** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. -** Furthermore, it may only be called if the type of change that the iterator -** currently points to is either [SQLITE_DELETE] or [SQLITE_UPDATE]. Otherwise, -** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. -** -** Argument iVal must be greater than or equal to 0, and less than the number -** of columns in the table affected by the current change. Otherwise, -** [SQLITE_RANGE] is returned and *ppValue is set to NULL. -** -** If successful, this function sets *ppValue to point to a protected -** sqlite3_value object containing the iVal'th value from the vector of -** original row values stored as part of the UPDATE or DELETE change and -** returns SQLITE_OK. The name of the function comes from the fact that this -** is similar to the "old.*" columns available to update or delete triggers. -** -** If some other error occurs (e.g. an OOM condition), an SQLite error code -** is returned and *ppValue is set to NULL. -*/ -int sqlite3changeset_old( - sqlite3_changeset_iter *pIter, /* Changeset iterator */ - int iVal, /* Column number */ - sqlite3_value **ppValue /* OUT: Old value (or NULL pointer) */ -); - -/* -** CAPI3REF: Obtain new.* Values From A Changeset Iterator -** -** The pIter argument passed to this function may either be an iterator -** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator -** created by [sqlite3changeset_start()]. In the latter case, the most recent -** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. -** Furthermore, it may only be called if the type of change that the iterator -** currently points to is either [SQLITE_UPDATE] or [SQLITE_INSERT]. Otherwise, -** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. -** -** Argument iVal must be greater than or equal to 0, and less than the number -** of columns in the table affected by the current change. Otherwise, -** [SQLITE_RANGE] is returned and *ppValue is set to NULL. -** -** If successful, this function sets *ppValue to point to a protected -** sqlite3_value object containing the iVal'th value from the vector of -** new row values stored as part of the UPDATE or INSERT change and -** returns SQLITE_OK. If the change is an UPDATE and does not include -** a new value for the requested column, *ppValue is set to NULL and -** SQLITE_OK returned. The name of the function comes from the fact that -** this is similar to the "new.*" columns available to update or delete -** triggers. -** -** If some other error occurs (e.g. an OOM condition), an SQLite error code -** is returned and *ppValue is set to NULL. -*/ -int sqlite3changeset_new( - sqlite3_changeset_iter *pIter, /* Changeset iterator */ - int iVal, /* Column number */ - sqlite3_value **ppValue /* OUT: New value (or NULL pointer) */ -); - -/* -** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator -** -** This function should only be used with iterator objects passed to a -** conflict-handler callback by [sqlite3changeset_apply()] with either -** [SQLITE_CHANGESET_DATA] or [SQLITE_CHANGESET_CONFLICT]. If this function -** is called on any other iterator, [SQLITE_MISUSE] is returned and *ppValue -** is set to NULL. -** -** Argument iVal must be greater than or equal to 0, and less than the number -** of columns in the table affected by the current change. Otherwise, -** [SQLITE_RANGE] is returned and *ppValue is set to NULL. -** -** If successful, this function sets *ppValue to point to a protected -** sqlite3_value object containing the iVal'th value from the -** "conflicting row" associated with the current conflict-handler callback -** and returns SQLITE_OK. -** -** If some other error occurs (e.g. an OOM condition), an SQLite error code -** is returned and *ppValue is set to NULL. -*/ -int sqlite3changeset_conflict( - sqlite3_changeset_iter *pIter, /* Changeset iterator */ - int iVal, /* Column number */ - sqlite3_value **ppValue /* OUT: Value from conflicting row */ -); - -/* -** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations -** -** This function may only be called with an iterator passed to an -** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case -** it sets the output variable to the total number of known foreign key -** violations in the destination database and returns SQLITE_OK. -** -** In all other cases this function returns SQLITE_MISUSE. -*/ -int sqlite3changeset_fk_conflicts( - sqlite3_changeset_iter *pIter, /* Changeset iterator */ - int *pnOut /* OUT: Number of FK violations */ -); - - -/* -** CAPI3REF: Finalize A Changeset Iterator -** -** This function is used to finalize an iterator allocated with -** [sqlite3changeset_start()]. -** -** This function should only be called on iterators created using the -** [sqlite3changeset_start()] function. If an application calls this -** function with an iterator passed to a conflict-handler by -** [sqlite3changeset_apply()], [SQLITE_MISUSE] is immediately returned and the -** call has no effect. -** -** If an error was encountered within a call to an sqlite3changeset_xxx() -** function (for example an [SQLITE_CORRUPT] in [sqlite3changeset_next()] or an -** [SQLITE_NOMEM] in [sqlite3changeset_new()]) then an error code corresponding -** to that error is returned by this function. Otherwise, SQLITE_OK is -** returned. This is to allow the following pattern (pseudo-code): -** -** sqlite3changeset_start(); -** while( SQLITE_ROW==sqlite3changeset_next() ){ -** // Do something with change. -** } -** rc = sqlite3changeset_finalize(); -** if( rc!=SQLITE_OK ){ -** // An error has occurred -** } -*/ -int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); - -/* -** CAPI3REF: Invert A Changeset -** -** This function is used to "invert" a changeset object. Applying an inverted -** changeset to a database reverses the effects of applying the uninverted -** changeset. Specifically: -** -**
      -**
    • Each DELETE change is changed to an INSERT, and -**
    • Each INSERT change is changed to a DELETE, and -**
    • For each UPDATE change, the old.* and new.* values are exchanged. -**
    -** -** This function does not change the order in which changes appear within -** the changeset. It merely reverses the sense of each individual change. -** -** If successful, a pointer to a buffer containing the inverted changeset -** is stored in *ppOut, the size of the same buffer is stored in *pnOut, and -** SQLITE_OK is returned. If an error occurs, both *pnOut and *ppOut are -** zeroed and an SQLite error code returned. -** -** It is the responsibility of the caller to eventually call sqlite3_free() -** on the *ppOut pointer to free the buffer allocation following a successful -** call to this function. -** -** WARNING/TODO: This function currently assumes that the input is a valid -** changeset. If it is not, the results are undefined. -*/ -int sqlite3changeset_invert( - int nIn, const void *pIn, /* Input changeset */ - int *pnOut, void **ppOut /* OUT: Inverse of input */ -); - -/* -** CAPI3REF: Concatenate Two Changeset Objects -** -** This function is used to concatenate two changesets, A and B, into a -** single changeset. The result is a changeset equivalent to applying -** changeset A followed by changeset B. -** -** This function combines the two input changesets using an -** sqlite3_changegroup object. Calling it produces similar results as the -** following code fragment: -** -** sqlite3_changegroup *pGrp; -** rc = sqlite3_changegroup_new(&pGrp); -** if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA); -** if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nB, pB); -** if( rc==SQLITE_OK ){ -** rc = sqlite3changegroup_output(pGrp, pnOut, ppOut); -** }else{ -** *ppOut = 0; -** *pnOut = 0; -** } -** -** Refer to the sqlite3_changegroup documentation below for details. -*/ -int sqlite3changeset_concat( - int nA, /* Number of bytes in buffer pA */ - void *pA, /* Pointer to buffer containing changeset A */ - int nB, /* Number of bytes in buffer pB */ - void *pB, /* Pointer to buffer containing changeset B */ - int *pnOut, /* OUT: Number of bytes in output changeset */ - void **ppOut /* OUT: Buffer containing output changeset */ -); - - -/* -** CAPI3REF: Changegroup Handle -*/ -typedef struct sqlite3_changegroup sqlite3_changegroup; - -/* -** CAPI3REF: Create A New Changegroup Object -** -** An sqlite3_changegroup object is used to combine two or more changesets -** (or patchsets) into a single changeset (or patchset). A single changegroup -** object may combine changesets or patchsets, but not both. The output is -** always in the same format as the input. -** -** If successful, this function returns SQLITE_OK and populates (*pp) with -** a pointer to a new sqlite3_changegroup object before returning. The caller -** should eventually free the returned object using a call to -** sqlite3changegroup_delete(). If an error occurs, an SQLite error code -** (i.e. SQLITE_NOMEM) is returned and *pp is set to NULL. -** -** The usual usage pattern for an sqlite3_changegroup object is as follows: -** -**
      -**
    • It is created using a call to sqlite3changegroup_new(). -** -**
    • Zero or more changesets (or patchsets) are added to the object -** by calling sqlite3changegroup_add(). -** -**
    • The result of combining all input changesets together is obtained -** by the application via a call to sqlite3changegroup_output(). -** -**
    • The object is deleted using a call to sqlite3changegroup_delete(). -**
    -** -** Any number of calls to add() and output() may be made between the calls to -** new() and delete(), and in any order. -** -** As well as the regular sqlite3changegroup_add() and -** sqlite3changegroup_output() functions, also available are the streaming -** versions sqlite3changegroup_add_strm() and sqlite3changegroup_output_strm(). -*/ -int sqlite3changegroup_new(sqlite3_changegroup **pp); - -/* -** CAPI3REF: Add A Changeset To A Changegroup -** -** Add all changes within the changeset (or patchset) in buffer pData (size -** nData bytes) to the changegroup. -** -** If the buffer contains a patchset, then all prior calls to this function -** on the same changegroup object must also have specified patchsets. Or, if -** the buffer contains a changeset, so must have the earlier calls to this -** function. Otherwise, SQLITE_ERROR is returned and no changes are added -** to the changegroup. -** -** Rows within the changeset and changegroup are identified by the values in -** their PRIMARY KEY columns. A change in the changeset is considered to -** apply to the same row as a change already present in the changegroup if -** the two rows have the same primary key. -** -** Changes to rows that do not already appear in the changegroup are -** simply copied into it. Or, if both the new changeset and the changegroup -** contain changes that apply to a single row, the final contents of the -** changegroup depends on the type of each change, as follows: -** -** -** -** -**
    Existing Change New Change Output Change -**
    INSERT INSERT -** The new change is ignored. This case does not occur if the new -** changeset was recorded immediately after the changesets already -** added to the changegroup. -**
    INSERT UPDATE -** The INSERT change remains in the changegroup. The values in the -** INSERT change are modified as if the row was inserted by the -** existing change and then updated according to the new change. -**
    INSERT DELETE -** The existing INSERT is removed from the changegroup. The DELETE is -** not added. -**
    UPDATE INSERT -** The new change is ignored. This case does not occur if the new -** changeset was recorded immediately after the changesets already -** added to the changegroup. -**
    UPDATE UPDATE -** The existing UPDATE remains within the changegroup. It is amended -** so that the accompanying values are as if the row was updated once -** by the existing change and then again by the new change. -**
    UPDATE DELETE -** The existing UPDATE is replaced by the new DELETE within the -** changegroup. -**
    DELETE INSERT -** If one or more of the column values in the row inserted by the -** new change differ from those in the row deleted by the existing -** change, the existing DELETE is replaced by an UPDATE within the -** changegroup. Otherwise, if the inserted row is exactly the same -** as the deleted row, the existing DELETE is simply discarded. -**
    DELETE UPDATE -** The new change is ignored. This case does not occur if the new -** changeset was recorded immediately after the changesets already -** added to the changegroup. -**
    DELETE DELETE -** The new change is ignored. This case does not occur if the new -** changeset was recorded immediately after the changesets already -** added to the changegroup. -**
    -** -** If the new changeset contains changes to a table that is already present -** in the changegroup, then the number of columns and the position of the -** primary key columns for the table must be consistent. If this is not the -** case, this function fails with SQLITE_SCHEMA. If the input changeset -** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is -** returned. Or, if an out-of-memory condition occurs during processing, this -** function returns SQLITE_NOMEM. In all cases, if an error occurs the -** final contents of the changegroup is undefined. -** -** If no error occurs, SQLITE_OK is returned. -*/ -int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); - -/* -** CAPI3REF: Obtain A Composite Changeset From A Changegroup -** -** Obtain a buffer containing a changeset (or patchset) representing the -** current contents of the changegroup. If the inputs to the changegroup -** were themselves changesets, the output is a changeset. Or, if the -** inputs were patchsets, the output is also a patchset. -** -** As with the output of the sqlite3session_changeset() and -** sqlite3session_patchset() functions, all changes related to a single -** table are grouped together in the output of this function. Tables appear -** in the same order as for the very first changeset added to the changegroup. -** If the second or subsequent changesets added to the changegroup contain -** changes for tables that do not appear in the first changeset, they are -** appended onto the end of the output changeset, again in the order in -** which they are first encountered. -** -** If an error occurs, an SQLite error code is returned and the output -** variables (*pnData) and (*ppData) are set to 0. Otherwise, SQLITE_OK -** is returned and the output variables are set to the size of and a -** pointer to the output buffer, respectively. In this case it is the -** responsibility of the caller to eventually free the buffer using a -** call to sqlite3_free(). -*/ -int sqlite3changegroup_output( - sqlite3_changegroup*, - int *pnData, /* OUT: Size of output buffer in bytes */ - void **ppData /* OUT: Pointer to output buffer */ -); - -/* -** CAPI3REF: Delete A Changegroup Object -*/ -void sqlite3changegroup_delete(sqlite3_changegroup*); - -/* -** CAPI3REF: Apply A Changeset To A Database -** -** Apply a changeset to a database. This function attempts to update the -** "main" database attached to handle db with the changes found in the -** changeset passed via the second and third arguments. -** -** The fourth argument (xFilter) passed to this function is the "filter -** callback". If it is not NULL, then for each table affected by at least one -** change in the changeset, the filter callback is invoked with -** the table name as the second argument, and a copy of the context pointer -** passed as the sixth argument to this function as the first. If the "filter -** callback" returns zero, then no attempt is made to apply any changes to -** the table. Otherwise, if the return value is non-zero or the xFilter -** argument to this function is NULL, all changes related to the table are -** attempted. -** -** For each table that is not excluded by the filter callback, this function -** tests that the target database contains a compatible table. A table is -** considered compatible if all of the following are true: -** -**
      -**
    • The table has the same name as the name recorded in the -** changeset, and -**
    • The table has the same number of columns as recorded in the -** changeset, and -**
    • The table has primary key columns in the same position as -** recorded in the changeset. -**
    -** -** If there is no compatible table, it is not an error, but none of the -** changes associated with the table are applied. A warning message is issued -** via the sqlite3_log() mechanism with the error code SQLITE_SCHEMA. At most -** one such warning is issued for each table in the changeset. -** -** For each change for which there is a compatible table, an attempt is made -** to modify the table contents according to the UPDATE, INSERT or DELETE -** change. If a change cannot be applied cleanly, the conflict handler -** function passed as the fifth argument to sqlite3changeset_apply() may be -** invoked. A description of exactly when the conflict handler is invoked for -** each type of change is below. -** -** Unlike the xFilter argument, xConflict may not be passed NULL. The results -** of passing anything other than a valid function pointer as the xConflict -** argument are undefined. -** -** Each time the conflict handler function is invoked, it must return one -** of [SQLITE_CHANGESET_OMIT], [SQLITE_CHANGESET_ABORT] or -** [SQLITE_CHANGESET_REPLACE]. SQLITE_CHANGESET_REPLACE may only be returned -** if the second argument passed to the conflict handler is either -** SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If the conflict-handler -** returns an illegal value, any changes already made are rolled back and -** the call to sqlite3changeset_apply() returns SQLITE_MISUSE. Different -** actions are taken by sqlite3changeset_apply() depending on the value -** returned by each invocation of the conflict-handler function. Refer to -** the documentation for the three -** [SQLITE_CHANGESET_OMIT|available return values] for details. -** -**
    -**
    DELETE Changes
    -** For each DELETE change, this function checks if the target database -** contains a row with the same primary key value (or values) as the -** original row values stored in the changeset. If it does, and the values -** stored in all non-primary key columns also match the values stored in -** the changeset the row is deleted from the target database. -** -** If a row with matching primary key values is found, but one or more of -** the non-primary key fields contains a value different from the original -** row value stored in the changeset, the conflict-handler function is -** invoked with [SQLITE_CHANGESET_DATA] as the second argument. -** -** If no row with matching primary key values is found in the database, -** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] -** passed as the second argument. -** -** If the DELETE operation is attempted, but SQLite returns SQLITE_CONSTRAINT -** (which can only happen if a foreign key constraint is violated), the -** conflict-handler function is invoked with [SQLITE_CHANGESET_CONSTRAINT] -** passed as the second argument. This includes the case where the DELETE -** operation is attempted because an earlier call to the conflict handler -** function returned [SQLITE_CHANGESET_REPLACE]. -** -**
    INSERT Changes
    -** For each INSERT change, an attempt is made to insert the new row into -** the database. -** -** If the attempt to insert the row fails because the database already -** contains a row with the same primary key values, the conflict handler -** function is invoked with the second argument set to -** [SQLITE_CHANGESET_CONFLICT]. -** -** If the attempt to insert the row fails because of some other constraint -** violation (e.g. NOT NULL or UNIQUE), the conflict handler function is -** invoked with the second argument set to [SQLITE_CHANGESET_CONSTRAINT]. -** This includes the case where the INSERT operation is re-attempted because -** an earlier call to the conflict handler function returned -** [SQLITE_CHANGESET_REPLACE]. -** -**
    UPDATE Changes
    -** For each UPDATE change, this function checks if the target database -** contains a row with the same primary key value (or values) as the -** original row values stored in the changeset. If it does, and the values -** stored in all non-primary key columns also match the values stored in -** the changeset the row is updated within the target database. -** -** If a row with matching primary key values is found, but one or more of -** the non-primary key fields contains a value different from an original -** row value stored in the changeset, the conflict-handler function is -** invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since -** UPDATE changes only contain values for non-primary key fields that are -** to be modified, only those fields need to match the original values to -** avoid the SQLITE_CHANGESET_DATA conflict-handler callback. -** -** If no row with matching primary key values is found in the database, -** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] -** passed as the second argument. -** -** If the UPDATE operation is attempted, but SQLite returns -** SQLITE_CONSTRAINT, the conflict-handler function is invoked with -** [SQLITE_CHANGESET_CONSTRAINT] passed as the second argument. -** This includes the case where the UPDATE operation is attempted after -** an earlier call to the conflict handler function returned -** [SQLITE_CHANGESET_REPLACE]. -**
    -** -** It is safe to execute SQL statements, including those that write to the -** table that the callback related to, from within the xConflict callback. -** This can be used to further customize the applications conflict -** resolution strategy. -** -** All changes made by this function are enclosed in a savepoint transaction. -** If any other error (aside from a constraint failure when attempting to -** write to the target database) occurs, then the savepoint transaction is -** rolled back, restoring the target database to its original state, and an -** SQLite error code returned. -*/ -int sqlite3changeset_apply( - sqlite3 *db, /* Apply change to "main" db of this handle */ - int nChangeset, /* Size of changeset in bytes */ - void *pChangeset, /* Changeset blob */ - int(*xFilter)( - void *pCtx, /* Copy of sixth arg to _apply() */ - const char *zTab /* Table name */ - ), - int(*xConflict)( - void *pCtx, /* Copy of sixth arg to _apply() */ - int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ - sqlite3_changeset_iter *p /* Handle describing change and conflict */ - ), - void *pCtx /* First argument passed to xConflict */ -); - -/* -** CAPI3REF: Constants Passed To The Conflict Handler -** -** Values that may be passed as the second argument to a conflict-handler. -** -**
    -**
    SQLITE_CHANGESET_DATA
    -** The conflict handler is invoked with CHANGESET_DATA as the second argument -** when processing a DELETE or UPDATE change if a row with the required -** PRIMARY KEY fields is present in the database, but one or more other -** (non primary-key) fields modified by the update do not contain the -** expected "before" values. -** -** The conflicting row, in this case, is the database row with the matching -** primary key. -** -**
    SQLITE_CHANGESET_NOTFOUND
    -** The conflict handler is invoked with CHANGESET_NOTFOUND as the second -** argument when processing a DELETE or UPDATE change if a row with the -** required PRIMARY KEY fields is not present in the database. -** -** There is no conflicting row in this case. The results of invoking the -** sqlite3changeset_conflict() API are undefined. -** -**
    SQLITE_CHANGESET_CONFLICT
    -** CHANGESET_CONFLICT is passed as the second argument to the conflict -** handler while processing an INSERT change if the operation would result -** in duplicate primary key values. -** -** The conflicting row in this case is the database row with the matching -** primary key. -** -**
    SQLITE_CHANGESET_FOREIGN_KEY
    -** If foreign key handling is enabled, and applying a changeset leaves the -** database in a state containing foreign key violations, the conflict -** handler is invoked with CHANGESET_FOREIGN_KEY as the second argument -** exactly once before the changeset is committed. If the conflict handler -** returns CHANGESET_OMIT, the changes, including those that caused the -** foreign key constraint violation, are committed. Or, if it returns -** CHANGESET_ABORT, the changeset is rolled back. -** -** No current or conflicting row information is provided. The only function -** it is possible to call on the supplied sqlite3_changeset_iter handle -** is sqlite3changeset_fk_conflicts(). -** -**
    SQLITE_CHANGESET_CONSTRAINT
    -** If any other constraint violation occurs while applying a change (i.e. -** a UNIQUE, CHECK or NOT NULL constraint), the conflict handler is -** invoked with CHANGESET_CONSTRAINT as the second argument. -** -** There is no conflicting row in this case. The results of invoking the -** sqlite3changeset_conflict() API are undefined. -** -**
    -*/ -#define SQLITE_CHANGESET_DATA 1 -#define SQLITE_CHANGESET_NOTFOUND 2 -#define SQLITE_CHANGESET_CONFLICT 3 -#define SQLITE_CHANGESET_CONSTRAINT 4 -#define SQLITE_CHANGESET_FOREIGN_KEY 5 - -/* -** CAPI3REF: Constants Returned By The Conflict Handler -** -** A conflict handler callback must return one of the following three values. -** -**
    -**
    SQLITE_CHANGESET_OMIT
    -** If a conflict handler returns this value no special action is taken. The -** change that caused the conflict is not applied. The session module -** continues to the next change in the changeset. -** -**
    SQLITE_CHANGESET_REPLACE
    -** This value may only be returned if the second argument to the conflict -** handler was SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If this -** is not the case, any changes applied so far are rolled back and the -** call to sqlite3changeset_apply() returns SQLITE_MISUSE. -** -** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_DATA conflict -** handler, then the conflicting row is either updated or deleted, depending -** on the type of change. -** -** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_CONFLICT conflict -** handler, then the conflicting row is removed from the database and a -** second attempt to apply the change is made. If this second attempt fails, -** the original row is restored to the database before continuing. -** -**
    SQLITE_CHANGESET_ABORT
    -** If this value is returned, any changes applied so far are rolled back -** and the call to sqlite3changeset_apply() returns SQLITE_ABORT. -**
    -*/ -#define SQLITE_CHANGESET_OMIT 0 -#define SQLITE_CHANGESET_REPLACE 1 -#define SQLITE_CHANGESET_ABORT 2 - -/* -** CAPI3REF: Streaming Versions of API functions. -** -** The six streaming API xxx_strm() functions serve similar purposes to the -** corresponding non-streaming API functions: -** -** -** -**
    Streaming functionNon-streaming equivalent
    sqlite3changeset_apply_str[sqlite3changeset_apply] -**
    sqlite3changeset_concat_str[sqlite3changeset_concat] -**
    sqlite3changeset_invert_str[sqlite3changeset_invert] -**
    sqlite3changeset_start_str[sqlite3changeset_start] -**
    sqlite3session_changeset_str[sqlite3session_changeset] -**
    sqlite3session_patchset_str[sqlite3session_patchset] -**
    -** -** Non-streaming functions that accept changesets (or patchsets) as input -** require that the entire changeset be stored in a single buffer in memory. -** Similarly, those that return a changeset or patchset do so by returning -** a pointer to a single large buffer allocated using sqlite3_malloc(). -** Normally this is convenient. However, if an application running in a -** low-memory environment is required to handle very large changesets, the -** large contiguous memory allocations required can become onerous. -** -** In order to avoid this problem, instead of a single large buffer, input -** is passed to a streaming API functions by way of a callback function that -** the sessions module invokes to incrementally request input data as it is -** required. In all cases, a pair of API function parameters such as -** -**
    -**        int nChangeset,
    -**        void *pChangeset,
    -**  
    -** -** Is replaced by: -** -**
    -**        int (*xInput)(void *pIn, void *pData, int *pnData),
    -**        void *pIn,
    -**  
    -** -** Each time the xInput callback is invoked by the sessions module, the first -** argument passed is a copy of the supplied pIn context pointer. The second -** argument, pData, points to a buffer (*pnData) bytes in size. Assuming no -** error occurs the xInput method should copy up to (*pnData) bytes of data -** into the buffer and set (*pnData) to the actual number of bytes copied -** before returning SQLITE_OK. If the input is completely exhausted, (*pnData) -** should be set to zero to indicate this. Or, if an error occurs, an SQLite -** error code should be returned. In all cases, if an xInput callback returns -** an error, all processing is abandoned and the streaming API function -** returns a copy of the error code to the caller. -** -** In the case of sqlite3changeset_start_strm(), the xInput callback may be -** invoked by the sessions module at any point during the lifetime of the -** iterator. If such an xInput callback returns an error, the iterator enters -** an error state, whereby all subsequent calls to iterator functions -** immediately fail with the same error code as returned by xInput. -** -** Similarly, streaming API functions that return changesets (or patchsets) -** return them in chunks by way of a callback function instead of via a -** pointer to a single large buffer. In this case, a pair of parameters such -** as: -** -**
    -**        int *pnChangeset,
    -**        void **ppChangeset,
    -**  
    -** -** Is replaced by: -** -**
    -**        int (*xOutput)(void *pOut, const void *pData, int nData),
    -**        void *pOut
    -**  
    -** -** The xOutput callback is invoked zero or more times to return data to -** the application. The first parameter passed to each call is a copy of the -** pOut pointer supplied by the application. The second parameter, pData, -** points to a buffer nData bytes in size containing the chunk of output -** data being returned. If the xOutput callback successfully processes the -** supplied data, it should return SQLITE_OK to indicate success. Otherwise, -** it should return some other SQLite error code. In this case processing -** is immediately abandoned and the streaming API function returns a copy -** of the xOutput error code to the application. -** -** The sessions module never invokes an xOutput callback with the third -** parameter set to a value less than or equal to zero. Other than this, -** no guarantees are made as to the size of the chunks of data returned. -*/ -int sqlite3changeset_apply_strm( - sqlite3 *db, /* Apply change to "main" db of this handle */ - int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ - void *pIn, /* First arg for xInput */ - int(*xFilter)( - void *pCtx, /* Copy of sixth arg to _apply() */ - const char *zTab /* Table name */ - ), - int(*xConflict)( - void *pCtx, /* Copy of sixth arg to _apply() */ - int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ - sqlite3_changeset_iter *p /* Handle describing change and conflict */ - ), - void *pCtx /* First argument passed to xConflict */ -); -int sqlite3changeset_concat_strm( - int (*xInputA)(void *pIn, void *pData, int *pnData), - void *pInA, - int (*xInputB)(void *pIn, void *pData, int *pnData), - void *pInB, - int (*xOutput)(void *pOut, const void *pData, int nData), - void *pOut -); -int sqlite3changeset_invert_strm( - int (*xInput)(void *pIn, void *pData, int *pnData), - void *pIn, - int (*xOutput)(void *pOut, const void *pData, int nData), - void *pOut -); -int sqlite3changeset_start_strm( - sqlite3_changeset_iter **pp, - int (*xInput)(void *pIn, void *pData, int *pnData), - void *pIn -); -int sqlite3session_changeset_strm( - sqlite3_session *pSession, - int (*xOutput)(void *pOut, const void *pData, int nData), - void *pOut -); -int sqlite3session_patchset_strm( - sqlite3_session *pSession, - int (*xOutput)(void *pOut, const void *pData, int nData), - void *pOut -); -int sqlite3changegroup_add_strm(sqlite3_changegroup*, - int (*xInput)(void *pIn, void *pData, int *pnData), - void *pIn -); -int sqlite3changegroup_output_strm(sqlite3_changegroup*, - int (*xOutput)(void *pOut, const void *pData, int nData), - void *pOut -); - - -/* -** Make sure we can call this stuff from C++. -*/ -#if 0 -} -#endif - -#endif /* !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) */ - -/******** End of sqlite3session.h *********/ -/******** Begin file fts5.h *********/ -/* -** 2014 May 31 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** Interfaces to extend FTS5. Using the interfaces defined in this file, -** FTS5 may be extended with: -** -** * custom tokenizers, and -** * custom auxiliary functions. -*/ - - -#ifndef _FTS5_H -#define _FTS5_H - - -#if 0 -extern "C" { -#endif - -/************************************************************************* -** CUSTOM AUXILIARY FUNCTIONS -** -** Virtual table implementations may overload SQL functions by implementing -** the sqlite3_module.xFindFunction() method. -*/ - -typedef struct Fts5ExtensionApi Fts5ExtensionApi; -typedef struct Fts5Context Fts5Context; -typedef struct Fts5PhraseIter Fts5PhraseIter; - -typedef void (*fts5_extension_function)( - const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ - Fts5Context *pFts, /* First arg to pass to pApi functions */ - sqlite3_context *pCtx, /* Context for returning result/error */ - int nVal, /* Number of values in apVal[] array */ - sqlite3_value **apVal /* Array of trailing arguments */ -); - -struct Fts5PhraseIter { - const unsigned char *a; - const unsigned char *b; -}; - -/* -** EXTENSION API FUNCTIONS -** -** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. -** -** xColumnTotalSize(pFts, iCol, pnToken): -** If parameter iCol is less than zero, set output variable *pnToken -** to the total number of tokens in the FTS5 table. Or, if iCol is -** non-negative but less than the number of columns in the table, return -** the total number of tokens in column iCol, considering all rows in -** the FTS5 table. -** -** If parameter iCol is greater than or equal to the number of columns -** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. -** an OOM condition or IO error), an appropriate SQLite error code is -** returned. -** -** xColumnCount(pFts): -** Return the number of columns in the table. -** -** xColumnSize(pFts, iCol, pnToken): -** If parameter iCol is less than zero, set output variable *pnToken -** to the total number of tokens in the current row. Or, if iCol is -** non-negative but less than the number of columns in the table, set -** *pnToken to the number of tokens in column iCol of the current row. -** -** If parameter iCol is greater than or equal to the number of columns -** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. -** an OOM condition or IO error), an appropriate SQLite error code is -** returned. -** -** This function may be quite inefficient if used with an FTS5 table -** created with the "columnsize=0" option. -** -** xColumnText: -** This function attempts to retrieve the text of column iCol of the -** current document. If successful, (*pz) is set to point to a buffer -** containing the text in utf-8 encoding, (*pn) is set to the size in bytes -** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, -** if an error occurs, an SQLite error code is returned and the final values -** of (*pz) and (*pn) are undefined. -** -** xPhraseCount: -** Returns the number of phrases in the current query expression. -** -** xPhraseSize: -** Returns the number of tokens in phrase iPhrase of the query. Phrases -** are numbered starting from zero. -** -** xInstCount: -** Set *pnInst to the total number of occurrences of all phrases within -** the query within the current row. Return SQLITE_OK if successful, or -** an error code (i.e. SQLITE_NOMEM) if an error occurs. -** -** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" or "detail=column" option. If the FTS5 table is created -** with either "detail=none" or "detail=column" and "content=" option -** (i.e. if it is a contentless table), then this API always returns 0. -** -** xInst: -** Query for the details of phrase match iIdx within the current row. -** Phrase matches are numbered starting from zero, so the iIdx argument -** should be greater than or equal to zero and smaller than the value -** output by xInstCount(). -** -** Usually, output parameter *piPhrase is set to the phrase number, *piCol -** to the column in which it occurs and *piOff the token offset of the -** first token of the phrase. The exception is if the table was created -** with the offsets=0 option specified. In this case *piOff is always -** set to -1. -** -** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) -** if an error occurs. -** -** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" or "detail=column" option. -** -** xRowid: -** Returns the rowid of the current row. -** -** xTokenize: -** Tokenize text using the tokenizer belonging to the FTS5 table. -** -** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): -** This API function is used to query the FTS table for phrase iPhrase -** of the current query. Specifically, a query equivalent to: -** -** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid -** -** with $p set to a phrase equivalent to the phrase iPhrase of the -** current query is executed. Any column filter that applies to -** phrase iPhrase of the current query is included in $p. For each -** row visited, the callback function passed as the fourth argument -** is invoked. The context and API objects passed to the callback -** function may be used to access the properties of each matched row. -** Invoking Api.xUserData() returns a copy of the pointer passed as -** the third argument to pUserData. -** -** If the callback function returns any value other than SQLITE_OK, the -** query is abandoned and the xQueryPhrase function returns immediately. -** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. -** Otherwise, the error code is propagated upwards. -** -** If the query runs to completion without incident, SQLITE_OK is returned. -** Or, if some error occurs before the query completes or is aborted by -** the callback, an SQLite error code is returned. -** -** -** xSetAuxdata(pFts5, pAux, xDelete) -** -** Save the pointer passed as the second argument as the extension functions -** "auxiliary data". The pointer may then be retrieved by the current or any -** future invocation of the same fts5 extension function made as part of -** of the same MATCH query using the xGetAuxdata() API. -** -** Each extension function is allocated a single auxiliary data slot for -** each FTS query (MATCH expression). If the extension function is invoked -** more than once for a single FTS query, then all invocations share a -** single auxiliary data context. -** -** If there is already an auxiliary data pointer when this function is -** invoked, then it is replaced by the new pointer. If an xDelete callback -** was specified along with the original pointer, it is invoked at this -** point. -** -** The xDelete callback, if one is specified, is also invoked on the -** auxiliary data pointer after the FTS5 query has finished. -** -** If an error (e.g. an OOM condition) occurs within this function, an -** the auxiliary data is set to NULL and an error code returned. If the -** xDelete parameter was not NULL, it is invoked on the auxiliary data -** pointer before returning. -** -** -** xGetAuxdata(pFts5, bClear) -** -** Returns the current auxiliary data pointer for the fts5 extension -** function. See the xSetAuxdata() method for details. -** -** If the bClear argument is non-zero, then the auxiliary data is cleared -** (set to NULL) before this function returns. In this case the xDelete, -** if any, is not invoked. -** -** -** xRowCount(pFts5, pnRow) -** -** This function is used to retrieve the total number of rows in the table. -** In other words, the same value that would be returned by: -** -** SELECT count(*) FROM ftstable; -** -** xPhraseFirst() -** This function is used, along with type Fts5PhraseIter and the xPhraseNext -** method, to iterate through all instances of a single query phrase within -** the current row. This is the same information as is accessible via the -** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient -** to use, this API may be faster under some circumstances. To iterate -** through instances of phrase iPhrase, use the following code: -** -** Fts5PhraseIter iter; -** int iCol, iOff; -** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); -** iCol>=0; -** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) -** ){ -** // An instance of phrase iPhrase at offset iOff of column iCol -** } -** -** The Fts5PhraseIter structure is defined above. Applications should not -** modify this structure directly - it should only be used as shown above -** with the xPhraseFirst() and xPhraseNext() API methods (and by -** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). -** -** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" or "detail=column" option. If the FTS5 table is created -** with either "detail=none" or "detail=column" and "content=" option -** (i.e. if it is a contentless table), then this API always iterates -** through an empty set (all calls to xPhraseFirst() set iCol to -1). -** -** xPhraseNext() -** See xPhraseFirst above. -** -** xPhraseFirstColumn() -** This function and xPhraseNextColumn() are similar to the xPhraseFirst() -** and xPhraseNext() APIs described above. The difference is that instead -** of iterating through all instances of a phrase in the current row, these -** APIs are used to iterate through the set of columns in the current row -** that contain one or more instances of a specified phrase. For example: -** -** Fts5PhraseIter iter; -** int iCol; -** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); -** iCol>=0; -** pApi->xPhraseNextColumn(pFts, &iter, &iCol) -** ){ -** // Column iCol contains at least one instance of phrase iPhrase -** } -** -** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" option. If the FTS5 table is created with either -** "detail=none" "content=" option (i.e. if it is a contentless table), -** then this API always iterates through an empty set (all calls to -** xPhraseFirstColumn() set iCol to -1). -** -** The information accessed using this API and its companion -** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext -** (or xInst/xInstCount). The chief advantage of this API is that it is -** significantly more efficient than those alternatives when used with -** "detail=column" tables. -** -** xPhraseNextColumn() -** See xPhraseFirstColumn above. -*/ -struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ - - void *(*xUserData)(Fts5Context*); - - int (*xColumnCount)(Fts5Context*); - int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); - int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); - - int (*xTokenize)(Fts5Context*, - const char *pText, int nText, /* Text to tokenize */ - void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ - ); - - int (*xPhraseCount)(Fts5Context*); - int (*xPhraseSize)(Fts5Context*, int iPhrase); - - int (*xInstCount)(Fts5Context*, int *pnInst); - int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); - - sqlite3_int64 (*xRowid)(Fts5Context*); - int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); - int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); - - int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, - int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) - ); - int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); - void *(*xGetAuxdata)(Fts5Context*, int bClear); - - int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); - void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); - - int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); - void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); -}; - -/* -** CUSTOM AUXILIARY FUNCTIONS -*************************************************************************/ - -/************************************************************************* -** CUSTOM TOKENIZERS -** -** Applications may also register custom tokenizer types. A tokenizer -** is registered by providing fts5 with a populated instance of the -** following structure. All structure methods must be defined, setting -** any member of the fts5_tokenizer struct to NULL leads to undefined -** behaviour. The structure methods are expected to function as follows: -** -** xCreate: -** This function is used to allocate and initialize a tokenizer instance. -** A tokenizer instance is required to actually tokenize text. -** -** The first argument passed to this function is a copy of the (void*) -** pointer provided by the application when the fts5_tokenizer object -** was registered with FTS5 (the third argument to xCreateTokenizer()). -** The second and third arguments are an array of nul-terminated strings -** containing the tokenizer arguments, if any, specified following the -** tokenizer name as part of the CREATE VIRTUAL TABLE statement used -** to create the FTS5 table. -** -** The final argument is an output variable. If successful, (*ppOut) -** should be set to point to the new tokenizer handle and SQLITE_OK -** returned. If an error occurs, some value other than SQLITE_OK should -** be returned. In this case, fts5 assumes that the final value of *ppOut -** is undefined. -** -** xDelete: -** This function is invoked to delete a tokenizer handle previously -** allocated using xCreate(). Fts5 guarantees that this function will -** be invoked exactly once for each successful call to xCreate(). -** -** xTokenize: -** This function is expected to tokenize the nText byte string indicated -** by argument pText. pText may or may not be nul-terminated. The first -** argument passed to this function is a pointer to an Fts5Tokenizer object -** returned by an earlier call to xCreate(). -** -** The second argument indicates the reason that FTS5 is requesting -** tokenization of the supplied text. This is always one of the following -** four values: -** -**
    • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into -** or removed from the FTS table. The tokenizer is being invoked to -** determine the set of tokens to add to (or delete from) the -** FTS index. -** -**
    • FTS5_TOKENIZE_QUERY - A MATCH query is being executed -** against the FTS index. The tokenizer is being called to tokenize -** a bareword or quoted string specified as part of the query. -** -**
    • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as -** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is -** followed by a "*" character, indicating that the last token -** returned by the tokenizer will be treated as a token prefix. -** -**
    • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to -** satisfy an fts5_api.xTokenize() request made by an auxiliary -** function. Or an fts5_api.xColumnSize() request made by the same -** on a columnsize=0 database. -**
    -** -** For each token in the input string, the supplied callback xToken() must -** be invoked. The first argument to it should be a copy of the pointer -** passed as the second argument to xTokenize(). The third and fourth -** arguments are a pointer to a buffer containing the token text, and the -** size of the token in bytes. The 4th and 5th arguments are the byte offsets -** of the first byte of and first byte immediately following the text from -** which the token is derived within the input. -** -** The second argument passed to the xToken() callback ("tflags") should -** normally be set to 0. The exception is if the tokenizer supports -** synonyms. In this case see the discussion below for details. -** -** FTS5 assumes the xToken() callback is invoked for each token in the -** order that they occur within the input text. -** -** If an xToken() callback returns any value other than SQLITE_OK, then -** the tokenization should be abandoned and the xTokenize() method should -** immediately return a copy of the xToken() return value. Or, if the -** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, -** if an error occurs with the xTokenize() implementation itself, it -** may abandon the tokenization and return any error code other than -** SQLITE_OK or SQLITE_DONE. -** -** SYNONYM SUPPORT -** -** Custom tokenizers may also support synonyms. Consider a case in which a -** user wishes to query for a phrase such as "first place". Using the -** built-in tokenizers, the FTS5 query 'first + place' will match instances -** of "first place" within the document set, but not alternative forms -** such as "1st place". In some applications, it would be better to match -** all instances of "first place" or "1st place" regardless of which form -** the user specified in the MATCH query text. -** -** There are several ways to approach this in FTS5: -** -**
    1. By mapping all synonyms to a single token. In this case, the -** In the above example, this means that the tokenizer returns the -** same token for inputs "first" and "1st". Say that token is in -** fact "first", so that when the user inserts the document "I won -** 1st place" entries are added to the index for tokens "i", "won", -** "first" and "place". If the user then queries for '1st + place', -** the tokenizer substitutes "first" for "1st" and the query works -** as expected. -** -**
    2. By adding multiple synonyms for a single term to the FTS index. -** In this case, when tokenizing query text, the tokenizer may -** provide multiple synonyms for a single term within the document. -** FTS5 then queries the index for each synonym individually. For -** example, faced with the query: -** -** -** ... MATCH 'first place' -** -** the tokenizer offers both "1st" and "first" as synonyms for the -** first token in the MATCH query and FTS5 effectively runs a query -** similar to: -** -** -** ... MATCH '(first OR 1st) place' -** -** except that, for the purposes of auxiliary functions, the query -** still appears to contain just two phrases - "(first OR 1st)" -** being treated as a single phrase. -** -**
    3. By adding multiple synonyms for a single term to the FTS index. -** Using this method, when tokenizing document text, the tokenizer -** provides multiple synonyms for each token. So that when a -** document such as "I won first place" is tokenized, entries are -** added to the FTS index for "i", "won", "first", "1st" and -** "place". -** -** This way, even if the tokenizer does not provide synonyms -** when tokenizing query text (it should not - to do would be -** inefficient), it doesn't matter if the user queries for -** 'first + place' or '1st + place', as there are entires in the -** FTS index corresponding to both forms of the first token. -**
    -** -** Whether it is parsing document or query text, any call to xToken that -** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit -** is considered to supply a synonym for the previous token. For example, -** when parsing the document "I won first place", a tokenizer that supports -** synonyms would call xToken() 5 times, as follows: -** -** -** xToken(pCtx, 0, "i", 1, 0, 1); -** xToken(pCtx, 0, "won", 3, 2, 5); -** xToken(pCtx, 0, "first", 5, 6, 11); -** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); -** xToken(pCtx, 0, "place", 5, 12, 17); -** -** -** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time -** xToken() is called. Multiple synonyms may be specified for a single token -** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. -** There is no limit to the number of synonyms that may be provided for a -** single token. -** -** In many cases, method (1) above is the best approach. It does not add -** extra data to the FTS index or require FTS5 to query for multiple terms, -** so it is efficient in terms of disk space and query speed. However, it -** does not support prefix queries very well. If, as suggested above, the -** token "first" is subsituted for "1st" by the tokenizer, then the query: -** -** -** ... MATCH '1s*' -** -** will not match documents that contain the token "1st" (as the tokenizer -** will probably not map "1s" to any prefix of "first"). -** -** For full prefix support, method (3) may be preferred. In this case, -** because the index contains entries for both "first" and "1st", prefix -** queries such as 'fi*' or '1s*' will match correctly. However, because -** extra entries are added to the FTS index, this method uses more space -** within the database. -** -** Method (2) offers a midpoint between (1) and (3). Using this method, -** a query such as '1s*' will match documents that contain the literal -** token "1st", but not "first" (assuming the tokenizer is not able to -** provide synonyms for prefixes). However, a non-prefix query like '1st' -** will match against "1st" and "first". This method does not require -** extra disk space, as no extra entries are added to the FTS index. -** On the other hand, it may require more CPU cycles to run MATCH queries, -** as separate queries of the FTS index are required for each synonym. -** -** When using methods (2) or (3), it is important that the tokenizer only -** provide synonyms when tokenizing document text (method (2)) or query -** text (method (3)), not both. Doing so will not cause any errors, but is -** inefficient. -*/ -typedef struct Fts5Tokenizer Fts5Tokenizer; -typedef struct fts5_tokenizer fts5_tokenizer; -struct fts5_tokenizer { - int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); - void (*xDelete)(Fts5Tokenizer*); - int (*xTokenize)(Fts5Tokenizer*, - void *pCtx, - int flags, /* Mask of FTS5_TOKENIZE_* flags */ - const char *pText, int nText, - int (*xToken)( - void *pCtx, /* Copy of 2nd argument to xTokenize() */ - int tflags, /* Mask of FTS5_TOKEN_* flags */ - const char *pToken, /* Pointer to buffer containing token */ - int nToken, /* Size of token in bytes */ - int iStart, /* Byte offset of token within input text */ - int iEnd /* Byte offset of end of token within input text */ - ) - ); -}; - -/* Flags that may be passed as the third argument to xTokenize() */ -#define FTS5_TOKENIZE_QUERY 0x0001 -#define FTS5_TOKENIZE_PREFIX 0x0002 -#define FTS5_TOKENIZE_DOCUMENT 0x0004 -#define FTS5_TOKENIZE_AUX 0x0008 - -/* Flags that may be passed by the tokenizer implementation back to FTS5 -** as the third argument to the supplied xToken callback. */ -#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ - -/* -** END OF CUSTOM TOKENIZERS -*************************************************************************/ - -/************************************************************************* -** FTS5 EXTENSION REGISTRATION API -*/ -typedef struct fts5_api fts5_api; -struct fts5_api { - int iVersion; /* Currently always set to 2 */ - - /* Create a new tokenizer */ - int (*xCreateTokenizer)( - fts5_api *pApi, - const char *zName, - void *pContext, - fts5_tokenizer *pTokenizer, - void (*xDestroy)(void*) - ); - - /* Find an existing tokenizer */ - int (*xFindTokenizer)( - fts5_api *pApi, - const char *zName, - void **ppContext, - fts5_tokenizer *pTokenizer - ); - - /* Create a new auxiliary function */ - int (*xCreateFunction)( - fts5_api *pApi, - const char *zName, - void *pContext, - fts5_extension_function xFunction, - void (*xDestroy)(void*) - ); -}; - -/* -** END OF REGISTRATION API -*************************************************************************/ - -#if 0 -} /* end of the 'extern "C"' block */ -#endif - -#endif /* _FTS5_H */ - -/******** End of fts5.h *********/ /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -10746,13 +8153,13 @@ struct fts5_api { ** The suggested maximum number of in-memory pages to use for ** the main database table and for temporary tables. ** -** IMPLEMENTATION-OF: R-30185-15359 The default suggested cache size is -2000, -** which means the cache size is limited to 2048000 bytes of memory. +** IMPLEMENTATION-OF: R-31093-59126 The default suggested cache size +** is 2000 pages. ** IMPLEMENTATION-OF: R-48205-43578 The default suggested cache size can be ** altered using the SQLITE_DEFAULT_CACHE_SIZE compile-time options. */ #ifndef SQLITE_DEFAULT_CACHE_SIZE -# define SQLITE_DEFAULT_CACHE_SIZE -2000 +# define SQLITE_DEFAULT_CACHE_SIZE 2000 #endif /* @@ -10765,9 +8172,8 @@ struct fts5_api { /* ** The maximum number of attached databases. This must be between 0 -** and 125. The upper bound of 125 is because the attached databases are -** counted using a signed 8-bit integer which has a maximum value of 127 -** and we have to allow 2 extra counts for the "main" and "temp" databases. +** and 62. The upper bound on 62 is because a 64-bit integer bitmap +** is used internally to track attached databases. */ #ifndef SQLITE_MAX_ATTACHED # define SQLITE_MAX_ATTACHED 10 @@ -10802,7 +8208,7 @@ struct fts5_api { ** The default size of a database page. */ #ifndef SQLITE_DEFAULT_PAGE_SIZE -# define SQLITE_DEFAULT_PAGE_SIZE 4096 +# define SQLITE_DEFAULT_PAGE_SIZE 1024 #endif #if SQLITE_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE # undef SQLITE_DEFAULT_PAGE_SIZE @@ -10883,7 +8289,7 @@ struct fts5_api { ** to the next, so we have developed the following set of #if statements ** to generate appropriate macros for a wide range of compilers. ** -** The correct "ANSI" way to do this is to use the intptr_t type. +** The correct "ANSI" way to do this is to use the intptr_t type. ** Unfortunately, that typedef is not available on all compilers, or ** if it is available, it requires an #include of specific headers ** that vary from one machine to the next. @@ -10919,24 +8325,6 @@ struct fts5_api { # define SQLITE_NOINLINE #endif -/* -** Make sure that the compiler intrinsics we desire are enabled when -** compiling with an appropriate version of MSVC unless prevented by -** the SQLITE_DISABLE_INTRINSIC define. -*/ -#if !defined(SQLITE_DISABLE_INTRINSIC) -# if defined(_MSC_VER) && _MSC_VER>=1400 -# if !defined(_WIN32_WCE) -# include -# pragma intrinsic(_byteswap_ushort) -# pragma intrinsic(_byteswap_ulong) -# pragma intrinsic(_ReadWriteBarrier) -# else -# include -# endif -# endif -#endif - /* ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. ** 0 means mutexes are permanently disable and the library is never @@ -11035,7 +8423,7 @@ struct fts5_api { ** is set. Thus NDEBUG becomes an opt-in rather than an opt-out ** feature. */ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif #if defined(NDEBUG) && defined(SQLITE_DEBUG) @@ -11050,7 +8438,7 @@ struct fts5_api { #endif /* -** The testcase() macro is used to aid in coverage testing. When +** The testcase() macro is used to aid in coverage testing. When ** doing coverage testing, the condition inside the argument to ** testcase() must be evaluated both true and false in order to ** get full branch coverage. The testcase() macro is inserted @@ -11096,7 +8484,7 @@ SQLITE_PRIVATE void sqlite3Coverage(int); #endif /* -** The ALWAYS and NEVER macros surround boolean expressions which +** The ALWAYS and NEVER macros surround boolean expressions which ** are intended to always be true or false, respectively. Such ** expressions could be omitted from the code completely. But they ** are included in a few cases in order to enhance the resilience @@ -11110,7 +8498,7 @@ SQLITE_PRIVATE void sqlite3Coverage(int); ** be true and false so that the unreachable code they specify will ** not be counted as untested code. */ -#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +#if defined(SQLITE_COVERAGE_TEST) # define ALWAYS(X) (1) # define NEVER(X) (0) #elif !defined(NDEBUG) @@ -11121,21 +8509,6 @@ SQLITE_PRIVATE void sqlite3Coverage(int); # define NEVER(X) (X) #endif -/* -** Some malloc failures are only possible if SQLITE_TEST_REALLOC_STRESS is -** defined. We need to defend against those failures when testing with -** SQLITE_TEST_REALLOC_STRESS, but we don't want the unreachable branches -** during a normal build. The following macro can be used to disable tests -** that are always false except when SQLITE_TEST_REALLOC_STRESS is set. -*/ -#if defined(SQLITE_TEST_REALLOC_STRESS) -# define ONLY_IF_REALLOC_STRESS(X) (X) -#elif !defined(NDEBUG) -# define ONLY_IF_REALLOC_STRESS(X) ((X)?(assert(0),1):0) -#else -# define ONLY_IF_REALLOC_STRESS(X) (0) -#endif - /* ** Declarations used for tracing the operating system interfaces. */ @@ -11162,13 +8535,6 @@ SQLITE_PRIVATE void sqlite3Coverage(int); # undef SQLITE_NEED_ERR_NAME #endif -/* -** SQLITE_ENABLE_EXPLAIN_COMMENTS is incompatible with SQLITE_OMIT_EXPLAIN -*/ -#ifdef SQLITE_OMIT_EXPLAIN -# undef SQLITE_ENABLE_EXPLAIN_COMMENTS -#endif - /* ** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() @@ -11202,8 +8568,8 @@ SQLITE_PRIVATE void sqlite3Coverage(int); ** This is the header file for the generic hash-table implementation ** used in SQLite. */ -#ifndef SQLITE_HASH_H -#define SQLITE_HASH_H +#ifndef _SQLITE_HASH_H_ +#define _SQLITE_HASH_H_ /* Forward declarations of structures. */ typedef struct Hash Hash; @@ -11283,7 +8649,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); */ /* #define sqliteHashCount(H) ((H)->count) // NOT USED */ -#endif /* SQLITE_HASH_H */ +#endif /* _SQLITE_HASH_H_ */ /************** End of hash.h ************************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -11315,76 +8681,76 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_AS 24 #define TK_WITHOUT 25 #define TK_COMMA 26 -#define TK_OR 27 -#define TK_AND 28 -#define TK_IS 29 -#define TK_MATCH 30 -#define TK_LIKE_KW 31 -#define TK_BETWEEN 32 -#define TK_IN 33 -#define TK_ISNULL 34 -#define TK_NOTNULL 35 -#define TK_NE 36 -#define TK_EQ 37 -#define TK_GT 38 -#define TK_LE 39 -#define TK_LT 40 -#define TK_GE 41 -#define TK_ESCAPE 42 -#define TK_BITAND 43 -#define TK_BITOR 44 -#define TK_LSHIFT 45 -#define TK_RSHIFT 46 -#define TK_PLUS 47 -#define TK_MINUS 48 -#define TK_STAR 49 -#define TK_SLASH 50 -#define TK_REM 51 -#define TK_CONCAT 52 -#define TK_COLLATE 53 -#define TK_BITNOT 54 -#define TK_ID 55 -#define TK_INDEXED 56 -#define TK_ABORT 57 -#define TK_ACTION 58 -#define TK_AFTER 59 -#define TK_ANALYZE 60 -#define TK_ASC 61 -#define TK_ATTACH 62 -#define TK_BEFORE 63 -#define TK_BY 64 -#define TK_CASCADE 65 -#define TK_CAST 66 -#define TK_COLUMNKW 67 -#define TK_CONFLICT 68 -#define TK_DATABASE 69 -#define TK_DESC 70 -#define TK_DETACH 71 -#define TK_EACH 72 -#define TK_FAIL 73 -#define TK_FOR 74 -#define TK_IGNORE 75 -#define TK_INITIALLY 76 -#define TK_INSTEAD 77 -#define TK_NO 78 -#define TK_KEY 79 -#define TK_OF 80 -#define TK_OFFSET 81 -#define TK_PRAGMA 82 -#define TK_RAISE 83 -#define TK_RECURSIVE 84 -#define TK_REPLACE 85 -#define TK_RESTRICT 86 -#define TK_ROW 87 -#define TK_TRIGGER 88 -#define TK_VACUUM 89 -#define TK_VIEW 90 -#define TK_VIRTUAL 91 -#define TK_WITH 92 -#define TK_REINDEX 93 -#define TK_RENAME 94 -#define TK_CTIME_KW 95 -#define TK_ANY 96 +#define TK_ID 27 +#define TK_INDEXED 28 +#define TK_ABORT 29 +#define TK_ACTION 30 +#define TK_AFTER 31 +#define TK_ANALYZE 32 +#define TK_ASC 33 +#define TK_ATTACH 34 +#define TK_BEFORE 35 +#define TK_BY 36 +#define TK_CASCADE 37 +#define TK_CAST 38 +#define TK_COLUMNKW 39 +#define TK_CONFLICT 40 +#define TK_DATABASE 41 +#define TK_DESC 42 +#define TK_DETACH 43 +#define TK_EACH 44 +#define TK_FAIL 45 +#define TK_FOR 46 +#define TK_IGNORE 47 +#define TK_INITIALLY 48 +#define TK_INSTEAD 49 +#define TK_LIKE_KW 50 +#define TK_MATCH 51 +#define TK_NO 52 +#define TK_KEY 53 +#define TK_OF 54 +#define TK_OFFSET 55 +#define TK_PRAGMA 56 +#define TK_RAISE 57 +#define TK_RECURSIVE 58 +#define TK_REPLACE 59 +#define TK_RESTRICT 60 +#define TK_ROW 61 +#define TK_TRIGGER 62 +#define TK_VACUUM 63 +#define TK_VIEW 64 +#define TK_VIRTUAL 65 +#define TK_WITH 66 +#define TK_REINDEX 67 +#define TK_RENAME 68 +#define TK_CTIME_KW 69 +#define TK_ANY 70 +#define TK_OR 71 +#define TK_AND 72 +#define TK_IS 73 +#define TK_BETWEEN 74 +#define TK_IN 75 +#define TK_ISNULL 76 +#define TK_NOTNULL 77 +#define TK_NE 78 +#define TK_EQ 79 +#define TK_GT 80 +#define TK_LE 81 +#define TK_LT 82 +#define TK_GE 83 +#define TK_ESCAPE 84 +#define TK_BITAND 85 +#define TK_BITOR 86 +#define TK_LSHIFT 87 +#define TK_RSHIFT 88 +#define TK_PLUS 89 +#define TK_MINUS 90 +#define TK_STAR 91 +#define TK_SLASH 92 +#define TK_REM 93 +#define TK_CONCAT 94 +#define TK_COLLATE 95 +#define TK_BITNOT 96 #define TK_STRING 97 #define TK_JOIN_KW 98 #define TK_CONSTRAINT 99 @@ -11420,9 +8786,9 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_LIMIT 129 #define TK_WHERE 130 #define TK_INTO 131 -#define TK_FLOAT 132 -#define TK_BLOB 133 -#define TK_INTEGER 134 +#define TK_INTEGER 132 +#define TK_FLOAT 133 +#define TK_BLOB 134 #define TK_VARIABLE 135 #define TK_CASE 136 #define TK_WHEN 137 @@ -11438,27 +8804,16 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_TO_REAL 147 #define TK_ISNOT 148 #define TK_END_OF_FILE 149 -#define TK_UNCLOSED_STRING 150 -#define TK_FUNCTION 151 -#define TK_COLUMN 152 -#define TK_AGG_FUNCTION 153 -#define TK_AGG_COLUMN 154 -#define TK_UMINUS 155 -#define TK_UPLUS 156 -#define TK_REGISTER 157 -#define TK_VECTOR 158 -#define TK_SELECT_COLUMN 159 -#define TK_ASTERISK 160 -#define TK_SPAN 161 -#define TK_SPACE 162 -#define TK_ILLEGAL 163 - -/* The token codes above must all fit in 8 bits */ -#define TKFLG_MASK 0xff - -/* Flags that can be added to a token code when it is not -** being stored in a u8: */ -#define TKFLG_DONTFOLD 0x100 /* Omit constant folding optimizations */ +#define TK_ILLEGAL 150 +#define TK_SPACE 151 +#define TK_UNCLOSED_STRING 152 +#define TK_FUNCTION 153 +#define TK_COLUMN 154 +#define TK_AGG_FUNCTION 155 +#define TK_AGG_COLUMN 156 +#define TK_UMINUS 157 +#define TK_UPLUS 158 +#define TK_REGISTER 159 /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -11490,7 +8845,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); /* ** OMIT_TEMPDB is set to 1 if SQLITE_OMIT_TEMPDB is defined, or 0 -** afterward. Having this macro allows us to cause the C compiler +** afterward. Having this macro allows us to cause the C compiler ** to omit code used by TEMP tables without messy #ifndef statements. */ #ifdef SQLITE_OMIT_TEMPDB @@ -11529,7 +8884,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); /* ** If no value has been provided for SQLITE_MAX_WORKER_THREADS, or if -** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it +** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it ** to zero. */ #if SQLITE_TEMP_STORE==3 || SQLITE_THREADSAFE==0 @@ -11547,15 +8902,6 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); # define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS #endif -/* -** The default initial allocation for the pagecache when using separate -** pagecaches for each database connection. A positive number is the -** number of pages. A negative number N translations means that a buffer -** of -1024*N bytes is allocated and used for as many pages as it will hold. -*/ -#ifndef SQLITE_DEFAULT_PCACHE_INITSZ -# define SQLITE_DEFAULT_PCACHE_INITSZ 100 -#endif /* ** GCC does not define the offsetof() macro so we'll have to do it @@ -11568,12 +8914,8 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); /* ** Macros to compute minimum and maximum of two numbers. */ -#ifndef MIN -# define MIN(A,B) ((A)<(B)?(A):(B)) -#endif -#ifndef MAX -# define MAX(A,B) ((A)>(B)?(A):(B)) -#endif +#define MIN(A,B) ((A)<(B)?(A):(B)) +#define MAX(A,B) ((A)>(B)?(A):(B)) /* ** Swap two objects of type TYPE. @@ -11681,7 +9023,7 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** 4 -> 20 1000 -> 99 1048576 -> 200 ** 10 -> 33 1024 -> 100 4294967296 -> 320 ** -** The LogEst can be negative to indicate fractional values. +** The LogEst can be negative to indicate fractional values. ** Examples: ** ** 0.5 -> -10 0.1 -> -33 0.0625 -> -40 @@ -11702,27 +9044,6 @@ typedef INT16_TYPE LogEst; # endif #endif -/* The uptr type is an unsigned integer large enough to hold a pointer -*/ -#if defined(HAVE_STDINT_H) - typedef uintptr_t uptr; -#elif SQLITE_PTRSIZE==4 - typedef u32 uptr; -#else - typedef u64 uptr; -#endif - -/* -** The SQLITE_WITHIN(P,S,E) macro checks to see if pointer P points to -** something between S (inclusive) and E (exclusive). -** -** In other words, S is a buffer and E is a pointer to the first byte after -** the end of buffer S. This macro returns true if P points to something -** contained within the buffer S. -*/ -#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) - - /* ** Macros to determine whether the machine is big or little endian, ** and whether or not that determination is run-time or compile-time. @@ -11732,6 +9053,11 @@ typedef INT16_TYPE LogEst; ** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined ** at run-time. */ +#ifdef SQLITE_AMALGAMATION +SQLITE_PRIVATE const int sqlite3one = 1; +#else +SQLITE_PRIVATE const int sqlite3one; +#endif #if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ @@ -11749,11 +9075,6 @@ typedef INT16_TYPE LogEst; # define SQLITE_UTF16NATIVE SQLITE_UTF16BE #endif #if !defined(SQLITE_BYTEORDER) -# ifdef SQLITE_AMALGAMATION - const int sqlite3one = 1; -# else - extern const int sqlite3one; -# endif # define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ # define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) # define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) @@ -11768,7 +9089,7 @@ typedef INT16_TYPE LogEst; #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -/* +/* ** Round up a number to the next larger multiple of 8. This is used ** to force 8-byte alignment on 64-bit architectures. */ @@ -11807,14 +9128,16 @@ typedef INT16_TYPE LogEst; */ #ifdef __APPLE__ # include +# if TARGET_OS_IPHONE +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +# endif #endif #ifndef SQLITE_MAX_MMAP_SIZE # if defined(__linux__) \ || defined(_WIN32) \ || (defined(__APPLE__) && defined(__MACH__)) \ - || defined(__sun) \ - || defined(__FreeBSD__) \ - || defined(__DragonFly__) + || defined(__sun) # define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ # else # define SQLITE_MAX_MMAP_SIZE 0 @@ -11862,7 +9185,7 @@ typedef INT16_TYPE LogEst; /* ** An instance of the following structure is used to store the busy-handler -** callback for a given sqlite handle. +** callback for a given sqlite handle. ** ** The sqlite.busyHandler member of the sqlite struct contains the busy ** callback for the database handle. Each pager opened via the sqlite @@ -11907,9 +9230,9 @@ struct BusyHandler { /* ** The following value as a destructor means to use sqlite3DbFree(). -** The sqlite3DbFree() routine requires two parameters instead of the -** one parameter that destructors normally want. So we have to introduce -** this magic value that the code knows to handle differently. Any +** The sqlite3DbFree() routine requires two parameters instead of the +** one parameter that destructors normally want. So we have to introduce +** this magic value that the code knows to handle differently. Any ** pointer will work here as long as it is distinct from SQLITE_STATIC ** and SQLITE_TRANSIENT. */ @@ -11933,19 +9256,19 @@ struct BusyHandler { #define SQLITE_WSD const #define GLOBAL(t,v) (*(t*)sqlite3_wsd_find((void*)&(v), sizeof(v))) #define sqlite3GlobalConfig GLOBAL(struct Sqlite3Config, sqlite3Config) -SQLITE_API int sqlite3_wsd_init(int N, int J); -SQLITE_API void *sqlite3_wsd_find(void *K, int L); +SQLITE_API int SQLITE_STDCALL sqlite3_wsd_init(int N, int J); +SQLITE_API void *SQLITE_STDCALL sqlite3_wsd_find(void *K, int L); #else - #define SQLITE_WSD + #define SQLITE_WSD #define GLOBAL(t,v) v #define sqlite3GlobalConfig sqlite3Config #endif /* ** The following macros are used to suppress compiler warnings and to -** make it clear to human readers when a function parameter is deliberately +** make it clear to human readers when a function parameter is deliberately ** left unused within the body of a function. This usually happens when -** a function is called via a function pointer. For example the +** a function is called via a function pointer. For example the ** implementation of an SQL aggregate step callback may not use the ** parameter indicating the number of arguments passed to the aggregate, ** if it knows that this is enforced elsewhere. @@ -11988,7 +9311,6 @@ typedef struct LookasideSlot LookasideSlot; typedef struct Module Module; typedef struct NameContext NameContext; typedef struct Parse Parse; -typedef struct PreUpdate PreUpdate; typedef struct PrintfArguments PrintfArguments; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; @@ -12012,7 +9334,7 @@ typedef struct WhereInfo WhereInfo; typedef struct With With; /* -** Defer sourcing vdbe.h and btree.h until after the "u8" and +** Defer sourcing vdbe.h and btree.h until after the "u8" and ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque ** pointer types (i.e. FuncDef) defined above. */ @@ -12033,8 +9355,8 @@ typedef struct With With; ** subsystem. See comments in the source code for a detailed description ** of what each interface routine does. */ -#ifndef SQLITE_BTREE_H -#define SQLITE_BTREE_H +#ifndef _BTREE_H_ +#define _BTREE_H_ /* TODO: This definition is just included so other modules compile. It ** needs to be revisited. @@ -12059,7 +9381,6 @@ typedef struct With With; typedef struct Btree Btree; typedef struct BtCursor BtCursor; typedef struct BtShared BtShared; -typedef struct BtreePayload BtreePayload; SQLITE_PRIVATE int sqlite3BtreeOpen( @@ -12084,11 +9405,11 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeSetSpillSize(Btree*,int); #if SQLITE_MAX_MMAP_SIZE>0 SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); #endif SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags(Btree*,unsigned); +SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree*,int); @@ -12110,9 +9431,7 @@ SQLITE_PRIVATE int sqlite3BtreeIsInReadTrans(Btree*); SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree*); SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *, int, void(*)(void *)); SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *pBtree); -#ifndef SQLITE_OMIT_SHARED_CACHE SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *pBtree, int iTab, u8 isWriteLock); -#endif SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *, int, int); SQLITE_PRIVATE const char *sqlite3BtreeGetFilename(Btree *); @@ -12173,37 +9492,8 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); #define BTREE_DATA_VERSION 15 /* A virtual meta-value */ /* -** Kinds of hints that can be passed into the sqlite3BtreeCursorHint() -** interface. -** -** BTREE_HINT_RANGE (arguments: Expr*, Mem*) -** -** The first argument is an Expr* (which is guaranteed to be constant for -** the lifetime of the cursor) that defines constraints on which rows -** might be fetched with this cursor. The Expr* tree may contain -** TK_REGISTER nodes that refer to values stored in the array of registers -** passed as the second parameter. In other words, if Expr.op==TK_REGISTER -** then the value of the node is the value in Mem[pExpr.iTable]. Any -** TK_COLUMN node in the expression tree refers to the Expr.iColumn-th -** column of the b-tree of the cursor. The Expr tree will not contain -** any function calls nor subqueries nor references to b-trees other than -** the cursor being hinted. -** -** The design of the _RANGE hint is aid b-tree implementations that try -** to prefetch content from remote machines - to provide those -** implementations with limits on what needs to be prefetched and thereby -** reduce network bandwidth. -** -** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by -** standard SQLite. The other hints are provided for extentions that use -** the SQLite parser and code generator but substitute their own storage -** engine. -*/ -#define BTREE_HINT_RANGE 0 /* Range constraints on queries */ - -/* -** Values that may be OR'd together to form the argument to the -** BTREE_HINT_FLAGS hint for sqlite3BtreeCursorHint(): +** Values that may be OR'd together to form the second argument of an +** sqlite3BtreeCursorHints() call. ** ** The BTREE_BULKLOAD flag is set on index cursors when the index is going ** to be filled with content that is already in sorted order. @@ -12217,32 +9507,6 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); #define BTREE_BULKLOAD 0x00000001 /* Used to full index in sorted order */ #define BTREE_SEEK_EQ 0x00000002 /* EQ seeks only - no range seeks */ -/* -** Flags passed as the third argument to sqlite3BtreeCursor(). -** -** For read-only cursors the wrFlag argument is always zero. For read-write -** cursors it may be set to either (BTREE_WRCSR|BTREE_FORDELETE) or just -** (BTREE_WRCSR). If the BTREE_FORDELETE bit is set, then the cursor will -** only be used by SQLite for the following: -** -** * to seek to and then delete specific entries, and/or -** -** * to read values that will be used to create keys that other -** BTREE_FORDELETE cursors will seek to and delete. -** -** The BTREE_FORDELETE flag is an optimization hint. It is not used by -** by this, the native b-tree engine of SQLite, but it is available to -** alternative storage engines that might be substituted in place of this -** b-tree system. For alternative storage engines in which a delete of -** the main table row automatically deletes corresponding index rows, -** the FORDELETE flag hint allows those alternative storage engines to -** skip a lot of work. Namely: FORDELETE cursors may treat all SEEK -** and DELETE operations as no-ops, and any READ operation against a -** FORDELETE cursor may return a null row: 0x01 0x00. -*/ -#define BTREE_WRCSR 0x00000004 /* read-write cursor */ -#define BTREE_FORDELETE 0x00000008 /* Cursor is for seek/delete only */ - SQLITE_PRIVATE int sqlite3BtreeCursor( Btree*, /* BTree containing table to open */ int iTable, /* Index of root page */ @@ -12252,10 +9516,6 @@ SQLITE_PRIVATE int sqlite3BtreeCursor( ); SQLITE_PRIVATE int sqlite3BtreeCursorSize(void); SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*); -SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned); -#ifdef SQLITE_ENABLE_CURSOR_HINTS -SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor*, int, ...); -#endif SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*); SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( @@ -12267,61 +9527,33 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( ); SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*); SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor*, int*); -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*, u8 flags); - -/* Allowed flags for the 2nd argument to sqlite3BtreeDelete() */ -#define BTREE_SAVEPOSITION 0x02 /* Leave cursor pointing at NEXT or PREV */ -#define BTREE_AUXDELETE 0x04 /* not the primary delete operation */ - -/* An instance of the BtreePayload object describes the content of a single -** entry in either an index or table btree. -** -** Index btrees (used for indexes and also WITHOUT ROWID tables) contain -** an arbitrary key and no data. These btrees have pKey,nKey set to their -** key and pData,nData,nZero set to zero. -** -** Table btrees (used for rowid tables) contain an integer rowid used as -** the key and passed in the nKey field. The pKey field is zero. -** pData,nData hold the content of the new entry. nZero extra zero bytes -** are appended to the end of the content when constructing the entry. -** -** This object is used to pass information into sqlite3BtreeInsert(). The -** same information used to be passed as five separate parameters. But placing -** the information into this object helps to keep the interface more -** organized and understandable, and it also helps the resulting code to -** run a little faster by using fewer registers for parameter passing. -*/ -struct BtreePayload { - const void *pKey; /* Key content for indexes. NULL for tables */ - sqlite3_int64 nKey; /* Size of pKey for indexes. PRIMARY KEY for tabs */ - const void *pData; /* Data for tables. NULL for indexes */ - int nData; /* Size of pData. 0 if none. */ - int nZero; /* Extra zero data appended after pData,nData */ -}; - -SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, - int bias, int seekResult); +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, + const void *pData, int nData, + int nZero, int bias, int seekResult); SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int *pRes); -SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor*, i64 *pSize); SQLITE_PRIVATE int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*); -SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); -SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor*); +SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor*, u32 *pAmt); +SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor*, u32 *pAmt); +SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor*, u32 *pSize); SQLITE_PRIVATE int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*); SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); -#ifndef SQLITE_OMIT_INCRBLOB SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *); -#endif SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *); SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); +SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); +#ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); +#endif SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *pBt); SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void); @@ -12350,19 +9582,15 @@ SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); #ifndef SQLITE_OMIT_SHARED_CACHE SQLITE_PRIVATE void sqlite3BtreeEnter(Btree*); SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3*); -SQLITE_PRIVATE int sqlite3BtreeSharable(Btree*); -SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor*); -SQLITE_PRIVATE int sqlite3BtreeConnectionCount(Btree*); #else # define sqlite3BtreeEnter(X) # define sqlite3BtreeEnterAll(X) -# define sqlite3BtreeSharable(X) 0 -# define sqlite3BtreeEnterCursor(X) -# define sqlite3BtreeConnectionCount(X) 1 #endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree*); SQLITE_PRIVATE void sqlite3BtreeLeave(Btree*); +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3*); #ifndef NDEBUG @@ -12373,7 +9601,9 @@ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); #endif #else +# define sqlite3BtreeSharable(X) 0 # define sqlite3BtreeLeave(X) +# define sqlite3BtreeEnterCursor(X) # define sqlite3BtreeLeaveCursor(X) # define sqlite3BtreeLeaveAll(X) @@ -12383,7 +9613,7 @@ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); #endif -#endif /* SQLITE_BTREE_H */ +#endif /* _BTREE_H_ */ /************** End of btree.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -12406,8 +9636,8 @@ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); ** or VDBE. The VDBE implements an abstract machine that runs a ** simple program to access and modify the underlying database. */ -#ifndef SQLITE_VDBE_H -#define SQLITE_VDBE_H +#ifndef _SQLITE_VDBE_H_ +#define _SQLITE_VDBE_H_ /* #include */ /* @@ -12432,29 +9662,24 @@ typedef struct SubProgram SubProgram; struct VdbeOp { u8 opcode; /* What operation to perform */ signed char p4type; /* One of the P4_xxx constants for p4 */ - u8 notUsed1; + u8 opflags; /* Mask of the OPFLG_* flags in opcodes.h */ u8 p5; /* Fifth parameter is an unsigned character */ int p1; /* First operand */ int p2; /* Second parameter (often the jump destination) */ int p3; /* The third parameter */ - union p4union { /* fourth parameter */ + union { /* fourth parameter */ int i; /* Integer value if p4type==P4_INT32 */ void *p; /* Generic pointer */ char *z; /* Pointer to data for string (char array) types */ i64 *pI64; /* Used when p4type is P4_INT64 */ double *pReal; /* Used when p4type is P4_REAL */ FuncDef *pFunc; /* Used when p4type is P4_FUNCDEF */ - sqlite3_context *pCtx; /* Used when p4type is P4_FUNCCTX */ CollSeq *pColl; /* Used when p4type is P4_COLLSEQ */ Mem *pMem; /* Used when p4type is P4_MEM */ VTable *pVtab; /* Used when p4type is P4_VTAB */ KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ - Table *pTab; /* Used when p4type is P4_TABLE */ -#ifdef SQLITE_ENABLE_CURSOR_HINTS - Expr *pExpr; /* Used when p4type is P4_EXPR */ -#endif int (*xAdvance)(BtCursor *, int *); } p4; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS @@ -12479,6 +9704,7 @@ struct SubProgram { int nOp; /* Elements in aOp[] */ int nMem; /* Number of memory cells required */ int nCsr; /* Number of cursors required */ + int nOnce; /* Number of OP_Once instructions */ void *token; /* id that may be used to recursive triggers */ SubProgram *pNext; /* Next sub-program already visited */ }; @@ -12504,7 +9730,6 @@ typedef struct VdbeOpList VdbeOpList; #define P4_COLLSEQ (-4) /* P4 is a pointer to a CollSeq structure */ #define P4_FUNCDEF (-5) /* P4 is a pointer to a FuncDef structure */ #define P4_KEYINFO (-6) /* P4 is a pointer to a KeyInfo structure */ -#define P4_EXPR (-7) /* P4 is a pointer to an Expr tree */ #define P4_MEM (-8) /* P4 is a pointer to a Mem* structure */ #define P4_TRANSIENT 0 /* P4 is a pointer to a transient string */ #define P4_VTAB (-10) /* P4 is a pointer to an sqlite3_vtab structure */ @@ -12515,8 +9740,6 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ #define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ -#define P4_TABLE (-20) /* P4 is a pointer to a Table structure */ -#define P4_FUNCCTX (-21) /* P4 is a pointer to an sqlite3_context object */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -12558,211 +9781,197 @@ typedef struct VdbeOpList VdbeOpList; /************** Include opcodes.h in the middle of vdbe.h ********************/ /************** Begin file opcodes.h *****************************************/ /* Automatically generated. Do not edit */ -/* See the tool/mkopcodeh.tcl script for details */ -#define OP_Savepoint 0 -#define OP_AutoCommit 1 -#define OP_Transaction 2 -#define OP_SorterNext 3 -#define OP_PrevIfOpen 4 -#define OP_NextIfOpen 5 -#define OP_Prev 6 -#define OP_Next 7 -#define OP_Checkpoint 8 -#define OP_JournalMode 9 -#define OP_Vacuum 10 -#define OP_VFilter 11 /* synopsis: iplan=r[P3] zplan='P4' */ -#define OP_VUpdate 12 /* synopsis: data=r[P3@P2] */ -#define OP_Goto 13 -#define OP_Gosub 14 -#define OP_InitCoroutine 15 -#define OP_Yield 16 -#define OP_MustBeInt 17 -#define OP_Jump 18 +/* See the mkopcodeh.awk script for details */ +#define OP_Function 1 /* synopsis: r[P3]=func(r[P2@P5]) */ +#define OP_Savepoint 2 +#define OP_AutoCommit 3 +#define OP_Transaction 4 +#define OP_SorterNext 5 +#define OP_PrevIfOpen 6 +#define OP_NextIfOpen 7 +#define OP_Prev 8 +#define OP_Next 9 +#define OP_AggStep 10 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_Checkpoint 11 +#define OP_JournalMode 12 +#define OP_Vacuum 13 +#define OP_VFilter 14 /* synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 15 /* synopsis: data=r[P3@P2] */ +#define OP_Goto 16 +#define OP_Gosub 17 +#define OP_Return 18 #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_Once 20 -#define OP_If 21 -#define OP_IfNot 22 -#define OP_SeekLT 23 /* synopsis: key=r[P3@P4] */ -#define OP_SeekLE 24 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGE 25 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGT 26 /* synopsis: key=r[P3@P4] */ -#define OP_Or 27 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ -#define OP_And 28 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_NoConflict 29 /* synopsis: key=r[P3@P4] */ -#define OP_NotFound 30 /* synopsis: key=r[P3@P4] */ -#define OP_Found 31 /* synopsis: key=r[P3@P4] */ -#define OP_SeekRowid 32 /* synopsis: intkey=r[P3] */ -#define OP_NotExists 33 /* synopsis: intkey=r[P3] */ -#define OP_IsNull 34 /* same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ -#define OP_NotNull 35 /* same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ -#define OP_Ne 36 /* same as TK_NE, synopsis: IF r[P3]!=r[P1] */ -#define OP_Eq 37 /* same as TK_EQ, synopsis: IF r[P3]==r[P1] */ -#define OP_Gt 38 /* same as TK_GT, synopsis: IF r[P3]>r[P1] */ -#define OP_Le 39 /* same as TK_LE, synopsis: IF r[P3]<=r[P1] */ -#define OP_Lt 40 /* same as TK_LT, synopsis: IF r[P3]=r[P1] */ -#define OP_ElseNotEq 42 /* same as TK_ESCAPE */ -#define OP_BitAnd 43 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ -#define OP_BitOr 44 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ -#define OP_ShiftLeft 45 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ -#define OP_Add 47 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ -#define OP_Subtract 48 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ -#define OP_Multiply 49 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ -#define OP_Divide 50 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ -#define OP_Remainder 51 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ -#define OP_Concat 52 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ -#define OP_Last 53 -#define OP_BitNot 54 /* same as TK_BITNOT, synopsis: r[P1]= ~r[P1] */ -#define OP_SorterSort 55 -#define OP_Sort 56 -#define OP_Rewind 57 -#define OP_IdxLE 58 /* synopsis: key=r[P3@P4] */ -#define OP_IdxGT 59 /* synopsis: key=r[P3@P4] */ -#define OP_IdxLT 60 /* synopsis: key=r[P3@P4] */ -#define OP_IdxGE 61 /* synopsis: key=r[P3@P4] */ -#define OP_RowSetRead 62 /* synopsis: r[P3]=rowset(P1) */ -#define OP_RowSetTest 63 /* synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 64 -#define OP_FkIfZero 65 /* synopsis: if fkctr[P1]==0 goto P2 */ -#define OP_IfPos 66 /* synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ -#define OP_IfNotZero 67 /* synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 */ -#define OP_DecrJumpZero 68 /* synopsis: if (--r[P1])==0 goto P2 */ -#define OP_IncrVacuum 69 -#define OP_VNext 70 -#define OP_Init 71 /* synopsis: Start at P2 */ -#define OP_Return 72 -#define OP_EndCoroutine 73 -#define OP_HaltIfNull 74 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 75 -#define OP_Integer 76 /* synopsis: r[P2]=P1 */ -#define OP_Int64 77 /* synopsis: r[P2]=P4 */ -#define OP_String 78 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_Null 79 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 80 /* synopsis: r[P1]=NULL */ -#define OP_Blob 81 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 82 /* synopsis: r[P2]=parameter(P1,P4) */ -#define OP_Move 83 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 84 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 85 /* synopsis: r[P2]=r[P1] */ -#define OP_IntCopy 86 /* synopsis: r[P2]=r[P1] */ -#define OP_ResultRow 87 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 88 -#define OP_Function0 89 /* synopsis: r[P3]=func(r[P2@P5]) */ -#define OP_Function 90 /* synopsis: r[P3]=func(r[P2@P5]) */ -#define OP_AddImm 91 /* synopsis: r[P1]=r[P1]+P2 */ -#define OP_RealAffinity 92 -#define OP_Cast 93 /* synopsis: affinity(r[P1]) */ -#define OP_Permutation 94 -#define OP_Compare 95 /* synopsis: r[P1@P3] <-> r[P2@P3] */ -#define OP_Column 96 /* synopsis: r[P3]=PX */ +#define OP_InitCoroutine 20 +#define OP_EndCoroutine 21 +#define OP_Yield 22 +#define OP_HaltIfNull 23 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 24 +#define OP_Integer 25 /* synopsis: r[P2]=P1 */ +#define OP_Int64 26 /* synopsis: r[P2]=P4 */ +#define OP_String 27 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 28 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 29 /* synopsis: r[P1]=NULL */ +#define OP_Blob 30 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 31 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 32 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 33 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 34 /* synopsis: r[P2]=r[P1] */ +#define OP_ResultRow 35 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 36 +#define OP_AddImm 37 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_MustBeInt 38 +#define OP_RealAffinity 39 +#define OP_Cast 40 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 41 +#define OP_Compare 42 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_Jump 43 +#define OP_Once 44 +#define OP_If 45 +#define OP_IfNot 46 +#define OP_Column 47 /* synopsis: r[P3]=PX */ +#define OP_Affinity 48 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 49 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 50 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 51 +#define OP_SetCookie 52 +#define OP_ReopenIdx 53 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 54 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 55 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenAutoindex 56 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 57 /* synopsis: nColumn=P2 */ +#define OP_SorterOpen 58 +#define OP_SequenceTest 59 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 60 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 61 +#define OP_SeekLT 62 /* synopsis: key=r[P3@P4] */ +#define OP_SeekLE 63 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGE 64 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGT 65 /* synopsis: key=r[P3@P4] */ +#define OP_Seek 66 /* synopsis: intkey=r[P2] */ +#define OP_NoConflict 67 /* synopsis: key=r[P3@P4] */ +#define OP_NotFound 68 /* synopsis: key=r[P3@P4] */ +#define OP_Found 69 /* synopsis: key=r[P3@P4] */ +#define OP_NotExists 70 /* synopsis: intkey=r[P3] */ +#define OP_Or 71 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ +#define OP_And 72 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ +#define OP_Sequence 73 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 74 /* synopsis: r[P2]=rowid */ +#define OP_Insert 75 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_IsNull 76 /* same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ +#define OP_NotNull 77 /* same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ +#define OP_Ne 78 /* same as TK_NE, synopsis: if r[P1]!=r[P3] goto P2 */ +#define OP_Eq 79 /* same as TK_EQ, synopsis: if r[P1]==r[P3] goto P2 */ +#define OP_Gt 80 /* same as TK_GT, synopsis: if r[P1]>r[P3] goto P2 */ +#define OP_Le 81 /* same as TK_LE, synopsis: if r[P1]<=r[P3] goto P2 */ +#define OP_Lt 82 /* same as TK_LT, synopsis: if r[P1]=r[P3] goto P2 */ +#define OP_InsertInt 84 /* synopsis: intkey=P3 data=r[P2] */ +#define OP_BitAnd 85 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 86 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 87 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ +#define OP_Add 89 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 90 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 91 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 92 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 93 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 94 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +#define OP_Delete 95 +#define OP_BitNot 96 /* same as TK_BITNOT, synopsis: r[P1]= ~r[P1] */ #define OP_String8 97 /* same as TK_STRING, synopsis: r[P2]='P4' */ -#define OP_Affinity 98 /* synopsis: affinity(r[P1@P2]) */ -#define OP_MakeRecord 99 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_Count 100 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 101 -#define OP_SetCookie 102 -#define OP_ReopenIdx 103 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenRead 104 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenWrite 105 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenAutoindex 106 /* synopsis: nColumn=P2 */ -#define OP_OpenEphemeral 107 /* synopsis: nColumn=P2 */ -#define OP_SorterOpen 108 -#define OP_SequenceTest 109 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ -#define OP_OpenPseudo 110 /* synopsis: P3 columns in r[P2] */ -#define OP_Close 111 -#define OP_ColumnsUsed 112 -#define OP_Sequence 113 /* synopsis: r[P2]=cursor[P1].ctr++ */ -#define OP_NewRowid 114 /* synopsis: r[P2]=rowid */ -#define OP_Insert 115 /* synopsis: intkey=r[P3] data=r[P2] */ -#define OP_InsertInt 116 /* synopsis: intkey=P3 data=r[P2] */ -#define OP_Delete 117 -#define OP_ResetCount 118 -#define OP_SorterCompare 119 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ -#define OP_SorterData 120 /* synopsis: r[P2]=data */ -#define OP_RowKey 121 /* synopsis: r[P2]=key */ -#define OP_RowData 122 /* synopsis: r[P2]=data */ -#define OP_Rowid 123 /* synopsis: r[P2]=rowid */ -#define OP_NullRow 124 -#define OP_SorterInsert 125 -#define OP_IdxInsert 126 /* synopsis: key=r[P2] */ -#define OP_IdxDelete 127 /* synopsis: key=r[P2@P3] */ -#define OP_Seek 128 /* synopsis: Move P3 to P1.rowid */ -#define OP_IdxRowid 129 /* synopsis: r[P2]=rowid */ -#define OP_Destroy 130 -#define OP_Clear 131 -#define OP_Real 132 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ -#define OP_ResetSorter 133 -#define OP_CreateIndex 134 /* synopsis: r[P2]=root iDb=P1 */ -#define OP_CreateTable 135 /* synopsis: r[P2]=root iDb=P1 */ -#define OP_ParseSchema 136 -#define OP_LoadAnalysis 137 -#define OP_DropTable 138 -#define OP_DropIndex 139 -#define OP_DropTrigger 140 -#define OP_IntegrityCk 141 -#define OP_RowSetAdd 142 /* synopsis: rowset(P1)=r[P2] */ -#define OP_Param 143 -#define OP_FkCounter 144 /* synopsis: fkctr[P1]+=P2 */ -#define OP_MemMax 145 /* synopsis: r[P1]=max(r[P1],r[P2]) */ -#define OP_OffsetLimit 146 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ -#define OP_AggStep0 147 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggStep 148 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggFinal 149 /* synopsis: accum=r[P1] N=P2 */ -#define OP_Expire 150 -#define OP_TableLock 151 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 152 -#define OP_VCreate 153 -#define OP_VDestroy 154 -#define OP_VOpen 155 -#define OP_VColumn 156 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 157 -#define OP_Pagecount 158 -#define OP_MaxPgcnt 159 -#define OP_CursorHint 160 -#define OP_Noop 161 -#define OP_Explain 162 +#define OP_ResetCount 98 +#define OP_SorterCompare 99 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 100 /* synopsis: r[P2]=data */ +#define OP_RowKey 101 /* synopsis: r[P2]=key */ +#define OP_RowData 102 /* synopsis: r[P2]=data */ +#define OP_Rowid 103 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 104 +#define OP_Last 105 +#define OP_SorterSort 106 +#define OP_Sort 107 +#define OP_Rewind 108 +#define OP_SorterInsert 109 +#define OP_IdxInsert 110 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 111 /* synopsis: key=r[P2@P3] */ +#define OP_IdxRowid 112 /* synopsis: r[P2]=rowid */ +#define OP_IdxLE 113 /* synopsis: key=r[P3@P4] */ +#define OP_IdxGT 114 /* synopsis: key=r[P3@P4] */ +#define OP_IdxLT 115 /* synopsis: key=r[P3@P4] */ +#define OP_IdxGE 116 /* synopsis: key=r[P3@P4] */ +#define OP_Destroy 117 +#define OP_Clear 118 +#define OP_ResetSorter 119 +#define OP_CreateIndex 120 /* synopsis: r[P2]=root iDb=P1 */ +#define OP_CreateTable 121 /* synopsis: r[P2]=root iDb=P1 */ +#define OP_ParseSchema 122 +#define OP_LoadAnalysis 123 +#define OP_DropTable 124 +#define OP_DropIndex 125 +#define OP_DropTrigger 126 +#define OP_IntegrityCk 127 +#define OP_RowSetAdd 128 /* synopsis: rowset(P1)=r[P2] */ +#define OP_RowSetRead 129 /* synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 130 /* synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 131 +#define OP_Param 132 +#define OP_Real 133 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_FkCounter 134 /* synopsis: fkctr[P1]+=P2 */ +#define OP_FkIfZero 135 /* synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_MemMax 136 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_IfPos 137 /* synopsis: if r[P1]>0 goto P2 */ +#define OP_IfNeg 138 /* synopsis: r[P1]+=P3, if r[P1]<0 goto P2 */ +#define OP_IfNotZero 139 /* synopsis: if r[P1]!=0 then r[P1]+=P3, goto P2 */ +#define OP_DecrJumpZero 140 /* synopsis: if (--r[P1])==0 goto P2 */ +#define OP_JumpZeroIncr 141 /* synopsis: if (r[P1]++)==0 ) goto P2 */ +#define OP_AggFinal 142 /* synopsis: accum=r[P1] N=P2 */ +#define OP_IncrVacuum 143 +#define OP_Expire 144 +#define OP_TableLock 145 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 146 +#define OP_VCreate 147 +#define OP_VDestroy 148 +#define OP_VOpen 149 +#define OP_VColumn 150 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VNext 151 +#define OP_VRename 152 +#define OP_Pagecount 153 +#define OP_MaxPgcnt 154 +#define OP_Init 155 /* synopsis: Start at P2 */ +#define OP_Noop 156 +#define OP_Explain 157 + /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c ** are encoded into bitvectors as follows: */ -#define OPFLG_JUMP 0x01 /* jump: P2 holds jmp target */ -#define OPFLG_IN1 0x02 /* in1: P1 is an input */ -#define OPFLG_IN2 0x04 /* in2: P2 is an input */ -#define OPFLG_IN3 0x08 /* in3: P3 is an input */ -#define OPFLG_OUT2 0x10 /* out2: P2 is an output */ -#define OPFLG_OUT3 0x20 /* out3: P3 is an output */ +#define OPFLG_JUMP 0x0001 /* jump: P2 holds jmp target */ +#define OPFLG_IN1 0x0002 /* in1: P1 is an input */ +#define OPFLG_IN2 0x0004 /* in2: P2 is an input */ +#define OPFLG_IN3 0x0008 /* in3: P3 is an input */ +#define OPFLG_OUT2 0x0010 /* out2: P2 is an output */ +#define OPFLG_OUT3 0x0020 /* out3: P3 is an output */ #define OPFLG_INITIALIZER {\ -/* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01,\ -/* 8 */ 0x00, 0x10, 0x00, 0x01, 0x00, 0x01, 0x01, 0x01,\ -/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x03, 0x03, 0x09,\ -/* 24 */ 0x09, 0x09, 0x09, 0x26, 0x26, 0x09, 0x09, 0x09,\ -/* 32 */ 0x09, 0x09, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ -/* 40 */ 0x0b, 0x0b, 0x01, 0x26, 0x26, 0x26, 0x26, 0x26,\ -/* 48 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x01, 0x12, 0x01,\ -/* 56 */ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x23, 0x0b,\ -/* 64 */ 0x01, 0x01, 0x03, 0x03, 0x03, 0x01, 0x01, 0x01,\ -/* 72 */ 0x02, 0x02, 0x08, 0x00, 0x10, 0x10, 0x10, 0x10,\ -/* 80 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x10, 0x10, 0x00,\ -/* 88 */ 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x00, 0x00,\ -/* 96 */ 0x00, 0x10, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ -/* 104 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 112 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 120 */ 0x00, 0x00, 0x00, 0x10, 0x00, 0x04, 0x04, 0x00,\ -/* 128 */ 0x00, 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10,\ -/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x10,\ -/* 144 */ 0x00, 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 152 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x10,\ -/* 160 */ 0x00, 0x00, 0x00,} - -/* The sqlite3P2Values() routine is able to run faster if it knows -** the value of the largest JUMP opcode. The smaller the maximum -** JUMP opcode the better, so the mkopcodeh.tcl script that -** generated this include file strives to group all JUMP opcodes -** together near the beginning of the list. -*/ -#define SQLITE_MX_JUMP_OPCODE 71 /* Maximum JUMP opcode */ +/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,\ +/* 8 */ 0x01, 0x01, 0x00, 0x00, 0x10, 0x00, 0x01, 0x00,\ +/* 16 */ 0x01, 0x01, 0x02, 0x12, 0x01, 0x02, 0x03, 0x08,\ +/* 24 */ 0x00, 0x10, 0x10, 0x10, 0x10, 0x00, 0x10, 0x10,\ +/* 32 */ 0x00, 0x00, 0x10, 0x00, 0x00, 0x02, 0x03, 0x02,\ +/* 40 */ 0x02, 0x00, 0x00, 0x01, 0x01, 0x03, 0x03, 0x00,\ +/* 48 */ 0x00, 0x00, 0x10, 0x10, 0x08, 0x00, 0x00, 0x00,\ +/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x09,\ +/* 64 */ 0x09, 0x09, 0x04, 0x09, 0x09, 0x09, 0x09, 0x26,\ +/* 72 */ 0x26, 0x10, 0x10, 0x00, 0x03, 0x03, 0x0b, 0x0b,\ +/* 80 */ 0x0b, 0x0b, 0x0b, 0x0b, 0x00, 0x26, 0x26, 0x26,\ +/* 88 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x00,\ +/* 96 */ 0x12, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,\ +/* 104 */ 0x00, 0x01, 0x01, 0x01, 0x01, 0x04, 0x04, 0x00,\ +/* 112 */ 0x10, 0x01, 0x01, 0x01, 0x01, 0x10, 0x00, 0x00,\ +/* 120 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 128 */ 0x06, 0x23, 0x0b, 0x01, 0x10, 0x10, 0x00, 0x01,\ +/* 136 */ 0x04, 0x03, 0x03, 0x03, 0x03, 0x03, 0x00, 0x01,\ +/* 144 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,\ +/* 152 */ 0x00, 0x10, 0x10, 0x01, 0x00, 0x00,} /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -12775,28 +9984,17 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); -SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe*,int); -SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe*,int,const char*); -SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...); SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); -SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); -SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe*,int); -#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) -SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N); -#else -# define sqlite3VdbeVerifyNoMallocRequired(A,B) -#endif -SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); +SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); -SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5); SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); -SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); +SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe*, int addr); SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); @@ -12804,7 +10002,6 @@ SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3*,Vdbe*); SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); @@ -12909,7 +10106,7 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const ch # define sqlite3VdbeScanStatus(a,b,c,d,e) #endif -#endif /* SQLITE_VDBE_H */ +#endif /************** End of vdbe.h ************************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -12931,8 +10128,8 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const ch ** at a time and provides a journal for rollback. */ -#ifndef SQLITE_PAGER_H -#define SQLITE_PAGER_H +#ifndef _PAGER_H_ +#define _PAGER_H_ /* ** Default maximum size for persistent journal files. A negative @@ -12985,11 +10182,7 @@ typedef struct PgHdr DbPage; #define PAGER_LOCKINGMODE_EXCLUSIVE 1 /* -** Numeric constants that encode the journalmode. -** -** The numeric values encoded here (other than PAGER_JOURNALMODE_QUERY) -** are exposed in the API via the "PRAGMA journal_mode" command and -** therefore cannot be changed without a compatibility break. +** Numeric constants that encode the journalmode. */ #define PAGER_JOURNALMODE_QUERY (-1) /* Query the value of journalmode */ #define PAGER_JOURNALMODE_DELETE 0 /* Commit by deleting journal file */ @@ -13000,28 +10193,22 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ /* -** Flags that make up the mask passed to sqlite3PagerGet(). +** Flags that make up the mask passed to sqlite3PagerAcquire(). */ #define PAGER_GET_NOCONTENT 0x01 /* Do not load data from disk */ #define PAGER_GET_READONLY 0x02 /* Read-only page is acceptable */ /* ** Flags for sqlite3PagerSetFlags() -** -** Value constraints (enforced via assert()): -** PAGER_FULLFSYNC == SQLITE_FullFSync -** PAGER_CKPT_FULLFSYNC == SQLITE_CkptFullFSync -** PAGER_CACHE_SPILL == SQLITE_CacheSpill */ #define PAGER_SYNCHRONOUS_OFF 0x01 /* PRAGMA synchronous=OFF */ #define PAGER_SYNCHRONOUS_NORMAL 0x02 /* PRAGMA synchronous=NORMAL */ #define PAGER_SYNCHRONOUS_FULL 0x03 /* PRAGMA synchronous=FULL */ -#define PAGER_SYNCHRONOUS_EXTRA 0x04 /* PRAGMA synchronous=EXTRA */ -#define PAGER_SYNCHRONOUS_MASK 0x07 /* Mask for four values above */ -#define PAGER_FULLFSYNC 0x08 /* PRAGMA fullfsync=ON */ -#define PAGER_CKPT_FULLFSYNC 0x10 /* PRAGMA checkpoint_fullfsync=ON */ -#define PAGER_CACHESPILL 0x20 /* PRAGMA cache_spill=ON */ -#define PAGER_FLAGS_MASK 0x38 /* All above except SYNCHRONOUS */ +#define PAGER_SYNCHRONOUS_MASK 0x03 /* Mask for three values above */ +#define PAGER_FULLFSYNC 0x04 /* PRAGMA fullfsync=ON */ +#define PAGER_CKPT_FULLFSYNC 0x08 /* PRAGMA checkpoint_fullfsync=ON */ +#define PAGER_CACHESPILL 0x10 /* PRAGMA cache_spill=ON */ +#define PAGER_FLAGS_MASK 0x1c /* All above except SYNCHRONOUS */ /* ** The remainder of this file contains the declarations of the functions @@ -13045,12 +10232,8 @@ SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); -#ifdef SQLITE_HAS_CODEC -SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager*,Pager*); -#endif SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); -SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); SQLITE_PRIVATE void sqlite3PagerShrink(Pager*); SQLITE_PRIVATE void sqlite3PagerSetFlags(Pager*,unsigned); @@ -13060,10 +10243,10 @@ SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager*); SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager*); SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *, i64); SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager*); -SQLITE_PRIVATE int sqlite3PagerFlush(Pager*); /* Functions used to obtain and release page references. */ -SQLITE_PRIVATE int sqlite3PagerGet(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); +SQLITE_PRIVATE int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); +#define sqlite3PagerGet(A,B,C) sqlite3PagerAcquire(A,B,C,0) SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); SQLITE_PRIVATE void sqlite3PagerRef(DbPage*); SQLITE_PRIVATE void sqlite3PagerUnref(DbPage*); @@ -13095,13 +10278,6 @@ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager); -SQLITE_PRIVATE int sqlite3PagerUseWal(Pager *pPager); -# ifdef SQLITE_ENABLE_SNAPSHOT -SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); -SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); -# endif -#else -# define sqlite3PagerUseWal(x) 0 #endif #ifdef SQLITE_ENABLE_ZIPVFS @@ -13111,19 +10287,17 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager); /* Functions used to query pager state and configuration. */ SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager*); SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager*); -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); -#endif +SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager*); SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*, int); -SQLITE_PRIVATE sqlite3_vfs *sqlite3PagerVfs(Pager*); +SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager*); SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager*); -SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager*); SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager*); +SQLITE_PRIVATE int sqlite3PagerNosync(Pager*); SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*); SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); -SQLITE_PRIVATE void sqlite3PagerClearCache(Pager*); +SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *); SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ @@ -13150,7 +10324,7 @@ SQLITE_PRIVATE void sqlite3PagerRefdump(Pager*); # define enable_simulated_io_errors() #endif -#endif /* SQLITE_PAGER_H */ +#endif /* _PAGER_H_ */ /************** End of pager.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -13184,7 +10358,7 @@ struct PgHdr { sqlite3_pcache_page *pPage; /* Pcache object page handle */ void *pData; /* Page data */ void *pExtra; /* Extra content */ - PgHdr *pDirty; /* Transient list of dirty sorted by pgno */ + PgHdr *pDirty; /* Transient list of dirty pages */ Pager *pPager; /* The pager this page is part of */ Pgno pgno; /* Page number for this page */ #ifdef SQLITE_CHECK_PAGES @@ -13204,15 +10378,14 @@ struct PgHdr { }; /* Bit values for PgHdr.flags */ -#define PGHDR_CLEAN 0x001 /* Page not on the PCache.pDirty list */ -#define PGHDR_DIRTY 0x002 /* Page is on the PCache.pDirty list */ -#define PGHDR_WRITEABLE 0x004 /* Journaled and ready to modify */ -#define PGHDR_NEED_SYNC 0x008 /* Fsync the rollback journal before - ** writing this page to the database */ -#define PGHDR_DONT_WRITE 0x010 /* Do not write content to disk */ -#define PGHDR_MMAP 0x020 /* This is an mmap page object */ +#define PGHDR_DIRTY 0x002 /* Page has changed */ +#define PGHDR_NEED_SYNC 0x004 /* Fsync the rollback journal before + ** writing this page to the database */ +#define PGHDR_NEED_READ 0x008 /* Content is unread */ +#define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ +#define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ -#define PGHDR_WAL_APPEND 0x040 /* Appended to wal file */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ /* Initialize and shutdown the page cache subsystem */ SQLITE_PRIVATE int sqlite3PcacheInitialize(void); @@ -13256,7 +10429,6 @@ SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ SQLITE_PRIVATE void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ -SQLITE_PRIVATE void sqlite3PcacheClearWritable(PCache*); /* Change a page number. Used by incr-vacuum. */ SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr*, Pgno); @@ -13295,11 +10467,6 @@ SQLITE_PRIVATE int sqlite3PcachePagecount(PCache*); SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)); #endif -#if defined(SQLITE_DEBUG) -/* Check invariants on a PgHdr object */ -SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr*); -#endif - /* Set and get the suggested cache-size for the specified pager-cache. ** ** If no global maximum is configured, then the system attempts to limit @@ -13311,13 +10478,6 @@ SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *, int); SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *); #endif -/* Set or get the suggested spill-size for the specified pager-cache. -** -** The spill-size is the minimum number of pages in cache before the cache -** will attempt to spill dirty pages by calling xStress. -*/ -SQLITE_PRIVATE int sqlite3PcacheSetSpillsize(PCache *, int); - /* Free up as much memory as possible from the page cache */ SQLITE_PRIVATE void sqlite3PcacheShrink(PCache*); @@ -13336,13 +10496,11 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); SQLITE_PRIVATE int sqlite3HeaderSizePcache(void); SQLITE_PRIVATE int sqlite3HeaderSizePcache1(void); -/* Number of dirty pages as a percentage of the configured cache size */ -SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache*); - #endif /* _PCACHE_H_ */ /************** End of pcache.h **********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ + /************** Include os.h in the middle of sqliteInt.h ********************/ /************** Begin file os.h **********************************************/ /* @@ -13388,8 +10546,8 @@ SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache*); ** This file contains pre-processor directives related to operating system ** detection and/or setup. */ -#ifndef SQLITE_OS_SETUP_H -#define SQLITE_OS_SETUP_H +#ifndef _OS_SETUP_H_ +#define _OS_SETUP_H_ /* ** Figure out if we are dealing with Unix, Windows, or some other operating @@ -13429,7 +10587,7 @@ SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache*); # endif #endif -#endif /* SQLITE_OS_SETUP_H */ +#endif /* _OS_SETUP_H_ */ /************** End of os_setup.h ********************************************/ /************** Continuing where we left off in os.h *************************/ @@ -13568,7 +10726,7 @@ SQLITE_PRIVATE int sqlite3OsInit(void); /* ** Functions for accessing sqlite3_file methods */ -SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*); +SQLITE_PRIVATE int sqlite3OsClose(sqlite3_file*); SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); @@ -13605,7 +10763,6 @@ SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); #endif /* SQLITE_OMIT_LOAD_EXTENSION */ SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); -SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); /* @@ -13613,7 +10770,7 @@ SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); ** sqlite3_malloc() to obtain space for the file-handle structure. */ SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); -SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); +SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); #endif /* _SQLITE_OS_H_ */ @@ -13695,36 +10852,6 @@ SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); /************** End of mutex.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ -/* The SQLITE_EXTRA_DURABLE compile-time option used to set the default -** synchronous setting to EXTRA. It is no longer supported. -*/ -#ifdef SQLITE_EXTRA_DURABLE -# warning Use SQLITE_DEFAULT_SYNCHRONOUS=3 instead of SQLITE_EXTRA_DURABLE -# define SQLITE_DEFAULT_SYNCHRONOUS 3 -#endif - -/* -** Default synchronous levels. -** -** Note that (for historcal reasons) the PAGER_SYNCHRONOUS_* macros differ -** from the SQLITE_DEFAULT_SYNCHRONOUS value by 1. -** -** PAGER_SYNCHRONOUS DEFAULT_SYNCHRONOUS -** OFF 1 0 -** NORMAL 2 1 -** FULL 3 2 -** EXTRA 4 3 -** -** The "PRAGMA synchronous" statement also uses the zero-based numbers. -** In other words, the zero-based numbers are used for all external interfaces -** and the one-based values are used internally. -*/ -#ifndef SQLITE_DEFAULT_SYNCHRONOUS -# define SQLITE_DEFAULT_SYNCHRONOUS (PAGER_SYNCHRONOUS_FULL-1) -#endif -#ifndef SQLITE_DEFAULT_WAL_SYNCHRONOUS -# define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS -#endif /* ** Each database file to be accessed by the system is an instance @@ -13734,10 +10861,9 @@ SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); ** databases may be attached. */ struct Db { - char *zDbSName; /* Name of this database. (schema name, not filename) */ + char *zName; /* Name of this database */ Btree *pBt; /* The B*Tree structure for this database file */ u8 safety_level; /* How aggressive at syncing data to disk */ - u8 bSyncSet; /* True if "PRAGMA synchronous=N" has been run */ Schema *pSchema; /* Pointer to database schema (possibly shared) */ }; @@ -13748,7 +10874,7 @@ struct Db { ** the Schema for the TEMP databaes (sqlite3.aDb[1]) which is free-standing. ** In shared cache mode, a single Schema object can be shared by multiple ** Btrees that refer to the same underlying BtShared object. -** +** ** Schema objects are automatically deallocated when the last Btree that ** references them is destroyed. The TEMP Schema is manually freed by ** sqlite3_close(). @@ -13773,7 +10899,7 @@ struct Schema { }; /* -** These macros can be used to test, set, or clear bits in the +** These macros can be used to test, set, or clear bits in the ** Db.pSchema->flags field. */ #define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))==(P)) @@ -13822,8 +10948,8 @@ struct Schema { ** lookaside allocations are not used to construct the schema objects. */ struct Lookaside { - u32 bDisable; /* Only operate the lookaside when zero */ u16 sz; /* Size of each buffer in bytes */ + u8 bEnabled; /* False to disable new lookaside allocations */ u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */ int nOut; /* Number of buffers currently checked out */ int mxOut; /* Highwater mark for nOut */ @@ -13837,15 +10963,13 @@ struct LookasideSlot { }; /* -** A hash table for built-in function definitions. (Application-defined -** functions use a regular table table from hash.h.) +** A hash table for function definitions. ** ** Hash each FuncDef structure into one of the FuncDefHash.a[] slots. -** Collisions are on the FuncDef.u.pHash chain. +** Collisions are on the FuncDef.pHash chain. */ -#define SQLITE_FUNC_HASH_SZ 23 struct FuncDefHash { - FuncDef *a[SQLITE_FUNC_HASH_SZ]; /* Hash table for functions */ + FuncDef *a[23]; /* Hash table for functions */ }; #ifdef SQLITE_USER_AUTHENTICATION @@ -13886,15 +11010,6 @@ SQLITE_PRIVATE void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); const char*); #endif -#ifndef SQLITE_OMIT_DEPRECATED -/* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing -** in the style of sqlite3_trace() -*/ -#define SQLITE_TRACE_LEGACY 0x80 -#else -#define SQLITE_TRACE_LEGACY 0 -#endif /* SQLITE_OMIT_DEPRECATED */ - /* ** Each database connection is an instance of the following structure. @@ -13912,19 +11027,16 @@ struct sqlite3 { unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ - int iSysErrno; /* Errno value from last system error */ u16 dbOptFlags; /* Flags to enable/disable optimizations */ u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ - u8 bBenignMalloc; /* Do not require OOMs if true */ u8 dfltLockMode; /* Default locking-mode for attached dbs */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ u8 suppressErr; /* Do not issue error messages if true */ u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ - u8 mTrace; /* zero or more SQLITE_TRACE flags */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ @@ -13945,23 +11057,16 @@ struct sqlite3 { int nVDestroy; /* Number of active OP_VDestroy operations */ int nExtension; /* Number of loaded extensions */ void **aExtension; /* Array of shared library handles */ - int (*xTrace)(u32,void*,void*,void*); /* Trace function */ + void (*xTrace)(void*,const char*); /* Trace function */ void *pTraceArg; /* Argument to the trace function */ void (*xProfile)(void*,const char*,u64); /* Profiling function */ void *pProfileArg; /* Argument to profile function */ - void *pCommitArg; /* Argument to xCommitCallback() */ + void *pCommitArg; /* Argument to xCommitCallback() */ int (*xCommitCallback)(void*); /* Invoked at every commit. */ - void *pRollbackArg; /* Argument to xRollbackCallback() */ + void *pRollbackArg; /* Argument to xRollbackCallback() */ void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK - void *pPreUpdateArg; /* First argument to xPreUpdateCallback */ - void (*xPreUpdateCallback)( /* Registered using sqlite3_preupdate_hook() */ - void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64 - ); - PreUpdate *pPreUpdate; /* Context for active pre-update callback */ -#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ #ifndef SQLITE_OMIT_WAL int (*xWalCallback)(void *, sqlite3 *, const char *, int); void *pWalArg; @@ -13991,7 +11096,7 @@ struct sqlite3 { VTable **aVTrans; /* Virtual tables with open transactions */ VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ #endif - Hash aFunc; /* Hash table of connection functions */ + FuncDefHash aFunc; /* Hash table of connection functions */ Hash aCollSeq; /* All collating sequences */ BusyHandler busyHandler; /* Busy callback */ Db aDbStatic[2]; /* Static space for the 2 default backends */ @@ -14003,8 +11108,8 @@ struct sqlite3 { i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY - /* The following variables are all protected by the STATIC_MASTER - ** mutex, not by sqlite3.mutex. They are used by code in notify.c. + /* The following variables are all protected by the STATIC_MASTER + ** mutex, not by sqlite3.mutex. They are used by code in notify.c. ** ** When X.pUnlockConnection==Y, that means that X is waiting for Y to ** unlock so that it can proceed. @@ -14032,18 +11137,13 @@ struct sqlite3 { /* ** Possible values for the sqlite3.flags. -** -** Value constraints (enforced via assert()): -** SQLITE_FullFSync == PAGER_FULLFSYNC -** SQLITE_CkptFullFSync == PAGER_CKPT_FULLFSYNC -** SQLITE_CacheSpill == PAGER_CACHE_SPILL */ #define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ #define SQLITE_InternChanges 0x00000002 /* Uncommitted Hash table changes */ -#define SQLITE_FullColNames 0x00000004 /* Show full column names on SELECT */ -#define SQLITE_FullFSync 0x00000008 /* Use full fsync on the backend */ -#define SQLITE_CkptFullFSync 0x00000010 /* Use full fsync for checkpoint */ -#define SQLITE_CacheSpill 0x00000020 /* OK to spill pager cache */ +#define SQLITE_FullFSync 0x00000004 /* Use full fsync on the backend */ +#define SQLITE_CkptFullFSync 0x00000008 /* Use full fsync for checkpoint */ +#define SQLITE_CacheSpill 0x00000010 /* OK to spill pager cache */ +#define SQLITE_FullColNames 0x00000020 /* Show full column names on SELECT */ #define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ #define SQLITE_CountRows 0x00000080 /* Count rows changed by INSERT, */ /* DELETE, or UPDATE and return */ @@ -14064,14 +11164,11 @@ struct sqlite3 { #define SQLITE_AutoIndex 0x00100000 /* Enable automatic indexes */ #define SQLITE_PreferBuiltin 0x00200000 /* Preference to built-in funcs */ #define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */ -#define SQLITE_LoadExtFunc 0x00800000 /* Enable load_extension() SQL func */ -#define SQLITE_EnableTrigger 0x01000000 /* True to enable triggers */ -#define SQLITE_DeferFKs 0x02000000 /* Defer all FK constraints */ -#define SQLITE_QueryOnly 0x04000000 /* Disable database changes */ -#define SQLITE_VdbeEQP 0x08000000 /* Debug EXPLAIN QUERY PLAN */ -#define SQLITE_Vacuum 0x10000000 /* Currently in a VACUUM */ -#define SQLITE_CellSizeCk 0x20000000 /* Check btree cell sizes on load */ -#define SQLITE_Fts3Tokenizer 0x40000000 /* Enable fts3_tokenizer(2) */ +#define SQLITE_EnableTrigger 0x00800000 /* True to enable triggers */ +#define SQLITE_DeferFKs 0x01000000 /* Defer all FK constraints */ +#define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ +#define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_Vacuum 0x08000000 /* Currently in a VACUUM */ /* @@ -14091,7 +11188,6 @@ struct sqlite3 { #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ #define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ -#define SQLITE_CursorHints 0x2000 /* Add OP_CursorHint opcodes */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* @@ -14125,33 +11221,28 @@ struct sqlite3 { /* ** Each SQL function is defined by an instance of the following -** structure. For global built-in functions (ex: substr(), max(), count()) -** a pointer to this structure is held in the sqlite3BuiltinFunctions object. -** For per-connection application-defined functions, a pointer to this -** structure is held in the db->aHash hash table. -** -** The u.pHash field is used by the global built-ins. The u.pDestructor -** field is used by per-connection app-def functions. +** structure. A pointer to this structure is stored in the sqlite.aFunc +** hash table. When multiple functions have the same name, the hash table +** points to a linked list of these structures. */ struct FuncDef { - i8 nArg; /* Number of arguments. -1 means unlimited */ + i16 nArg; /* Number of arguments. -1 means unlimited */ u16 funcFlags; /* Some combination of SQLITE_FUNC_* */ void *pUserData; /* User data parameter */ FuncDef *pNext; /* Next function with same name */ - void (*xSFunc)(sqlite3_context*,int,sqlite3_value**); /* func or agg-step */ - void (*xFinalize)(sqlite3_context*); /* Agg finalizer */ - const char *zName; /* SQL name of the function. */ - union { - FuncDef *pHash; /* Next with a different name but the same hash */ - FuncDestructor *pDestructor; /* Reference counted destructor function */ - } u; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); /* Regular function */ + void (*xStep)(sqlite3_context*,int,sqlite3_value**); /* Aggregate step */ + void (*xFinalize)(sqlite3_context*); /* Aggregate finalizer */ + char *zName; /* SQL name of the function. */ + FuncDef *pHash; /* Next with a different name but the same hash */ + FuncDestructor *pDestructor; /* Reference counted destructor function */ }; /* ** This structure encapsulates a user-function destructor callback (as ** configured using create_function_v2()) and a reference counter. When ** create_function_v2() is called to create a function with a destructor, -** a single object of this type is allocated. FuncDestructor.nRef is set to +** a single object of this type is allocated. FuncDestructor.nRef is set to ** the number of FuncDef objects created (either 1 or 3, depending on whether ** or not the specified encoding is SQLITE_ANY). The FuncDef.pDestructor ** member of each of the new FuncDef objects is set to point to the allocated @@ -14169,52 +11260,36 @@ struct FuncDestructor { /* ** Possible values for FuncDef.flags. Note that the _LENGTH and _TYPEOF -** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. And -** SQLITE_FUNC_CONSTANT must be the same as SQLITE_DETERMINISTIC. There +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There ** are assert() statements in the code to verify this. -** -** Value constraints (enforced via assert()): -** SQLITE_FUNC_MINMAX == NC_MinMaxAgg == SF_MinMaxAgg -** SQLITE_FUNC_LENGTH == OPFLAG_LENGTHARG -** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG -** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API -** SQLITE_FUNC_ENCMASK depends on SQLITE_UTF* macros in the API -*/ -#define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ -#define SQLITE_FUNC_LIKE 0x0004 /* Candidate for the LIKE optimization */ -#define SQLITE_FUNC_CASE 0x0008 /* Case-sensitive LIKE-type function */ -#define SQLITE_FUNC_EPHEM 0x0010 /* Ephemeral. Delete with VDBE */ -#define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/ -#define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ -#define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ -#define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x0200 /* Built-in coalesce() or ifnull() */ -#define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ -#define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ -#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ -#define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a - ** single query - might change over time */ +*/ +#define SQLITE_FUNC_ENCMASK 0x003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ +#define SQLITE_FUNC_LIKE 0x004 /* Candidate for the LIKE optimization */ +#define SQLITE_FUNC_CASE 0x008 /* Case-sensitive LIKE-type function */ +#define SQLITE_FUNC_EPHEM 0x010 /* Ephemeral. Delete with VDBE */ +#define SQLITE_FUNC_NEEDCOLL 0x020 /* sqlite3GetFuncCollSeq() might be called */ +#define SQLITE_FUNC_LENGTH 0x040 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x080 /* Built-in typeof() function */ +#define SQLITE_FUNC_COUNT 0x100 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x200 /* Built-in coalesce() or ifnull() */ +#define SQLITE_FUNC_UNLIKELY 0x400 /* Built-in unlikely() function */ +#define SQLITE_FUNC_CONSTANT 0x800 /* Constant inputs give a constant output */ +#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are ** used to create the initializers for the FuncDef structures. ** ** FUNCTION(zName, nArg, iArg, bNC, xFunc) -** Used to create a scalar function definition of a function zName +** Used to create a scalar function definition of a function zName ** implemented by C function xFunc that accepts nArg arguments. The ** value passed as iArg is cast to a (void*) and made available -** as the user-data (sqlite3_user_data()) for the function. If +** as the user-data (sqlite3_user_data()) for the function. If ** argument bNC is true, then the SQLITE_FUNC_NEEDCOLL flag is set. ** ** VFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. ** -** DFUNCTION(zName, nArg, iArg, bNC, xFunc) -** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and -** adds the SQLITE_FUNC_SLOCHNG flag. Used for date & time functions -** and functions like sqlite_version() that can change, but not during -** a single query. -** ** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) ** Used to create an aggregate function definition implemented by ** the C functions xStep and xFinal. The first four parameters @@ -14222,8 +11297,8 @@ struct FuncDestructor { ** FUNCTION(). ** ** LIKEFUNC(zName, nArg, pArg, flags) -** Used to create a scalar function definition of a function zName -** that accepts nArg arguments and is implemented by a call to C +** Used to create a scalar function definition of a function zName +** that accepts nArg arguments and is implemented by a call to C ** function likeFunc. Argument pArg is cast to a (void *) and made ** available as the function user-data (sqlite3_user_data()). The ** FuncDef.flags variable is set to the value passed as the flags @@ -14231,28 +11306,25 @@ struct FuncDestructor { */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } -#define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - pArg, 0, xFunc, 0, #zName, } + {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + pArg, 0, xFunc, 0, 0, #zName, 0, 0} #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ - (void *)arg, 0, likeFunc, 0, #zName, {0} } + (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0} #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} + SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} #define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ - SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} + SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} /* ** All current savepoints are stored in a linked list starting at @@ -14286,7 +11358,6 @@ struct Module { const char *zName; /* Name passed to create_module() */ void *pAux; /* pAux passed to create_module() */ void (*xDestroy)(void *); /* Module destructor function */ - Table *pEpoTab; /* Eponymous table for this module */ }; /* @@ -14294,12 +11365,14 @@ struct Module { ** of this structure. */ struct Column { - char *zName; /* Name of this column, \000, then the type */ + char *zName; /* Name of this column */ Expr *pDflt; /* Default value of this column */ + char *zDflt; /* Original text of the default value */ + char *zType; /* Data type for this column */ char *zColl; /* Collating sequence. If NULL, use the default */ u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ char affinity; /* One of the SQLITE_AFF_... values */ - u8 szEst; /* Estimated size of value in this column. sizeof(INT)==1 */ + u8 szEst; /* Estimated size of this column. INT==1 */ u8 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; @@ -14307,7 +11380,6 @@ struct Column { */ #define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ #define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ -#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ /* ** A "Collating Sequence" is defined by an instance of the following @@ -14331,23 +11403,22 @@ struct CollSeq { */ #define SQLITE_SO_ASC 0 /* Sort in ascending order */ #define SQLITE_SO_DESC 1 /* Sort in ascending order */ -#define SQLITE_SO_UNDEFINED -1 /* No sort order specified */ /* ** Column affinity types. ** ** These used to have mnemonic name like 'i' for SQLITE_AFF_INTEGER and ** 't' for SQLITE_AFF_TEXT. But we can save a little space and improve -** the speed a little by numbering the values consecutively. +** the speed a little by numbering the values consecutively. ** ** But rather than start with 0 or 1, we begin with 'A'. That way, ** when multiple affinity types are concatenated into a string and ** used as the P4 operand, they will be more readable. ** ** Note also that the numeric types are grouped together so that testing -** for a numeric type is a single comparison. And the BLOB type is first. +** for a numeric type is a single comparison. And the NONE type is first. */ -#define SQLITE_AFF_BLOB 'A' +#define SQLITE_AFF_NONE 'A' #define SQLITE_AFF_TEXT 'B' #define SQLITE_AFF_NUMERIC 'C' #define SQLITE_AFF_INTEGER 'D' @@ -14357,7 +11428,7 @@ struct CollSeq { /* ** The SQLITE_AFF_MASK values masks off the significant bits of an -** affinity value. +** affinity value. */ #define SQLITE_AFF_MASK 0x47 @@ -14370,7 +11441,6 @@ struct CollSeq { ** operator is NULL. It is added to certain comparison operators to ** prove that the operands are always NOT NULL. */ -#define SQLITE_KEEPNULL 0x08 /* Used by vector == or <> */ #define SQLITE_JUMPIFNULL 0x10 /* jumps if either operand is NULL */ #define SQLITE_STOREP2 0x20 /* Store result in reg[P2] rather than jump */ #define SQLITE_NULLEQ 0x80 /* NULL=NULL */ @@ -14378,20 +11448,20 @@ struct CollSeq { /* ** An object of this type is created for each virtual table present in -** the database schema. +** the database schema. ** ** If the database schema is shared, then there is one instance of this ** structure for each database connection (sqlite3*) that uses the shared ** schema. This is because each database connection requires its own unique -** instance of the sqlite3_vtab* handle used to access the virtual table -** implementation. sqlite3_vtab* handles can not be shared between -** database connections, even when the rest of the in-memory database +** instance of the sqlite3_vtab* handle used to access the virtual table +** implementation. sqlite3_vtab* handles can not be shared between +** database connections, even when the rest of the in-memory database ** schema is shared, as the implementation often stores the database ** connection handle passed to it via the xConnect() or xCreate() method ** during initialization internally. This database connection handle may -** then be used by the virtual table implementation to access real tables -** within the database. So that they appear as part of the callers -** transaction, these accesses need to be made via the same database +** then be used by the virtual table implementation to access real tables +** within the database. So that they appear as part of the callers +** transaction, these accesses need to be made via the same database ** connection as that used to execute SQL operations on the virtual table. ** ** All VTable objects that correspond to a single table in a shared @@ -14403,19 +11473,19 @@ struct CollSeq { ** sqlite3_vtab* handle in the compiled query. ** ** When an in-memory Table object is deleted (for example when the -** schema is being reloaded for some reason), the VTable objects are not -** deleted and the sqlite3_vtab* handles are not xDisconnect()ed +** schema is being reloaded for some reason), the VTable objects are not +** deleted and the sqlite3_vtab* handles are not xDisconnect()ed ** immediately. Instead, they are moved from the Table.pVTable list to ** another linked list headed by the sqlite3.pDisconnect member of the -** corresponding sqlite3 structure. They are then deleted/xDisconnected +** corresponding sqlite3 structure. They are then deleted/xDisconnected ** next time a statement is prepared using said sqlite3*. This is done ** to avoid deadlock issues involving multiple sqlite3.mutex mutexes. ** Refer to comments above function sqlite3VtabUnlockList() for an ** explanation as to why it is safe to add an entry to an sqlite3.pDisconnect ** list without holding the corresponding sqlite3.mutex mutex. ** -** The memory for objects of this type is always allocated by -** sqlite3DbMalloc(), using the connection handle stored in VTable.db as +** The memory for objects of this type is always allocated by +** sqlite3DbMalloc(), using the connection handle stored in VTable.db as ** the first argument. */ struct VTable { @@ -14439,8 +11509,9 @@ struct Table { Select *pSelect; /* NULL for tables. Points to definition if a view. */ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ +#ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ - /* ... also used as column name list in a VIEW */ +#endif int tnum; /* Root BTree page for this table */ i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ @@ -14457,7 +11528,7 @@ struct Table { #endif #ifndef SQLITE_OMIT_VIRTUALTABLE int nModuleArg; /* Number of arguments to the module */ - char **azModuleArg; /* 0: module 1: schema 2: vtab name 3...: args */ + char **azModuleArg; /* Text of all module args. [0] is module name */ VTable *pVTable; /* List of VTable objects. */ #endif Trigger *pTrigger; /* List of triggers stored in pSchema */ @@ -14468,7 +11539,7 @@ struct Table { /* ** Allowed values for Table.tabFlags. ** -** TF_OOOHidden applies to tables or view that have hidden columns that are +** TF_OOOHidden applies to virtual tables that have hidden columns that are ** followed by non-hidden columns. Example: "CREATE VIRTUAL TABLE x USING ** vtab1(a HIDDEN, b);". Since "b" is a non-hidden column but "a" is hidden, ** the TF_OOOHidden attribute would apply in this case. Such tables require @@ -14479,9 +11550,8 @@ struct Table { #define TF_HasPrimaryKey 0x04 /* Table has a primary key */ #define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ #define TF_Virtual 0x10 /* Is a virtual table */ -#define TF_WithoutRowid 0x20 /* No rowid. PRIMARY KEY is the key */ -#define TF_NoVisibleRowid 0x40 /* No user-visible "rowid" column */ -#define TF_OOOHidden 0x80 /* Out-of-Order hidden columns */ +#define TF_WithoutRowid 0x20 /* No rowid used. PRIMARY KEY is the key */ +#define TF_OOOHidden 0x40 /* Out-of-Order hidden columns */ /* @@ -14491,31 +11561,14 @@ struct Table { */ #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) (((X)->tabFlags & TF_Virtual)!=0) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) #else # define IsVirtual(X) 0 +# define IsHiddenColumn(X) 0 #endif -/* -** Macros to determine if a column is hidden. IsOrdinaryHiddenColumn() -** only works for non-virtual tables (ordinary tables and views) and is -** always false unless SQLITE_ENABLE_HIDDEN_COLUMNS is defined. The -** IsHiddenColumn() macro is general purpose. -*/ -#if defined(SQLITE_ENABLE_HIDDEN_COLUMNS) -# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) -# define IsOrdinaryHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) -#elif !defined(SQLITE_OMIT_VIRTUALTABLE) -# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) -# define IsOrdinaryHiddenColumn(X) 0 -#else -# define IsHiddenColumn(X) 0 -# define IsOrdinaryHiddenColumn(X) 0 -#endif - - /* Does the table have a rowid */ #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) -#define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) /* ** Each foreign key constraint is an instance of the following structure. @@ -14583,7 +11636,7 @@ struct FKey { ** key is set to NULL. CASCADE means that a DELETE or UPDATE of the ** referenced table row is propagated into the row that holds the ** foreign key. -** +** ** The following symbolic values are used to record which type ** of action to take. */ @@ -14604,7 +11657,7 @@ struct FKey { /* ** An instance of the following structure is passed as the first -** argument to sqlite3VdbeKeyCompare and is used to control the +** argument to sqlite3VdbeKeyCompare and is used to control the ** comparison of the two index keys. ** ** Note that aSortOrder[] and aColl[] have nField+1 slots. There @@ -14622,8 +11675,9 @@ struct KeyInfo { }; /* -** This object holds a record which has been parsed out into individual -** fields, for the purposes of doing a comparison. +** An instance of the following structure holds information about a +** single index record that has already been parsed out into individual +** values. ** ** A record is an object that contains one or more fields of data. ** Records are used to store the content of a table row and to store @@ -14631,40 +11685,20 @@ struct KeyInfo { ** the OP_MakeRecord opcode of the VDBE and is disassembled by the ** OP_Column opcode. ** -** An instance of this object serves as a "key" for doing a search on -** an index b+tree. The goal of the search is to find the entry that -** is closed to the key described by this object. This object might hold -** just a prefix of the key. The number of fields is given by -** pKeyInfo->nField. -** -** The r1 and r2 fields are the values to return if this key is less than -** or greater than a key in the btree, respectively. These are normally -** -1 and +1 respectively, but might be inverted to +1 and -1 if the b-tree -** is in DESC order. -** -** The key comparison functions actually return default_rc when they find -** an equals comparison. default_rc can be -1, 0, or +1. If there are -** multiple entries in the b-tree with the same key (when only looking -** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to -** cause the search to find the last match, or +1 to cause the search to -** find the first match. -** -** The key comparison functions will set eqSeen to true if they ever -** get and equal results when comparing this structure to a b-tree record. -** When default_rc!=0, the search might end up on the record immediately -** before the first match or immediately after the last match. The -** eqSeen field will indicate whether or not an exact match exists in the -** b-tree. +** This structure holds a record that has already been disassembled +** into its constituent fields. +** +** The r1 and r2 member variables are only used by the optimized comparison +** functions vdbeRecordCompareInt() and vdbeRecordCompareString(). */ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ - Mem *aMem; /* Values */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ - i8 r1; /* Value to return if (lhs > rhs) */ - i8 r2; /* Value to return if (rhs < lhs) */ - u8 eqSeen; /* True if an equality comparison has been seen */ + Mem *aMem; /* Values */ + int r1; /* Value to return if (lhs > rhs) */ + int r2; /* Value to return if (rhs < lhs) */ }; @@ -14682,7 +11716,7 @@ struct UnpackedRecord { ** In the Table structure describing Ex1, nCol==3 because there are ** three columns in the table. In the Index structure describing ** Ex2, nColumn==2 since 2 of the 3 columns of Ex1 are indexed. -** The value of aiColumn is {2, 0}. aiColumn[0]==2 because the +** The value of aiColumn is {2, 0}. aiColumn[0]==2 because the ** first column to be indexed (c3) has an index of 2 in Ex1.aCol[]. ** The second column to be indexed (c1) has an index of 0 in ** Ex1.aCol[], hence Ex2.aiColumn[1]==0. @@ -14690,17 +11724,9 @@ struct UnpackedRecord { ** The Index.onError field determines whether or not the indexed columns ** must be unique and what to do if they are not. When Index.onError=OE_None, ** it means this is not a unique index. Otherwise it is a unique index -** and the value of Index.onError indicate the which conflict resolution +** and the value of Index.onError indicate the which conflict resolution ** algorithm to employ whenever an attempt is made to insert a non-unique ** element. -** -** While parsing a CREATE TABLE or CREATE INDEX statement in order to -** generate VDBE code (as opposed to parsing one read from an sqlite_master -** table as part of parsing an existing database schema), transient instances -** of this structure may be created. In this case the Index.tnum variable is -** used to store the address of a VDBE instruction, not a database page -** number (it cannot - the database page is not allocated until the VDBE -** program is executed). See convertToWithoutRowidTable() for details. */ struct Index { char *zName; /* Name of this index */ @@ -14711,9 +11737,8 @@ struct Index { Index *pNext; /* The next index associated with the same table */ Schema *pSchema; /* Schema containing this index */ u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ - const char **azColl; /* Array of collation sequence names for index */ + char **azColl; /* Array of collation sequence names for index */ Expr *pPartIdxWhere; /* WHERE clause for partial indices */ - ExprList *aColExpr; /* Column expressions */ int tnum; /* DB Page containing root of this index */ LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nKeyCol; /* Number of columns forming the key */ @@ -14748,14 +11773,8 @@ struct Index { /* Return true if index X is a UNIQUE index */ #define IsUniqueIndex(X) ((X)->onError!=OE_None) -/* The Index.aiColumn[] values are normally positive integer. But -** there are some negative values that have special meaning: -*/ -#define XN_ROWID (-1) /* Indexed column is the rowid */ -#define XN_EXPR (-2) /* Indexed column is an expression */ - /* -** Each sample stored in the sqlite_stat3 table is represented in memory +** Each sample stored in the sqlite_stat3 table is represented in memory ** using a structure of this type. See documentation at the top of the ** analyze.c source file for additional information. */ @@ -14850,9 +11869,9 @@ typedef int ynVar; ** to represent the greater-than-or-equal-to operator in the expression ** tree. ** -** If the expression is an SQL literal (TK_INTEGER, TK_FLOAT, TK_BLOB, +** If the expression is an SQL literal (TK_INTEGER, TK_FLOAT, TK_BLOB, ** or TK_STRING), then Expr.token contains the text of the SQL literal. If -** the expression is a variable (TK_VARIABLE), then Expr.token contains the +** the expression is a variable (TK_VARIABLE), then Expr.token contains the ** variable name. Finally, if the expression is an SQL function (TK_FUNCTION), ** then Expr.token contains the name of the function. ** @@ -14863,7 +11882,7 @@ typedef int ynVar; ** a CASE expression or an IN expression of the form " IN (, ...)". ** Expr.x.pSelect is used if the expression is a sub-select or an expression of ** the form " IN (SELECT ...)". If the EP_xIsSelect bit is set in the -** Expr.flags mask, then Expr.x.pSelect is valid. Otherwise, Expr.x.pList is +** Expr.flags mask, then Expr.x.pSelect is valid. Otherwise, Expr.x.pList is ** valid. ** ** An expression of the form ID or ID.ID refers to a column in a table. @@ -14874,8 +11893,8 @@ typedef int ynVar; ** value is also stored in the Expr.iAgg column in the aggregate so that ** it can be accessed after all aggregates are computed. ** -** If the expression is an unbound variable marker (a question mark -** character '?' in the original SQL) then the Expr.iTable holds the index +** If the expression is an unbound variable marker (a question mark +** character '?' in the original SQL) then the Expr.iTable holds the index ** number for that variable. ** ** If the expression is a subquery then Expr.iColumn holds an integer @@ -14914,7 +11933,7 @@ struct Expr { /* If the EP_TokenOnly flag is set in the Expr.flags mask, then no ** space is allocated for the fields below this point. An attempt to - ** access them will result in a segfault or malfunction. + ** access them will result in a segfault or malfunction. *********************************************************************/ Expr *pLeft; /* Left subnode */ @@ -14935,11 +11954,9 @@ struct Expr { int iTable; /* TK_COLUMN: cursor number of table holding column ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old - ** EP_Unlikely: 134217728 times likelihood - ** TK_SELECT: 1st register of result vector */ + ** EP_Unlikely: 134217728 times likelihood */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. - ** TK_VARIABLE: variable number (always >= 1). - ** TK_SELECT_COLUMN: column of the result vector */ + ** TK_VARIABLE: variable number (always >= 1). */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ u8 op2; /* TK_REGISTER: original value of Expr.op @@ -14971,11 +11988,9 @@ struct Expr { #define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ +#define EP_ConstFunc 0x080000 /* Node is a SQLITE_FUNC_CONSTANT function */ #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ #define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ -#define EP_Alias 0x400000 /* Is an alias for a result set column */ -#define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ /* ** Combinations of two or more EP_* flags @@ -14983,7 +11998,7 @@ struct Expr { #define EP_Propagate (EP_Collate|EP_Subquery) /* Propagate these bits up tree */ /* -** These macros can be used to test, set, or clear bits in the +** These macros can be used to test, set, or clear bits in the ** Expr.flags field. */ #define ExprHasProperty(E,P) (((E)->flags&(P))!=0) @@ -15002,8 +12017,8 @@ struct Expr { #endif /* -** Macros to determine the number of bytes required by a normal Expr -** struct, an Expr struct with the EP_Reduced flag set in Expr.flags +** Macros to determine the number of bytes required by a normal Expr +** struct, an Expr struct with the EP_Reduced flag set in Expr.flags ** and an Expr struct with the EP_TokenOnly flag set. */ #define EXPR_FULLSIZE sizeof(Expr) /* Full size */ @@ -15011,7 +12026,7 @@ struct Expr { #define EXPR_TOKENONLYSIZE offsetof(Expr,pLeft) /* Fewer features */ /* -** Flags passed to the sqlite3ExprDup() function. See the header comment +** Flags passed to the sqlite3ExprDup() function. See the header comment ** above sqlite3ExprDup() for details. */ #define EXPRDUP_REDUCE 0x0001 /* Used reduced-size Expr nodes */ @@ -15093,11 +12108,7 @@ struct IdList { ** tables in a join to 32 instead of 64. But it also reduces the size ** of the library by 738 bytes on ix86. */ -#ifdef SQLITE_BITMASK_TYPE - typedef SQLITE_BITMASK_TYPE Bitmask; -#else - typedef u64 Bitmask; -#endif +typedef u64 Bitmask; /* ** The number of bits in a Bitmask. "BMS" means "BitMask Size". @@ -15109,7 +12120,6 @@ struct IdList { */ #define MASKBIT(n) (((Bitmask)1)<<(n)) #define MASKBIT32(n) (((unsigned int)1)<<(n)) -#define ALLBITS ((Bitmask)-1) /* ** The following structure describes the FROM clause of a SELECT statement. @@ -15143,15 +12153,11 @@ struct SrcList { int addrFillSub; /* Address of subroutine to manifest a subquery */ int regReturn; /* Register holding return address of addrFillSub */ int regResult; /* Registers holding results of a co-routine */ - struct { - u8 jointype; /* Type of join between this table and the previous */ - unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ - unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ - unsigned isTabFunc :1; /* True if table-valued-function syntax */ - unsigned isCorrelated :1; /* True if sub-query is correlated */ - unsigned viaCoroutine :1; /* Implemented as a co-routine */ - unsigned isRecursive :1; /* True for recursive reference in WITH */ - } fg; + u8 jointype; /* Type of join between this able and the previous */ + unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif @@ -15159,11 +12165,8 @@ struct SrcList { Expr *pOn; /* The ON clause of a join */ IdList *pUsing; /* The USING clause of a join */ Bitmask colUsed; /* Bit N (1<" clause */ - ExprList *pFuncArg; /* Arguments to table-valued-function */ - } u1; - Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ + char *zIndex; /* Identifier from "INDEXED BY " clause */ + Index *pIndex; /* Index structure corresponding to zIndex, if any */ } a[1]; /* One entry for each identifier on the list */ }; @@ -15182,28 +12185,21 @@ struct SrcList { /* ** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin() ** and the WhereInfo.wctrlFlags member. -** -** Value constraints (enforced via assert()): -** WHERE_USE_LIMIT == SF_FixedLimit */ #define WHERE_ORDERBY_NORMAL 0x0000 /* No-op */ #define WHERE_ORDERBY_MIN 0x0001 /* ORDER BY processing for min() func */ #define WHERE_ORDERBY_MAX 0x0002 /* ORDER BY processing for max() func */ #define WHERE_ONEPASS_DESIRED 0x0004 /* Want to do one-pass UPDATE/DELETE */ -#define WHERE_ONEPASS_MULTIROW 0x0008 /* ONEPASS is ok with multiple rows */ -#define WHERE_DUPLICATES_OK 0x0010 /* Ok to return a row more than once */ -#define WHERE_OR_SUBCLAUSE 0x0020 /* Processing a sub-WHERE as part of - ** the OR optimization */ -#define WHERE_GROUPBY 0x0040 /* pOrderBy is really a GROUP BY */ -#define WHERE_DISTINCTBY 0x0080 /* pOrderby is really a DISTINCT clause */ -#define WHERE_WANT_DISTINCT 0x0100 /* All output needs to be distinct */ -#define WHERE_SORTBYGROUP 0x0200 /* Support sqlite3WhereIsSorted() */ -#define WHERE_SEEK_TABLE 0x0400 /* Do not defer seeks on main table */ -#define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ - /* 0x1000 not currently used */ - /* 0x2000 not currently used */ -#define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ - /* 0x8000 not currently used */ +#define WHERE_DUPLICATES_OK 0x0008 /* Ok to return a row more than once */ +#define WHERE_OMIT_OPEN_CLOSE 0x0010 /* Table cursors are already open */ +#define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ +#define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ +#define WHERE_NO_AUTOINDEX 0x0080 /* Disallow automatic indexes */ +#define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ +#define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ +#define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ +#define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ +#define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -15221,12 +12217,12 @@ struct SrcList { ** pEList corresponds to the result set of a SELECT and is NULL for ** other statements. ** -** NameContexts can be nested. When resolving names, the inner-most +** NameContexts can be nested. When resolving names, the inner-most ** context is searched first. If no match is found, the next outer ** context is checked. If there is still no match, the next context ** is checked. This process continues until either a match is found ** or all contexts are check. When a match is found, the nRef member of -** the context containing the match is incremented. +** the context containing the match is incremented. ** ** Each subquery gets a new NameContext. The pNext field points to the ** NameContext in the parent query. Thus the process of scanning the @@ -15247,18 +12243,15 @@ struct NameContext { /* ** Allowed values for the NameContext, ncFlags field. ** -** Value constraints (all checked via assert()): -** NC_HasAgg == SF_HasAgg -** NC_MinMaxAgg == SF_MinMaxAgg == SQLITE_FUNC_MINMAX -** +** Note: NC_MinMaxAgg must have the same value as SF_MinMaxAgg and +** SQLITE_FUNC_MINMAX. +** */ #define NC_AllowAgg 0x0001 /* Aggregate functions are allowed here */ -#define NC_PartIdx 0x0002 /* True if resolving a partial index WHERE */ +#define NC_HasAgg 0x0002 /* One or more aggregate functions seen */ #define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ #define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ -#define NC_HasAgg 0x0010 /* One or more aggregate functions seen */ -#define NC_IdxExpr 0x0020 /* True if resolving columns of CREATE INDEX */ -#define NC_VarSelect 0x0040 /* A correlated subquery has been seen */ +#define NC_PartIdx 0x0010 /* True if resolving a partial index WHERE */ #define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ /* @@ -15284,13 +12277,13 @@ struct NameContext { struct Select { ExprList *pEList; /* The fields of the result */ u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ - LogEst nSelectRow; /* Estimated number of result rows */ - u32 selFlags; /* Various SF_* values */ + u16 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ #if SELECTTRACE_ENABLED char zSelName[12]; /* Symbolic name of this SELECT use for debugging */ #endif int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ + u64 nSelectRow; /* Estimated number of result rows */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ ExprList *pGroupBy; /* The GROUP BY clause */ @@ -15306,30 +12299,21 @@ struct Select { /* ** Allowed values for Select.selFlags. The "SF" prefix stands for ** "Select Flag". -** -** Value constraints (all checked via assert()) -** SF_HasAgg == NC_HasAgg -** SF_MinMaxAgg == NC_MinMaxAgg == SQLITE_FUNC_MINMAX -** SF_FixedLimit == WHERE_USE_LIMIT -*/ -#define SF_Distinct 0x00001 /* Output should be DISTINCT */ -#define SF_All 0x00002 /* Includes the ALL keyword */ -#define SF_Resolved 0x00004 /* Identifiers have been resolved */ -#define SF_Aggregate 0x00008 /* Contains agg functions or a GROUP BY */ -#define SF_HasAgg 0x00010 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x00020 /* Uses the OpenEphemeral opcode */ -#define SF_Expanded 0x00040 /* sqlite3SelectExpand() called on this */ -#define SF_HasTypeInfo 0x00080 /* FROM subqueries have Table metadata */ -#define SF_Compound 0x00100 /* Part of a compound query */ -#define SF_Values 0x00200 /* Synthesized from VALUES clause */ -#define SF_MultiValue 0x00400 /* Single VALUES term with multiple rows */ -#define SF_NestedFrom 0x00800 /* Part of a parenthesized FROM clause */ -#define SF_MinMaxAgg 0x01000 /* Aggregate containing min() or max() */ -#define SF_Recursive 0x02000 /* The recursive part of a recursive CTE */ -#define SF_FixedLimit 0x04000 /* nSelectRow set by a constant LIMIT */ -#define SF_MaybeConvert 0x08000 /* Need convertCompoundSelectToSubquery() */ -#define SF_Converted 0x10000 /* By convertCompoundSelectToSubquery() */ -#define SF_IncludeHidden 0x20000 /* Include hidden columns in output */ +*/ +#define SF_Distinct 0x0001 /* Output should be DISTINCT */ +#define SF_Resolved 0x0002 /* Identifiers have been resolved */ +#define SF_Aggregate 0x0004 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ +#define SF_Compound 0x0040 /* Part of a compound query */ +#define SF_Values 0x0080 /* Synthesized from VALUES clause */ +#define SF_MultiValue 0x0100 /* Single VALUES term with multiple rows */ +#define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ +#define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ +#define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ +#define SF_MinMaxAgg 0x1000 /* Aggregate containing min() or max() */ +#define SF_Converted 0x2000 /* By convertCompoundSelectToSubquery() */ /* @@ -15337,7 +12321,7 @@ struct Select { ** by one of the following macros. The "SRT" prefix means "SELECT Result ** Type". ** -** SRT_Union Store results as a key in a temporary index +** SRT_Union Store results as a key in a temporary index ** identified by pDest->iSDParm. ** ** SRT_Except Remove results from the temporary index pDest->iSDParm. @@ -15361,7 +12345,7 @@ struct Select { ** of the query. This destination implies "LIMIT 1". ** ** SRT_Set The result must be a single column. Store each -** row of result as the key in table pDest->iSDParm. +** row of result as the key in table pDest->iSDParm. ** Apply the affinity pDest->affSdst before storing ** results. Used to implement "IN (SELECT ...)". ** @@ -15421,7 +12405,7 @@ struct Select { */ struct SelectDest { u8 eDest; /* How to dispose of the results. On of SRT_* above. */ - char *zAffSdst; /* Affinity used when eDest==SRT_Set */ + char affSdst; /* Affinity used when eDest==SRT_Set */ int iSDParm; /* A parameter used by the eDest disposal method */ int iSdst; /* Base register where results are written */ int nSdst; /* Number of registers allocated */ @@ -15429,11 +12413,11 @@ struct SelectDest { }; /* -** During code generation of statements that do inserts into AUTOINCREMENT +** During code generation of statements that do inserts into AUTOINCREMENT ** tables, the following information is attached to the Table.u.autoInc.p ** pointer of each autoincrement table to record some side information that ** the code generator needs. We have to keep per-table autoincrement -** information in case inserts are done within triggers. Triggers do not +** information in case inserts are down within triggers. Triggers do not ** normally coordinate their activities, but we do need to coordinate the ** loading and saving of autoincrement information. */ @@ -15452,7 +12436,7 @@ struct AutoincInfo { #endif /* -** At least one instance of the following structure is created for each +** At least one instance of the following structure is created for each ** trigger that may be fired while parsing an INSERT, UPDATE or DELETE ** statement. All such objects are stored in the linked list headed at ** Parse.pTriggerPrg and deleted once statement compilation has been @@ -15465,7 +12449,7 @@ struct AutoincInfo { ** values for both pTrigger and orconf. ** ** The TriggerPrg.aColmask[0] variable is set to a mask of old.* columns -** accessed (or set to 0 for triggers fired as a result of INSERT +** accessed (or set to 0 for triggers fired as a result of INSERT ** statements). Similarly, the TriggerPrg.aColmask[1] variable is set to ** a mask of new.* columns used by the program. */ @@ -15506,7 +12490,7 @@ struct TriggerPrg { ** is constant but the second part is reset at the beginning and end of ** each recursion. ** -** The nTableLock and aTableLock variables are only used if the shared-cache +** The nTableLock and aTableLock variables are only used if the shared-cache ** feature is enabled (if sqlite3Tsd()->useSharedData is true). They are ** used to store the set of table-locks required by the statement being ** compiled. Function sqlite3TableLock() is used to add entries to the @@ -15525,25 +12509,35 @@ struct Parse { u8 mayAbort; /* True if statement may throw an ABORT exception */ u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ - u8 disableLookaside; /* Number of times lookaside has been disabled */ - u8 nColCache; /* Number of entries in aColCache[] */ + int aTempReg[8]; /* Holding area for temporary registers */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ int nTab; /* Number of previously allocated VDBE cursors */ int nMem; /* Number of memory cells used so far */ + int nSet; /* Number of sets used so far */ + int nOnce; /* Number of OP_Once instructions so far */ int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ - int szOpAlloc; /* Bytes of memory space allocated for Vdbe.aOp[] */ + int iFixedOp; /* Never back out opcodes iFixedOp-1 or earlier */ int ckBase; /* Base register of data during check constraints */ - int iSelfTab; /* Table of an index whose exprs are being coded */ + int iPartIdxTab; /* Table corresponding to a partial index */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ int nLabel; /* Number of labels used */ int *aLabel; /* Space to hold the labels */ + struct yColCache { + int iTable; /* Table cursor number */ + i16 iColumn; /* Table column number */ + u8 tempReg; /* iReg is a temp register that needs to be freed */ + int iLevel; /* Nesting level */ + int iReg; /* Reg with value of this column. 0 means none. */ + int lru; /* Least recently used entry has the smallest value */ + } aColCache[SQLITE_N_COLCACHE]; /* One for each column cache entry */ ExprList *pConstExpr;/* Constant expressions */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ + int cookieValue[SQLITE_MAX_ATTACHED+2]; /* Values of cookies to verify */ int regRowid; /* Register holding rowid of CREATE TABLE entry */ int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ @@ -15556,9 +12550,12 @@ struct Parse { TableLock *aTableLock; /* Required table locks for shared-cache mode */ #endif AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ + + /* Information used while coding trigger programs. */ Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ int addrCrTab; /* Address of OP_CreateTable opcode on CREATE TABLE */ + int addrSkipPK; /* Address of instruction to skip PRIMARY KEY index */ u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ @@ -15566,40 +12563,23 @@ struct Parse { u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ u8 disableTriggers; /* True to disable triggers */ - /************************************************************************** - ** Fields above must be initialized to zero. The fields that follow, - ** down to the beginning of the recursive section, do not need to be - ** initialized as they will be set before being used. The boundary is - ** determined by offsetof(Parse,aColCache). - **************************************************************************/ - - struct yColCache { - int iTable; /* Table cursor number */ - i16 iColumn; /* Table column number */ - u8 tempReg; /* iReg is a temp register that needs to be freed */ - int iLevel; /* Nesting level */ - int iReg; /* Reg with value of this column. 0 means none. */ - int lru; /* Least recently used entry has the smallest value */ - } aColCache[SQLITE_N_COLCACHE]; /* One for each column cache entry */ - int aTempReg[8]; /* Holding area for temporary registers */ - Token sNameToken; /* Token with unqualified schema object name */ - /************************************************************************ ** Above is constant between recursions. Below is reset before and after ** each recursion. The boundary between these two regions is determined - ** using offsetof(Parse,sLastToken) so the sLastToken field must be the - ** first field in the recursive region. + ** using offsetof(Parse,nVar) so the nVar field must be the first field + ** in the recursive region. ************************************************************************/ - Token sLastToken; /* The last token parsed */ - ynVar nVar; /* Number of '?' variables seen in the SQL so far */ + int nVar; /* Number of '?' variables seen in the SQL so far */ int nzVar; /* Number of available slots in azVar[] */ u8 iPkSortOrder; /* ASC or DESC for INTEGER PRIMARY KEY */ + u8 bFreeWith; /* True if pWith should be freed with parser */ u8 explain; /* True if the EXPLAIN flag is found on the query */ #ifndef SQLITE_OMIT_VIRTUALTABLE u8 declareVtab; /* True if inside sqlite3_declare_vtab() */ int nVtabLock; /* Number of virtual tables to lock */ #endif + int nAlias; /* Number of aliased result set columns */ int nHeight; /* Expression tree height of current sub-select */ #ifndef SQLITE_OMIT_EXPLAIN int iSelectId; /* ID of current select for EXPLAIN output */ @@ -15611,6 +12591,8 @@ struct Parse { Table *pNewTable; /* A table being constructed by CREATE TABLE */ Trigger *pNewTrigger; /* Trigger under construct by a CREATE TRIGGER */ const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */ + Token sNameToken; /* Token with unqualified schema object name */ + Token sLastToken; /* The last token parsed */ #ifndef SQLITE_OMIT_VIRTUALTABLE Token sArg; /* Complete text of a module argument */ Table **apVtabLock; /* Pointer to virtual tables needing locking */ @@ -15618,17 +12600,8 @@ struct Parse { Table *pZombieTab; /* List of Table objects to delete after code gen */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ With *pWith; /* Current WITH clause, or NULL */ - With *pWithToFree; /* Free this WITH object at the end of the parse */ }; -/* -** Sizes and pointers of various parts of the Parse object. -*/ -#define PARSE_HDR_SZ offsetof(Parse,aColCache) /* Recursive part w/o aColCache*/ -#define PARSE_RECURSE_SZ offsetof(Parse,sLastToken) /* Recursive part */ -#define PARSE_TAIL_SZ (sizeof(Parse)-PARSE_RECURSE_SZ) /* Non-recursive part */ -#define PARSE_TAIL(X) (((char*)(X))+PARSE_RECURSE_SZ) /* Pointer to tail */ - /* ** Return true if currently inside an sqlite3_declare_vtab() call. */ @@ -15649,42 +12622,26 @@ struct AuthContext { /* ** Bitfield flags for P5 value in various opcodes. -** -** Value constraints (enforced via assert()): -** OPFLAG_LENGTHARG == SQLITE_FUNC_LENGTH -** OPFLAG_TYPEOFARG == SQLITE_FUNC_TYPEOF -** OPFLAG_BULKCSR == BTREE_BULKLOAD -** OPFLAG_SEEKEQ == BTREE_SEEK_EQ -** OPFLAG_FORDELETE == BTREE_FORDELETE -** OPFLAG_SAVEPOSITION == BTREE_SAVEPOSITION -** OPFLAG_AUXDELETE == BTREE_AUXDELETE -*/ -#define OPFLAG_NCHANGE 0x01 /* OP_Insert: Set to update db->nChange */ - /* Also used in P2 (not P5) of OP_Delete */ +*/ +#define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ #define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK -#define OPFLAG_ISNOOP 0x40 /* OP_Delete does pre-update-hook only */ -#endif #define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ #define OPFLAG_SEEKEQ 0x02 /* OP_Open** cursor uses EQ seek only */ -#define OPFLAG_FORDELETE 0x08 /* OP_Open should use BTREE_FORDELETE */ -#define OPFLAG_P2ISREG 0x10 /* P2 to OP_Open** is a register number */ +#define OPFLAG_P2ISREG 0x04 /* P2 to OP_Open** is a register number */ #define OPFLAG_PERMUTE 0x01 /* OP_Compare: use the permutation */ -#define OPFLAG_SAVEPOSITION 0x02 /* OP_Delete: keep cursor position */ -#define OPFLAG_AUXDELETE 0x04 /* OP_Delete: index in a DELETE op */ /* * Each trigger present in the database schema is stored as an instance of - * struct Trigger. + * struct Trigger. * * Pointers to instances of struct Trigger are stored in two ways. - * 1. In the "trigHash" hash table (part of the sqlite3* that represents the + * 1. In the "trigHash" hash table (part of the sqlite3* that represents the * database). This allows Trigger structures to be retrieved by name. * 2. All triggers associated with a single table form a linked list, using the * pNext member of struct Trigger. A pointer to the first element of the @@ -15710,7 +12667,7 @@ struct Trigger { /* ** A trigger is either a BEFORE or an AFTER trigger. The following constants -** determine which. +** determine which. ** ** If there are multiple triggers, you might of some BEFORE and some AFTER. ** In that cases, the constants below can be ORed together. @@ -15720,15 +12677,15 @@ struct Trigger { /* * An instance of struct TriggerStep is used to store a single SQL statement - * that is a part of a trigger-program. + * that is a part of a trigger-program. * * Instances of struct TriggerStep are stored in a singly linked list (linked - * using the "pNext" member) referenced by the "step_list" member of the + * using the "pNext" member) referenced by the "step_list" member of the * associated struct Trigger instance. The first element of the linked list is * the first step of the trigger-program. - * + * * The "op" member indicates whether this is a "DELETE", "INSERT", "UPDATE" or - * "SELECT" statement. The meanings of the other members is determined by the + * "SELECT" statement. The meanings of the other members is determined by the * value of "op" as follows: * * (op == TK_INSERT) @@ -15738,7 +12695,7 @@ struct Trigger { * zTarget -> Dequoted name of the table to insert into. * pExprList -> If this is an INSERT INTO ... VALUES ... statement, then * this stores values to be inserted. Otherwise NULL. - * pIdList -> If this is an INSERT INTO ... () VALUES ... + * pIdList -> If this is an INSERT INTO ... () VALUES ... * statement, then this stores the column-names to be * inserted into. * @@ -15746,7 +12703,7 @@ struct Trigger { * zTarget -> Dequoted name of the table to delete from. * pWhere -> The WHERE clause of the DELETE statement if one is specified. * Otherwise NULL. - * + * * (op == TK_UPDATE) * zTarget -> Dequoted name of the table to update. * pWhere -> The WHERE clause of the UPDATE statement if one is specified. @@ -15754,7 +12711,7 @@ struct Trigger { * pExprList -> A list of the columns to update and the expressions to update * them to. See sqlite3Update() documentation of "pChanges" * argument. - * + * */ struct TriggerStep { u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT */ @@ -15772,7 +12729,7 @@ struct TriggerStep { /* ** The following structure contains information used by the sqliteFix... ** routines as they walk the parse tree to make database references -** explicit. +** explicit. */ typedef struct DbFixer DbFixer; struct DbFixer { @@ -15792,20 +12749,13 @@ struct StrAccum { sqlite3 *db; /* Optional database for lookaside. Can be NULL */ char *zBase; /* A base allocation. Not from malloc. */ char *zText; /* The string collected so far */ - u32 nChar; /* Length of the string so far */ - u32 nAlloc; /* Amount of space allocated in zText */ - u32 mxAlloc; /* Maximum allowed allocation. 0 for no malloc usage */ + int nChar; /* Length of the string so far */ + int nAlloc; /* Amount of space allocated in zText */ + int mxAlloc; /* Maximum allowed allocation. 0 for no malloc usage */ u8 accError; /* STRACCUM_NOMEM or STRACCUM_TOOBIG */ - u8 printfFlags; /* SQLITE_PRINTF flags below */ }; #define STRACCUM_NOMEM 1 #define STRACCUM_TOOBIG 2 -#define SQLITE_PRINTF_INTERNAL 0x01 /* Internal-use-only converters allowed */ -#define SQLITE_PRINTF_SQLFUNC 0x02 /* SQL function arguments to VXPrintf */ -#define SQLITE_PRINTF_MALLOCED 0x04 /* True if xText is allocated space */ - -#define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) - /* ** A pointer to this structure is used to communicate information @@ -15833,7 +12783,6 @@ struct Sqlite3Config { int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ int nLookaside; /* Default lookaside buffer count */ - int nStmtSpill; /* Stmt-journal spill-to-disk threshold */ sqlite3_mem_methods m; /* Low-level memory allocation interface */ sqlite3_mutex_methods mutex; /* Low-level mutex interface */ sqlite3_pcache_methods2 pcache2; /* Low-level page-cache interface */ @@ -15877,7 +12826,6 @@ struct Sqlite3Config { int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ #endif int bLocaltimeFault; /* True to fail localtime() calls */ - int iOnceResetThreshold; /* When to reset OP_Once counters */ }; /* @@ -15902,10 +12850,10 @@ struct Sqlite3Config { ** Context pointer passed down through the tree-walk. */ struct Walker { - Parse *pParse; /* Parser context. */ int (*xExprCallback)(Walker*, Expr*); /* Callback for expressions */ int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ + Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ u8 eCode; /* A small processing code */ union { /* Extra data for callback */ @@ -15914,9 +12862,6 @@ struct Walker { int iCur; /* A cursor number */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ - struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ - int *aiCol; /* array of column indexes */ - struct IdxCover *pIdxCover; /* Check for index coverage */ } u; }; @@ -15926,7 +12871,6 @@ SQLITE_PRIVATE int sqlite3WalkExprList(Walker*, ExprList*); SQLITE_PRIVATE int sqlite3WalkSelect(Walker*, Select*); SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker*, Select*); SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker*, Select*); -SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker*, Expr*); /* ** Return code from the parse-tree walking primitives and their @@ -15947,7 +12891,7 @@ struct With { char *zName; /* Name of this CTE */ ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The definition of this CTE */ - const char *zCteErr; /* Error message for circular references */ + const char *zErr; /* Error message for circular references */ } a[1]; }; @@ -15985,23 +12929,7 @@ SQLITE_PRIVATE int sqlite3CantopenError(int); #define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__) #define SQLITE_MISUSE_BKPT sqlite3MisuseError(__LINE__) #define SQLITE_CANTOPEN_BKPT sqlite3CantopenError(__LINE__) -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3NomemError(int); -SQLITE_PRIVATE int sqlite3IoerrnomemError(int); -# define SQLITE_NOMEM_BKPT sqlite3NomemError(__LINE__) -# define SQLITE_IOERR_NOMEM_BKPT sqlite3IoerrnomemError(__LINE__) -#else -# define SQLITE_NOMEM_BKPT SQLITE_NOMEM -# define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM -#endif -/* -** FTS3 and FTS4 both require virtual table support -*/ -#if defined(SQLITE_OMIT_VIRTUALTABLE) -# undef SQLITE_ENABLE_FTS3 -# undef SQLITE_ENABLE_FTS4 -#endif /* ** FTS4 is really an extension for FTS3. It is enabled using the @@ -16034,7 +12962,6 @@ SQLITE_PRIVATE int sqlite3IoerrnomemError(int); # define sqlite3Isdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x04) # define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) # define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) -# define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) #else # define sqlite3Toupper(x) toupper((unsigned char)(x)) # define sqlite3Isspace(x) isspace((unsigned char)(x)) @@ -16043,18 +12970,14 @@ SQLITE_PRIVATE int sqlite3IoerrnomemError(int); # define sqlite3Isdigit(x) isdigit((unsigned char)(x)) # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) -# define sqlite3Isquote(x) ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`') #endif -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_PRIVATE int sqlite3IsIdChar(u8); -#endif /* ** Internal function prototypes */ -SQLITE_PRIVATE int sqlite3StrICmp(const char*,const char*); +#define sqlite3StrICmp sqlite3_stricmp SQLITE_PRIVATE int sqlite3Strlen30(const char*); -SQLITE_PRIVATE char *sqlite3ColumnType(Column*,char*); #define sqlite3StrNICmp sqlite3_strnicmp SQLITE_PRIVATE int sqlite3MallocInit(void); @@ -16063,7 +12986,6 @@ SQLITE_PRIVATE void *sqlite3Malloc(u64); SQLITE_PRIVATE void *sqlite3MallocZero(u64); SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3*, u64); SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3*, u64); -SQLITE_PRIVATE void *sqlite3DbMallocRawNN(sqlite3*, u64); SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3*,const char*); SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3*,const char*, u64); SQLITE_PRIVATE void *sqlite3Realloc(void*, u64); @@ -16077,9 +12999,7 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void*); SQLITE_PRIVATE void *sqlite3PageMalloc(int); SQLITE_PRIVATE void sqlite3PageFree(void*); SQLITE_PRIVATE void sqlite3MemSetDefault(void); -#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); -#endif SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); /* @@ -16093,23 +13013,19 @@ SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); #ifdef SQLITE_USE_ALLOCA # define sqlite3StackAllocRaw(D,N) alloca(N) # define sqlite3StackAllocZero(D,N) memset(alloca(N), 0, N) -# define sqlite3StackFree(D,P) +# define sqlite3StackFree(D,P) #else # define sqlite3StackAllocRaw(D,N) sqlite3DbMallocRaw(D,N) # define sqlite3StackAllocZero(D,N) sqlite3DbMallocZero(D,N) # define sqlite3StackFree(D,P) sqlite3DbFree(D,P) #endif -/* Do not allow both MEMSYS5 and MEMSYS3 to be defined together. If they -** are, disable MEMSYS3 -*/ -#ifdef SQLITE_ENABLE_MEMSYS5 -SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void); -#undef SQLITE_ENABLE_MEMSYS3 -#endif #ifdef SQLITE_ENABLE_MEMSYS3 SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void); #endif +#ifdef SQLITE_ENABLE_MEMSYS5 +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void); +#endif #ifndef SQLITE_MUTEX_OMIT @@ -16119,16 +13035,11 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int); SQLITE_PRIVATE int sqlite3MutexInit(void); SQLITE_PRIVATE int sqlite3MutexEnd(void); #endif -#if !defined(SQLITE_MUTEX_OMIT) && !defined(SQLITE_MUTEX_NOOP) -SQLITE_PRIVATE void sqlite3MemoryBarrier(void); -#else -# define sqlite3MemoryBarrier() -#endif SQLITE_PRIVATE sqlite3_int64 sqlite3StatusValue(int); SQLITE_PRIVATE void sqlite3StatusUp(int, int); SQLITE_PRIVATE void sqlite3StatusDown(int, int); -SQLITE_PRIVATE void sqlite3StatusHighwater(int, int); +SQLITE_PRIVATE void sqlite3StatusSet(int, int); /* Access to mutexes used by sqlite3_status() */ SQLITE_PRIVATE sqlite3_mutex *sqlite3Pcache1Mutex(void); @@ -16150,10 +13061,13 @@ struct PrintfArguments { sqlite3_value **apArg; /* The argument values */ }; -SQLITE_PRIVATE void sqlite3VXPrintf(StrAccum*, const char*, va_list); -SQLITE_PRIVATE void sqlite3XPrintf(StrAccum*, const char*, ...); +#define SQLITE_PRINTF_INTERNAL 0x01 +#define SQLITE_PRINTF_SQLFUNC 0x02 +SQLITE_PRIVATE void sqlite3VXPrintf(StrAccum*, u32, const char*, va_list); +SQLITE_PRIVATE void sqlite3XPrintf(StrAccum*, u32, const char*, ...); SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3*,const char*, ...); SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3*,const char*, va_list); +SQLITE_PRIVATE char *sqlite3MAppendf(sqlite3*,char*,const char*,...); #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) SQLITE_PRIVATE void sqlite3DebugPrintf(const char*, ...); #endif @@ -16162,18 +13076,19 @@ SQLITE_PRIVATE void *sqlite3TestTextToPtr(const char*); #endif #if defined(SQLITE_DEBUG) +SQLITE_PRIVATE TreeView *sqlite3TreeViewPush(TreeView*,u8); +SQLITE_PRIVATE void sqlite3TreeViewPop(TreeView*); +SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView*, const char*, ...); +SQLITE_PRIVATE void sqlite3TreeViewItem(TreeView*, const char*, u8); SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); -SQLITE_PRIVATE void sqlite3TreeViewBareExprList(TreeView*, const ExprList*, const char*); SQLITE_PRIVATE void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); -SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView*, const With*, u8); #endif -SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); +SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*, ...); SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); -SQLITE_PRIVATE void sqlite3Dequote(char*); -SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); +SQLITE_PRIVATE int sqlite3Dequote(char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*, char **); SQLITE_PRIVATE void sqlite3FinishCoding(Parse*); @@ -16182,21 +13097,15 @@ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse*,int); SQLITE_PRIVATE int sqlite3GetTempRange(Parse*,int); SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse*,int,int); SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse*); -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse*,int,int); -#endif SQLITE_PRIVATE Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); SQLITE_PRIVATE Expr *sqlite3Expr(sqlite3*,int,const char*); SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*, const Token*); -SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); -SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); +SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); -SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); -SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int); SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*); SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); @@ -16207,30 +13116,25 @@ SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int); SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3*); SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3*,int); SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3*); +SQLITE_PRIVATE void sqlite3BeginParse(Parse*,int); SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); -SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); -SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); -SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*); SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*); SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *, int); SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index*, i16); SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int); -#if SQLITE_ENABLE_HIDDEN_COLUMNS -SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table*, Column*); -#else -# define sqlite3ColumnPropertiesFromName(T,C) /* no-op */ -#endif -SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token*,Token*); +SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token*); SQLITE_PRIVATE void sqlite3AddNotNull(Parse*, int); SQLITE_PRIVATE void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int); SQLITE_PRIVATE void sqlite3AddCheckConstraint(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3AddColumnType(Parse*,Token*); SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse*,ExprSpan*); SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*); SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*); SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, sqlite3_vfs**,char**,char **); SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3*,const char*); +SQLITE_PRIVATE int sqlite3CodeOnce(Parse *); #ifdef SQLITE_OMIT_BUILTIN_TEST # define sqlite3FaultSim(X) SQLITE_OK @@ -16240,14 +13144,11 @@ SQLITE_PRIVATE int sqlite3FaultSim(int); SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32); SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec*, u32); -SQLITE_PRIVATE int sqlite3BitvecTestNotNull(Bitvec*, u32); SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec*, u32); SQLITE_PRIVATE void sqlite3BitvecClear(Bitvec*, u32, void*); SQLITE_PRIVATE void sqlite3BitvecDestroy(Bitvec*); SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec*); -#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int,int*); -#endif SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*, void*, unsigned int); SQLITE_PRIVATE void sqlite3RowSetClear(RowSet*); @@ -16255,7 +13156,7 @@ SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet*, i64); SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, int iBatch, i64); SQLITE_PRIVATE int sqlite3RowSetNext(RowSet*, i64*); -SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int); +SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse*,Table*); @@ -16285,19 +13186,18 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*) SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, Token*, Select*, Expr*, IdList*); SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); -SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse*, SrcList*, ExprList*); SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *, struct SrcList_item *); SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList*); SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse*, SrcList*); SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3*, IdList*); SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3*, SrcList*); SQLITE_PRIVATE Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**); -SQLITE_PRIVATE void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, - Expr*, int, int, u8); +SQLITE_PRIVATE Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, + Expr*, int, int); SQLITE_PRIVATE void sqlite3DropIndex(Parse*, SrcList*, int); SQLITE_PRIVATE int sqlite3Select(Parse*, Select*, SelectDest*); SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, - Expr*,ExprList*,u32,Expr*,Expr*); + Expr*,ExprList*,u16,Expr*,Expr*); SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int); @@ -16309,20 +13209,14 @@ SQLITE_PRIVATE void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); SQLITE_PRIVATE void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*); -SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo*); +SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo*); -SQLITE_PRIVATE int sqlite3WhereOrderedInnerLoop(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo*, int*); -#define ONEPASS_OFF 0 /* Use of ONEPASS not allowed */ -#define ONEPASS_SINGLE 1 /* ONEPASS valid for a single row update */ -#define ONEPASS_MULTI 2 /* ONEPASS is valid for multiple rows */ -SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); -SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg(Parse*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse*, int, int, int); @@ -16332,42 +13226,34 @@ SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse*); SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCode(Parse*, Expr*, int); -SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeAtInit(Parse*, Expr*, int, u8); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse*, Expr*, int); -SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); +SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, u8); #define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ #define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ -#define SQLITE_ECEL_REF 0x04 /* Use ExprList.u.x.iOrderByCol */ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse*, Expr*, int, int); SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse*, Expr*, int, int); -SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int); SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3*,const char*, const char*); -#define LOCATE_VIEW 0x01 -#define LOCATE_NOERR 0x02 -SQLITE_PRIVATE Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*); -SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *); +SQLITE_PRIVATE Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*); +SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *); SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3*,const char*, const char*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); -SQLITE_PRIVATE void sqlite3Vacuum(Parse*,Token*); -SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*, int); +SQLITE_PRIVATE void sqlite3Vacuum(Parse*); +SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*); SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, Token*); SQLITE_PRIVATE int sqlite3ExprCompare(Expr*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList*, ExprList*, int); SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Expr*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); -SQLITE_PRIVATE int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx); SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse*); -#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE void sqlite3PrngSaveState(void); SQLITE_PRIVATE void sqlite3PrngRestoreState(void); -#endif SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3*,int); SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse*, int); SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); @@ -16381,22 +13267,18 @@ SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); -#ifdef SQLITE_ENABLE_CURSOR_HINTS -SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); -#endif SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); SQLITE_PRIVATE int sqlite3IsRowid(const char*); -SQLITE_PRIVATE void sqlite3GenerateRowDelete( - Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int); -SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); +SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8); +SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*); SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse*,int); SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, - u8,u8,int,int*,int*); + u8,u8,int,int*); SQLITE_PRIVATE void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); -SQLITE_PRIVATE int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*); +SQLITE_PRIVATE int sqlite3OpenTableAndIndices(Parse*, Table*, int, int, u8*, int*, int*); SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse*, int, int); SQLITE_PRIVATE void sqlite3MultiWrite(Parse*); SQLITE_PRIVATE void sqlite3MayAbort(Parse*); @@ -16413,11 +13295,11 @@ SQLITE_PRIVATE void sqlite3SelectSetName(Select*,const char*); #else # define sqlite3SelectSetName(A,B) #endif -SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs(FuncDef*,int); -SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); -SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void); +SQLITE_PRIVATE void sqlite3FuncDefInsert(FuncDefHash*, FuncDef*); +SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,u8); +SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3*); SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void); -SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*); +SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void); SQLITE_PRIVATE int sqlite3SafetyCheckOk(sqlite3*); SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3*); SQLITE_PRIVATE void sqlite3ChangeCookie(Parse*, int); @@ -16448,7 +13330,6 @@ SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) -# define sqlite3IsToplevel(p) ((p)->pToplevel==0) #else # define sqlite3TriggersExist(B,C,D,E,F) 0 # define sqlite3DeleteTrigger(A,B) @@ -16458,7 +13339,6 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Tab # define sqlite3CodeRowTriggerDirect(A,B,C,D,E,F) # define sqlite3TriggerList(X, Y) 0 # define sqlite3ParseToplevel(p) p -# define sqlite3IsToplevel(p) 1 # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 #endif @@ -16496,11 +13376,7 @@ SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst,LogEst); #ifndef SQLITE_OMIT_VIRTUALTABLE SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double); #endif -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst); -#endif /* ** Routines to read and write variable-length integers. These used to @@ -16526,17 +13402,15 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); #define putVarint sqlite3PutVarint -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *, Index *); SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); -SQLITE_PRIVATE char sqlite3TableColumnAffinity(Table*,int); SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char*, i64*); SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); SQLITE_PRIVATE void sqlite3Error(sqlite3*,int); -SQLITE_PRIVATE void sqlite3SystemError(sqlite3*,int); SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); @@ -16569,7 +13443,7 @@ SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z,u8); SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value*, u8); -SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, +SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value*); SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); @@ -16579,12 +13453,11 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8); #ifndef SQLITE_AMALGAMATION SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; -SQLITE_PRIVATE const char sqlite3StrBINARY[]; SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[]; SQLITE_PRIVATE const Token sqlite3IntTokens[]; SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config; -SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; +SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; #ifndef SQLITE_OMIT_WSD SQLITE_PRIVATE int sqlite3PendingByte; #endif @@ -16596,12 +13469,10 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *, int *); SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*); -SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr *, int, int); +SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *, Expr *, int, int); SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); -SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); SQLITE_PRIVATE int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); SQLITE_PRIVATE int sqlite3ResolveExprNames(NameContext*, Expr*); -SQLITE_PRIVATE int sqlite3ResolveExprListNames(NameContext*, ExprList*); SQLITE_PRIVATE void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); SQLITE_PRIVATE void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); @@ -16619,6 +13490,7 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3*,Index*); SQLITE_PRIVATE void sqlite3DefaultRowEst(Index*); SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3*, int); SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3*,Expr*,int*,char*); +SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse*, int, int); SQLITE_PRIVATE void sqlite3SchemaClear(void *); SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *, Btree *); SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *); @@ -16629,13 +13501,11 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse*, Index*); #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo*); #endif -SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, +SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*), FuncDestructor *pDestructor ); -SQLITE_PRIVATE void sqlite3OomFault(sqlite3*); -SQLITE_PRIVATE void sqlite3OomClear(sqlite3*); SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); @@ -16651,20 +13521,12 @@ SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *); SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); -#ifndef SQLITE_OMIT_SUBQUERY -SQLITE_PRIVATE int sqlite3ExprCheckIN(Parse*, Expr*); -#else -# define sqlite3ExprCheckIN(x,y) SQLITE_OK -#endif - #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void); -SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue( - Parse*,Index*,UnpackedRecord**,Expr*,int,int,int*); +SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*); SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**); SQLITE_PRIVATE void sqlite3Stat4ProbeFree(UnpackedRecord*); SQLITE_PRIVATE int sqlite3Stat4Column(sqlite3*, const void*, int, int, sqlite3_value**); -SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3*, Index*, int); #endif /* @@ -16700,7 +13562,7 @@ SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); # define sqlite3VtabRollback(X) # define sqlite3VtabCommit(X) # define sqlite3VtabInSync(db) 0 -# define sqlite3VtabLock(X) +# define sqlite3VtabLock(X) # define sqlite3VtabUnlock(X) # define sqlite3VtabUnlockList(X) # define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK @@ -16719,8 +13581,6 @@ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*); SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3*, Table*); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif -SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse*,Module*); -SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3*,Module*); SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); SQLITE_PRIVATE void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int); SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse*, Token*); @@ -16758,7 +13618,7 @@ SQLITE_PRIVATE void sqlite3WithPush(Parse*, With*, u8); ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign ** key functionality is available. If OMIT_TRIGGER is defined but ** OMIT_FOREIGN_KEY is not, only some of the functions are no-oped. In -** this case foreign keys are parsed, but no other functionality is +** this case foreign keys are parsed, but no other functionality is ** provided (enforcement of FK constraints requires the triggers sub-system). */ #if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) @@ -16817,16 +13677,21 @@ SQLITE_PRIVATE void sqlite3EndBenignMalloc(void); #define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ #define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ #define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ -SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*, int*); +SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*); -SQLITE_PRIVATE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); -SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *); #ifdef SQLITE_ENABLE_ATOMIC_WRITE +SQLITE_PRIVATE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); +SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *); SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *); +SQLITE_PRIVATE int sqlite3JournalExists(sqlite3_file *p); +#else + #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile) + #define sqlite3JournalExists(p) 1 #endif -SQLITE_PRIVATE int sqlite3JournalIsInMemory(sqlite3_file *p); SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *); +SQLITE_PRIVATE int sqlite3MemJournalSize(void); +SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *); SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p); #if SQLITE_MAX_EXPR_DEPTH>0 @@ -16857,7 +13722,7 @@ SQLITE_PRIVATE void sqlite3ParserTrace(FILE*, char *); /* ** If the SQLITE_ENABLE IOTRACE exists then the global variable ** sqlite3IoTrace is a pointer to a printf-like routine used to -** print I/O tracing messages. +** print I/O tracing messages. */ #ifdef SQLITE_ENABLE_IOTRACE # define IOTRACE(A) if( sqlite3IoTrace ){ sqlite3IoTrace A; } @@ -16891,7 +13756,7 @@ SQLITE_API SQLITE_EXTERN void (SQLITE_CDECL *sqlite3IoTrace)(const char*,...); ** that allocations that might have been satisfied by lookaside are not ** passed back to non-lookaside free() routines. Asserts such as the ** example above are placed on the non-lookaside free() routines to verify -** this constraint. +** this constraint. ** ** All of this is no-op for a production build. It only comes into ** play when the SQLITE_MEMDEBUG compile-time option is used. @@ -16918,16 +13783,7 @@ SQLITE_PRIVATE int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread*, void**); #endif -#if defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST) -SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3*); -#endif - -SQLITE_PRIVATE int sqlite3ExprVectorSize(Expr *pExpr); -SQLITE_PRIVATE int sqlite3ExprIsVector(Expr *pExpr); -SQLITE_PRIVATE Expr *sqlite3VectorFieldSubexpr(Expr*, int); -SQLITE_PRIVATE Expr *sqlite3ExprForVectorField(Parse*,Expr*,int); - -#endif /* SQLITEINT_H */ +#endif /* _SQLITEINT_H_ */ /************** End of sqliteInt.h *******************************************/ /************** Begin file crypto.c ******************************************/ @@ -16965,7 +13821,6 @@ SQLITE_PRIVATE Expr *sqlite3ExprForVectorField(Parse*,Expr*,int); #ifdef SQLITE_HAS_CODEC /* #include */ -/* #include "sqliteInt.h" */ /************** Include btreeInt.h in the middle of crypto.c *****************/ /************** Begin file btreeInt.h ****************************************/ /* @@ -17183,7 +14038,6 @@ SQLITE_PRIVATE Expr *sqlite3ExprForVectorField(Parse*,Expr*,int); ** 4 Number of leaf pointers on this page ** * zero or more pages numbers of leaves */ -/* #include "sqliteInt.h" */ /* The following value is the maximum cell size assuming a maximum page @@ -17201,7 +14055,6 @@ SQLITE_PRIVATE Expr *sqlite3ExprForVectorField(Parse*,Expr*,int); /* Forward declarations */ typedef struct MemPage MemPage; typedef struct BtLock BtLock; -typedef struct CellInfo CellInfo; /* ** This is a magic string that appears at the beginning of every @@ -17246,6 +14099,7 @@ struct MemPage { u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ u8 intKey; /* True if table b-trees. False for index b-trees */ u8 intKeyLeaf; /* True if the leaf of an intKey table */ + u8 noPayload; /* True if internal intKey page (thus w/o data) */ u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ @@ -17264,10 +14118,7 @@ struct MemPage { u8 *aData; /* Pointer to disk image of the page data */ u8 *aDataEnd; /* One byte past the end of usable data */ u8 *aCellIdx; /* The cell index area */ - u8 *aDataOfst; /* Same as aData for leaves. aData+4 for interior */ DbPage *pDbPage; /* Pager page handle */ - u16 (*xCellSize)(MemPage*,u8*); /* cellSizePtr method */ - void (*xParseCell)(MemPage*,u8*,CellInfo*); /* btreeParseCell method */ Pgno pgno; /* Page number for this page */ }; @@ -17323,7 +14174,6 @@ struct Btree { u8 inTrans; /* TRANS_NONE, TRANS_READ or TRANS_WRITE */ u8 sharable; /* True if we can share pBt with another db */ u8 locked; /* True if db currently has pBt locked */ - u8 hasIncrblobCur; /* True if there are one or more Incrblob cursors */ int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ int nBackup; /* Number of backup operations reading this btree */ u32 iDataVersion; /* Combines with pBt->pPager->iDataVersion */ @@ -17434,11 +14284,13 @@ struct BtShared { ** about a cell. The parseCellPtr() function fills in this structure ** based on information extract from the raw disk page. */ +typedef struct CellInfo CellInfo; struct CellInfo { i64 nKey; /* The key for INTKEY tables, or nPayload otherwise */ u8 *pPayload; /* Pointer to the start of payload */ u32 nPayload; /* Bytes of payload */ u16 nLocal; /* Amount of payload held locally, not on overflow */ + u16 iOverflow; /* Offset to overflow page number. Zero if no overflow */ u16 nSize; /* Size of the cell content on the main b-tree page */ }; @@ -17475,7 +14327,8 @@ struct CellInfo { struct BtCursor { Btree *pBtree; /* The Btree to which this cursor belongs */ BtShared *pBt; /* The BtShared this cursor points to */ - BtCursor *pNext; /* Forms a linked list of all cursors */ + BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ + struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ Pgno *aOverflow; /* Cache of overflow page locations */ CellInfo info; /* A parse of the cell we are pointing at */ i64 nKey; /* Size of pKey, or last integer key */ @@ -17485,16 +14338,9 @@ struct BtCursor { int skipNext; /* Prev() is noop if negative. Next() is noop if positive. ** Error code if eState==CURSOR_FAULT */ u8 curFlags; /* zero or more BTCF_* flags defined below */ - u8 curPagerFlags; /* Flags to send to sqlite3PagerGet() */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ - u8 hints; /* As configured by CursorSetHints() */ - /* All fields above are zeroed when the cursor is allocated. See - ** sqlite3BtreeCursorZero(). Fields that follow must be manually - ** initialized. */ - i8 iPage; /* Index of current page in apPage */ - u8 curIntKey; /* Value of apPage[0]->intKey */ - struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ - void *padding1; /* Make object size a multiple of 16 */ + u8 hints; /* As configured by CursorSetHints() */ + i16 iPage; /* Index of current page in apPage */ u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ }; @@ -17507,7 +14353,6 @@ struct BtCursor { #define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ #define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ #define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ -#define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ /* ** Potential values for BtCursor.eState. @@ -17650,7 +14495,6 @@ struct IntegrityCk { const char *zPfx; /* Error message prefix */ int v1, v2; /* Values for up to two %d fields in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ - u32 *heap; /* Min-heap used for analyzing cell coverage */ }; /* @@ -17661,23 +14505,6 @@ struct IntegrityCk { #define get4byte sqlite3Get4byte #define put4byte sqlite3Put4byte -/* -** get2byteAligned(), unlike get2byte(), requires that its argument point to a -** two-byte aligned address. get2bytea() is only used for accessing the -** cell addresses in a btree header. -*/ -#if SQLITE_BYTEORDER==4321 -# define get2byteAligned(x) (*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && GCC_VERSION>=4008000 -# define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 -# define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) -#else -# define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) -#endif - /************** End of btreeInt.h ********************************************/ /************** Continuing where we left off in crypto.c *********************/ /************** Include crypto.h in the middle of crypto.c *******************/ @@ -17729,9 +14556,9 @@ struct IntegrityCk { #ifndef CIPHER_VERSION #ifdef SQLCIPHER_FIPS -#define CIPHER_VERSION "3.4.1 FIPS" +#define CIPHER_VERSION "3.3.1 FIPS" #else -#define CIPHER_VERSION "3.4.1" +#define CIPHER_VERSION "3.3.1" #endif #endif @@ -17808,17 +14635,17 @@ struct IntegrityCk { #endif /* extensions defined in pager.c */ -SQLITE_API void sqlite3pager_get_codec(Pager *pPager, void **ctx); -SQLITE_API int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno); +SQLITE_PRIVATE void sqlite3pager_get_codec(Pager *pPager, void **ctx); +SQLITE_PRIVATE int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno); SQLITE_PRIVATE sqlite3_file *sqlite3Pager_get_fd(Pager *pPager); -SQLITE_API void sqlite3pager_sqlite3PagerSetCodec( +SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetCodec( Pager *pPager, void *(*xCodec)(void*,void*,Pgno,int), void (*xCodecSizeChng)(void*,int,int), void (*xCodecFree)(void*), void *pCodec ); -SQLITE_API void sqlite3pager_sqlite3PagerSetError(Pager *pPager, int error); +SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetError(Pager *pPager, int error); /* end extensions defined in pager.c */ /* @@ -17844,19 +14671,6 @@ static void cipher_bin2hex(const unsigned char* in, int sz, char *out) { } } -static int cipher_isHex(const unsigned char *hex, int sz){ - int i; - for(i = 0; i < sz; i++) { - unsigned char c = hex[i]; - if ((c < '0' || c > '9') && - (c < 'A' || c > 'F') && - (c < 'a' || c > 'f')) { - return 0; - } - } - return 1; -} - /* extensions defined in crypto_impl.c */ typedef struct codec_ctx codec_ctx; @@ -17924,7 +14738,7 @@ static int sqlcipher_codec_get_store_pass(codec_ctx *ctx); static void sqlcipher_codec_get_pass(codec_ctx *ctx, void **zKey, int *nKey); static void sqlcipher_codec_set_store_pass(codec_ctx *ctx, int value); int sqlcipher_codec_fips_status(codec_ctx *ctx); -const char* sqlcipher_codec_get_provider_version(codec_ctx *ctx); + #endif #endif /* END SQLCIPHER */ @@ -17975,7 +14789,6 @@ static int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, } int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLeft, const char *zRight) { - char *pragma_cipher_deprecated_msg = "PRAGMA cipher command is deprecated, please remove from usage."; struct Db *pDb = &db->aDb[iDb]; codec_ctx *ctx = NULL; int rc; @@ -18029,21 +14842,13 @@ int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLef sqlcipher_codec_get_cipher_provider(ctx)); } } else - if( sqlite3StrICmp(zLeft, "cipher_provider_version")==0 && !zRight){ - if(ctx) { codec_vdbe_return_static_string(pParse, "cipher_provider_version", - sqlcipher_codec_get_provider_version(ctx)); - } - } else if( sqlite3StrICmp(zLeft, "cipher_version")==0 && !zRight ){ codec_vdbe_return_static_string(pParse, "cipher_version", codec_get_cipher_version()); }else if( sqlite3StrICmp(zLeft, "cipher")==0 ){ if(ctx) { if( zRight ) { - rc = sqlcipher_codec_ctx_set_cipher(ctx, zRight, 2); // change cipher for both - codec_vdbe_return_static_string(pParse, "cipher", pragma_cipher_deprecated_msg); - sqlite3_log(SQLITE_WARNING, pragma_cipher_deprecated_msg); - return rc; + sqlcipher_codec_ctx_set_cipher(ctx, zRight, 2); // change cipher for both }else { codec_vdbe_return_static_string(pParse, "cipher", sqlcipher_codec_ctx_get_cipher(ctx, 2)); @@ -18261,11 +15066,7 @@ SQLITE_PRIVATE int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, in /* point the internal codec argument against the contet to be prepared */ rc = sqlcipher_codec_ctx_init(&ctx, pDb, pDb->pBt->pBt->pPager, fd, zKey, nKey); - if(rc != SQLITE_OK) { - /* initialization failed, do not attach potentially corrupted context */ - sqlite3_mutex_leave(db->mutex); - return rc; - } + if(rc != SQLITE_OK) return rc; /* initialization failed, do not attach potentially corrupted context */ sqlite3pager_sqlite3PagerSetCodec(sqlite3BtreePager(pDb->pBt), sqlite3Codec, NULL, sqlite3FreeCodecArg, (void *) ctx); @@ -18287,7 +15088,7 @@ SQLITE_PRIVATE int sqlite3CodecAttach(sqlite3* db, int nDb, const void *zKey, in return SQLITE_OK; } -SQLITE_API void sqlite3_activate_see(const char* in) { +SQLITE_API void SQLITE_STDCALL sqlite3_activate_see(const char* in) { /* do nothing, security enhancements are always active */ } @@ -18298,19 +15099,19 @@ static int sqlcipher_find_db_index(sqlite3 *db, const char *zDb) { } for(db_index = 0; db_index < db->nDb; db_index++) { struct Db *pDb = &db->aDb[db_index]; - if(strcmp(pDb->zDbSName, zDb) == 0) { + if(strcmp(pDb->zName, zDb) == 0) { return db_index; } } return 0; } -SQLITE_API int sqlite3_key(sqlite3 *db, const void *pKey, int nKey) { +SQLITE_API int SQLITE_STDCALL sqlite3_key(sqlite3 *db, const void *pKey, int nKey) { CODEC_TRACE(("sqlite3_key entered: db=%p pKey=%s nKey=%d\n", db, (char *)pKey, nKey)); return sqlite3_key_v2(db, "main", pKey, nKey); } -SQLITE_API int sqlite3_key_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { +SQLITE_API int SQLITE_STDCALL sqlite3_key_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { CODEC_TRACE(("sqlite3_key_v2: entered db=%p zDb=%s pKey=%s nKey=%d\n", db, zDb, (char *)pKey, nKey)); /* attach key if db and pKey are not null and nKey is > 0 */ if(db && pKey && nKey) { @@ -18320,7 +15121,7 @@ SQLITE_API int sqlite3_key_v2(sqlite3 *db, const char *zDb, const void *pKey, in return SQLITE_ERROR; } -SQLITE_API int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { +SQLITE_API int SQLITE_STDCALL sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { CODEC_TRACE(("sqlite3_rekey entered: db=%p pKey=%s nKey=%d\n", db, (char *)pKey, nKey)); return sqlite3_rekey_v2(db, "main", pKey, nKey); } @@ -18335,7 +15136,7 @@ SQLITE_API int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { ** 2. If there is NOT already a key present do nothing ** 3. If there is a key present, re-encrypt the database with the new key */ -SQLITE_API int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { +SQLITE_API int SQLITE_STDCALL sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { CODEC_TRACE(("sqlite3_rekey_v2: entered db=%p zDb=%s pKey=%s, nKey=%d\n", db, zDb, (char *)pKey, nKey)); if(db && pKey && nKey) { int db_index = sqlcipher_find_db_index(db, zDb); @@ -18370,7 +15171,7 @@ SQLITE_API int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, sqlite3PagerPagecount(pPager, &page_count); for(pgno = 1; rc == SQLITE_OK && pgno <= (unsigned int)page_count; pgno++) { /* pgno's start at 1 see pager.c:pagerAcquire */ if(!sqlite3pager_is_mj_pgno(pPager, pgno)) { /* skip this page (see pager.c:pagerAcquire for reasoning) */ - rc = sqlite3PagerGet(pPager, pgno, &page, 0); + rc = sqlite3PagerGet(pPager, pgno, &page); if(rc == SQLITE_OK) { /* write page see pager_incr_changecounter for example */ rc = sqlite3PagerWrite(page); if(rc == SQLITE_OK) { @@ -18506,7 +15307,7 @@ void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_value **ar int saved_flags; /* Saved value of the db->flags */ int saved_nChange; /* Saved value of db->nChange */ int saved_nTotalChange; /* Saved value of db->nTotalChange */ - int (*saved_xTrace)(u32,void*,void*,void*); /* Saved db->xTrace */ + void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ int rc = SQLITE_OK; /* Return code from service routines */ char *zSql = NULL; /* SQL statements */ char *pzErrMsg = NULL; @@ -18656,8 +15457,6 @@ void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_value **ar /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC -/* #include "sqliteInt.h" */ -/* #include "btreeInt.h" */ /************** Include sqlcipher.h in the middle of crypto_impl.c ***********/ /************** Begin file sqlcipher.h ***************************************/ /* @@ -18718,7 +15517,6 @@ typedef struct { int (*ctx_init)(void **ctx); int (*ctx_free)(void **ctx); int (*fips_status)(void *ctx); - const char* (*get_provider_version)(void *ctx); } sqlcipher_provider; /* utility functions */ @@ -18740,7 +15538,6 @@ sqlcipher_provider* sqlcipher_get_provider(); /************** End of sqlcipher.h *******************************************/ /************** Continuing where we left off in crypto_impl.c ****************/ -/* #include "crypto.h" */ #ifndef OMIT_MEMLOCK #if defined(__unix__) || defined(__APPLE__) || defined(_AIX) #include @@ -18776,7 +15573,7 @@ typedef struct { static unsigned int default_flags = DEFAULT_CIPHER_FLAGS; static unsigned char hmac_salt_mask = HMAC_SALT_MASK; static int default_kdf_iter = PBKDF2_ITER; -static int default_page_size = 1024; +static int default_page_size = SQLITE_DEFAULT_PAGE_SIZE; static unsigned int sqlcipher_activate_count = 0; static sqlite3_mutex* sqlcipher_provider_mutex = NULL; static sqlcipher_provider *default_provider = NULL; @@ -19194,11 +15991,8 @@ int sqlcipher_codec_ctx_set_cipher(codec_ctx *ctx, const char *cipher_name, int cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; int rc; - rc = c_ctx->provider->set_cipher(c_ctx->provider_ctx, cipher_name); - if(rc != SQLITE_OK){ - sqlcipher_codec_ctx_set_error(ctx, rc); - return rc; - } + c_ctx->provider->set_cipher(c_ctx->provider_ctx, cipher_name); + c_ctx->key_sz = c_ctx->provider->get_key_sz(c_ctx->provider_ctx); c_ctx->iv_sz = c_ctx->provider->get_iv_sz(c_ctx->provider_ctx); c_ctx->block_sz = c_ctx->provider->get_block_sz(c_ctx->provider_ctx); @@ -19594,12 +16388,12 @@ static int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { if(ctx->read_ctx->provider->random(ctx->read_ctx->provider_ctx, ctx->kdf_salt, FILE_HEADER_SZ) != SQLITE_OK) return SQLITE_ERROR; ctx->need_kdf_salt = 0; } - if (c_ctx->pass_sz == ((c_ctx->key_sz * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0 && cipher_isHex(c_ctx->pass + 2, c_ctx->key_sz * 2)) { + if (c_ctx->pass_sz == ((c_ctx->key_sz * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0) { int n = c_ctx->pass_sz - 3; /* adjust for leading x' and tailing ' */ const unsigned char *z = c_ctx->pass + 2; /* adjust lead offset of x' */ CODEC_TRACE(("cipher_ctx_key_derive: using raw key from hex\n")); cipher_hex2bin(z, n, c_ctx->key); - } else if (c_ctx->pass_sz == (((c_ctx->key_sz + ctx->kdf_salt_sz) * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0 && cipher_isHex(c_ctx->pass + 2, (c_ctx->key_sz + ctx->kdf_salt_sz) * 2)) { + } else if (c_ctx->pass_sz == (((c_ctx->key_sz + ctx->kdf_salt_sz) * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0) { const unsigned char *z = c_ctx->pass + 2; /* adjust lead offset of x' */ CODEC_TRACE(("cipher_ctx_key_derive: using raw key from hex\n")); cipher_hex2bin(z, (c_ctx->key_sz * 2), c_ctx->key); @@ -19728,7 +16522,7 @@ int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { int saved_flags; int saved_nChange; int saved_nTotalChange; - int (*saved_xTrace)(u32,void*,void*,void*); /* Saved db->xTrace */ + void (*saved_xTrace)(void*,const char*); Db *pDb = 0; sqlite3 *db = ctx->pBt->db; const char *db_filename = sqlite3_db_filename(db, "main"); @@ -19912,22 +16706,17 @@ int sqlcipher_codec_add_random(codec_ctx *ctx, const char *zRight, int random_sz int sqlcipher_cipher_profile(sqlite3 *db, const char *destination){ FILE *f; - if(sqlite3StrICmp(destination, "stdout") == 0){ + if( strcmp(destination,"stdout")==0 ){ f = stdout; - }else if(sqlite3StrICmp(destination, "stderr") == 0){ + }else if( strcmp(destination, "stderr")==0 ){ f = stderr; - }else if(sqlite3StrICmp(destination, "off") == 0){ + }else if( strcmp(destination, "off")==0 ){ f = 0; }else{ -#if defined(_WIN32) && (__STDC_VERSION__ > 199901L) || defined(SQLITE_OS_WINRT) - if(fopen_s(&f, destination, "a") != 0){ -#else - f = fopen(destination, "a"); - if(f == 0){ -#endif - return SQLITE_ERROR; - } - + f = fopen(destination, "wb"); + if( f==0 ){ + return SQLITE_ERROR; + } } sqlite3_profile(db, sqlcipher_profile_callback, f); return SQLITE_OK; @@ -19936,17 +16725,13 @@ int sqlcipher_cipher_profile(sqlite3 *db, const char *destination){ static void sqlcipher_profile_callback(void *file, const char *sql, sqlite3_uint64 run_time){ FILE *f = (FILE*)file; double elapsed = run_time/1000000.0; - if(f) fprintf(f, "Elapsed time:%.3f ms - %s\n", elapsed, sql); + if( f ) fprintf(f, "Elapsed time:%.3f ms - %s\n", elapsed, sql); } int sqlcipher_codec_fips_status(codec_ctx *ctx) { return ctx->read_ctx->provider->fips_status(ctx->read_ctx); } -const char* sqlcipher_codec_get_provider_version(codec_ctx *ctx) { - return ctx->read_ctx->provider->get_provider_version(ctx->read_ctx); -} - #endif /* END SQLCIPHER */ @@ -19985,8 +16770,6 @@ const char* sqlcipher_codec_get_provider_version(codec_ctx *ctx) { /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC #ifdef SQLCIPHER_CRYPTO_LIBTOMCRYPT -/* #include "sqliteInt.h" */ -/* #include "sqlcipher.h" */ #include #define FORTUNA_MAX_SZ 32 @@ -20078,10 +16861,6 @@ static const char* sqlcipher_ltc_get_provider_name(void *ctx) { return "libtomcrypt"; } -static const char* sqlcipher_ltc_get_provider_version(void *ctx) { - return SCRYPT; -} - static int sqlcipher_ltc_random(void *ctx, void *buffer, int length) { #ifndef SQLCIPHER_LTC_NO_MUTEX_RAND sqlite3_mutex_enter(ltc_rand_mutex); @@ -20209,7 +16988,6 @@ int sqlcipher_ltc_setup(sqlcipher_provider *p) { p->ctx_free = sqlcipher_ltc_ctx_free; p->add_random = sqlcipher_ltc_add_random; p->fips_status = sqlcipher_ltc_fips_status; - p->get_provider_version = sqlcipher_ltc_get_provider_version; return SQLITE_OK; } @@ -20252,9 +17030,6 @@ int sqlcipher_ltc_setup(sqlcipher_provider *p) { /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC #ifdef SQLCIPHER_CRYPTO_OPENSSL -/* #include "sqliteInt.h" */ -/* #include "crypto.h" */ -/* #include "sqlcipher.h" */ #include #include #include @@ -20267,29 +17042,6 @@ static unsigned int openssl_external_init = 0; static unsigned int openssl_init_count = 0; static sqlite3_mutex* openssl_rand_mutex = NULL; -#if OPENSSL_VERSION_NUMBER < 0x10100000L -static HMAC_CTX *HMAC_CTX_new(void) -{ - HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); - if (ctx != NULL) { - HMAC_CTX_init(ctx); - } - return ctx; -} - -// Per 1.1.0 (https://wiki.openssl.org/index.php/1.1_API_Changes) -// HMAC_CTX_free should call HMAC_CTX_cleanup, then EVP_MD_CTX_Cleanup. -// HMAC_CTX_cleanup internally calls EVP_MD_CTX_cleanup so these -// calls are not needed. -static void HMAC_CTX_free(HMAC_CTX *ctx) -{ - if (ctx != NULL) { - HMAC_CTX_cleanup(ctx); - OPENSSL_free(ctx); - } -} -#endif - static int sqlcipher_openssl_add_random(void *ctx, void *buffer, int length) { #ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND sqlite3_mutex_enter(openssl_rand_mutex); @@ -20361,8 +17113,6 @@ static int sqlcipher_openssl_deactivate(void *ctx) { is called by SQLCipher internally. This should prevent SQLCipher from "cleaning up" openssl when it was initialized externally by the program */ EVP_cleanup(); - } else { - openssl_external_init = 0; } #ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND sqlite3_mutex_free(openssl_rand_mutex); @@ -20377,10 +17127,6 @@ static const char* sqlcipher_openssl_get_provider_name(void *ctx) { return "openssl"; } -static const char* sqlcipher_openssl_get_provider_version(void *ctx) { - return OPENSSL_VERSION_TEXT; -} - /* generate a defined number of random bytes */ static int sqlcipher_openssl_random (void *ctx, void *buffer, int length) { int rc = 0; @@ -20401,14 +17147,14 @@ static int sqlcipher_openssl_random (void *ctx, void *buffer, int length) { } static int sqlcipher_openssl_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { + HMAC_CTX hctx; unsigned int outlen; - HMAC_CTX* hctx = HMAC_CTX_new(); - if(hctx == NULL) return SQLITE_ERROR; - HMAC_Init_ex(hctx, hmac_key, key_sz, EVP_sha1(), NULL); - HMAC_Update(hctx, in, in_sz); - HMAC_Update(hctx, in2, in2_sz); - HMAC_Final(hctx, out, &outlen); - HMAC_CTX_free(hctx); + HMAC_CTX_init(&hctx); + HMAC_Init_ex(&hctx, hmac_key, key_sz, EVP_sha1(), NULL); + HMAC_Update(&hctx, in, in_sz); + HMAC_Update(&hctx, in2, in2_sz); + HMAC_Final(&hctx, out, &outlen); + HMAC_CTX_cleanup(&hctx); return SQLITE_OK; } @@ -20418,18 +17164,18 @@ static int sqlcipher_openssl_kdf(void *ctx, const unsigned char *pass, int pass_ } static int sqlcipher_openssl_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { + EVP_CIPHER_CTX ectx; int tmp_csz, csz; - EVP_CIPHER_CTX* ectx = EVP_CIPHER_CTX_new(); - if(ectx == NULL) return SQLITE_ERROR; - EVP_CipherInit_ex(ectx, ((openssl_ctx *)ctx)->evp_cipher, NULL, NULL, NULL, mode); - EVP_CIPHER_CTX_set_padding(ectx, 0); // no padding - EVP_CipherInit_ex(ectx, NULL, NULL, key, iv, mode); - EVP_CipherUpdate(ectx, out, &tmp_csz, in, in_sz); + + EVP_CipherInit(&ectx, ((openssl_ctx *)ctx)->evp_cipher, NULL, NULL, mode); + EVP_CIPHER_CTX_set_padding(&ectx, 0); // no padding + EVP_CipherInit(&ectx, NULL, key, iv, mode); + EVP_CipherUpdate(&ectx, out, &tmp_csz, in, in_sz); csz = tmp_csz; out += tmp_csz; - EVP_CipherFinal_ex(ectx, out, &tmp_csz); + EVP_CipherFinal(&ectx, out, &tmp_csz); csz += tmp_csz; - EVP_CIPHER_CTX_free(ectx); + EVP_CIPHER_CTX_cleanup(&ectx); assert(in_sz == csz); return SQLITE_OK; } @@ -20513,7 +17259,6 @@ int sqlcipher_openssl_setup(sqlcipher_provider *p) { p->ctx_free = sqlcipher_openssl_ctx_free; p->add_random = sqlcipher_openssl_add_random; p->fips_status = sqlcipher_openssl_fips_status; - p->get_provider_version = sqlcipher_openssl_get_provider_version; return SQLITE_OK; } @@ -20556,11 +17301,8 @@ int sqlcipher_openssl_setup(sqlcipher_provider *p) { /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC #ifdef SQLCIPHER_CRYPTO_CC -/* #include "crypto.h" */ -/* #include "sqlcipher.h" */ #include #include -#include static int sqlcipher_cc_add_random(void *ctx, void *buffer, int length) { return SQLITE_OK; @@ -20575,20 +17317,6 @@ static const char* sqlcipher_cc_get_provider_name(void *ctx) { return "commoncrypto"; } -static const char* sqlcipher_cc_get_provider_version(void *ctx) { -#if TARGET_OS_MAC - CFTypeRef version; - CFBundleRef bundle = CFBundleGetBundleWithIdentifier(CFSTR("com.apple.security")); - if(bundle == NULL) { - return "unknown"; - } - version = CFBundleGetValueForInfoDictionaryKey(bundle, CFSTR("CFBundleShortVersionString")); - return CFStringGetCStringPtr(version, kCFStringEncodingUTF8); -#else - return "unknown"; -#endif -} - static int sqlcipher_cc_hmac(void *ctx, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { CCHmacContext hmac_context; CCHmacInit(&hmac_context, kCCHmacAlgSHA1, hmac_key, key_sz); @@ -20615,7 +17343,7 @@ static int sqlcipher_cc_cipher(void *ctx, int mode, unsigned char *key, int key_ CCCryptorFinal(cryptor, out, in_sz - csz, &tmp_csz); csz += tmp_csz; CCCryptorRelease(cryptor); - assert(in_sz == csz); + assert(size == csz); return SQLITE_OK; } @@ -20682,7 +17410,6 @@ int sqlcipher_cc_setup(sqlcipher_provider *p) { p->ctx_free = sqlcipher_cc_ctx_free; p->add_random = sqlcipher_cc_add_random; p->fips_status = sqlcipher_cc_fips_status; - p->get_provider_version = sqlcipher_cc_get_provider_version; return SQLITE_OK; } @@ -20706,7 +17433,6 @@ int sqlcipher_cc_setup(sqlcipher_provider *p) { ** ** This file contains definitions of global variables and constants. */ -/* #include "sqliteInt.h" */ /* An array to map all upper-case characters into their corresponding ** lower-case character. @@ -20764,7 +17490,6 @@ SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { ** isxdigit() 0x08 ** toupper() 0x20 ** SQLite identifier character 0x40 -** Quote character 0x80 ** ** Bit 0x20 is set if the mapped character requires translation to upper ** case. i.e. if the character is a lower-case ASCII character. @@ -20773,13 +17498,16 @@ SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { ** ** (x & ~(map[x]&0x20)) ** -** The equivalent of tolower() is implemented using the sqlite3UpperToLower[] +** Standard function tolower() is implemented using the sqlite3UpperToLower[] ** array. tolower() is used more often than toupper() by SQLite. ** -** Bit 0x40 is set if the character is non-alphanumeric and can be used in an +** Bit 0x40 is set if the character non-alphanumeric and can be used in an ** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any ** non-ASCII UTF character. Hence the test for whether or not a character is ** part of an identifier is 0x46. +** +** SQLite's versions are identical to the standard versions assuming a +** locale of "C". They are implemented as macros in sqliteInt.h. */ #ifdef SQLITE_ASCII SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { @@ -20787,7 +17515,7 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */ - 0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80, /* 20..27 !"#$%&' */ + 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */ 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */ @@ -20795,8 +17523,8 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ - 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ - 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ + 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ + 0x00, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ @@ -20851,18 +17579,6 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { # define SQLITE_SORTER_PMASZ 250 #endif -/* Statement journals spill to disk when their size exceeds the following -** threshold (in bytes). 0 means that statement journals are created and -** written to disk immediately (the default behavior for SQLite versions -** before 3.12.0). -1 means always keep the entire statement journal in -** memory. (The statement journal is also always held entirely in memory -** if journal_mode=MEMORY or if temp_store=MEMORY, regardless of this -** setting.) -*/ -#ifndef SQLITE_STMTJRNL_SPILL -# define SQLITE_STMTJRNL_SPILL (64*1024) -#endif - /* ** The following singleton contains the global configuration for ** the SQLite library. @@ -20877,7 +17593,6 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* neverCorrupt */ 128, /* szLookaside */ 500, /* nLookaside */ - SQLITE_STMTJRNL_SPILL, /* nStmtSpill */ {0,0,0,0,0,0,0,0}, /* m */ {0,0,0,0,0,0,0,0,0}, /* mutex */ {0,0,0,0,0,0,0,0,0,0,0,0,0},/* pcache2 */ @@ -20891,7 +17606,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nScratch */ (void*)0, /* pPage */ 0, /* szPage */ - SQLITE_DEFAULT_PCACHE_INITSZ, /* nPage */ + 0, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ SQLITE_SORTER_PMASZ, /* szPma */ @@ -20916,8 +17631,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { #ifndef SQLITE_OMIT_BUILTIN_TEST 0, /* xTestCallback */ #endif - 0, /* bLocaltimeFault */ - 0x7ffffffe /* iOnceResetThreshold */ + 0 /* bLocaltimeFault */ }; /* @@ -20925,7 +17639,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { ** database connections. After initialization, this table is ** read-only. */ -SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; +SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; /* ** Constant tokens for values 0 and 1. @@ -20940,7 +17654,7 @@ SQLITE_PRIVATE const Token sqlite3IntTokens[] = { ** The value of the "pending" byte must be 0x40000000 (1 byte past the ** 1-gibabyte boundary) in a compatible database. SQLite never uses ** the database page that contains the pending byte. It never attempts -** to read or write that page. The pending byte page is set aside +** to read or write that page. The pending byte page is set assign ** for use by the VFS layers as space for managing file locks. ** ** During testing, it is often desirable to move the pending byte to @@ -20958,7 +17672,6 @@ SQLITE_PRIVATE const Token sqlite3IntTokens[] = { SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000; #endif -/* #include "opcodes.h" */ /* ** Properties of opcodes. The OPFLG_INITIALIZER macro is ** created by mkopcodeh.awk during compilation. Data is obtained @@ -20967,11 +17680,6 @@ SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000; */ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; -/* -** Name of the default collating sequence -*/ -SQLITE_PRIVATE const char sqlite3StrBINARY[] = "BINARY"; - /************** End of global.c **********************************************/ /************** Begin file ctime.c *******************************************/ /* @@ -20992,7 +17700,6 @@ SQLITE_PRIVATE const char sqlite3StrBINARY[] = "BINARY"; #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS -/* #include "sqliteInt.h" */ /* ** An array of names of all compile-time options. This array should @@ -21021,15 +17728,6 @@ static const char * const azCompileOpt[] = { #if SQLITE_CHECK_PAGES "CHECK_PAGES", #endif -#if defined(__clang__) && defined(__clang_major__) - "COMPILER=clang-" CTIMEOPT_VAL(__clang_major__) "." - CTIMEOPT_VAL(__clang_minor__) "." - CTIMEOPT_VAL(__clang_patchlevel__), -#elif defined(_MSC_VER) - "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER), -#elif defined(__GNUC__) && defined(__VERSION__) - "COMPILER=gcc-" __VERSION__, -#endif #if SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif @@ -21048,9 +17746,6 @@ static const char * const azCompileOpt[] = { #if SQLITE_DISABLE_LFS "DISABLE_LFS", #endif -#if SQLITE_ENABLE_8_3_NAMES - "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), -#endif #if SQLITE_ENABLE_API_ARMOR "ENABLE_API_ARMOR", #endif @@ -21084,18 +17779,12 @@ static const char * const azCompileOpt[] = { #if SQLITE_ENABLE_FTS4 "ENABLE_FTS4", #endif -#if SQLITE_ENABLE_FTS5 - "ENABLE_FTS5", -#endif #if SQLITE_ENABLE_ICU "ENABLE_ICU", #endif #if SQLITE_ENABLE_IOTRACE "ENABLE_IOTRACE", #endif -#if SQLITE_ENABLE_JSON1 - "ENABLE_JSON1", -#endif #if SQLITE_ENABLE_LOAD_EXTENSION "ENABLE_LOAD_EXTENSION", #endif @@ -21146,9 +17835,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_INT64_TYPE "INT64_TYPE", #endif -#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS - "LIKE_DOESNT_MATCH_BLOBS", -#endif #if SQLITE_LOCK_TRACE "LOCK_TRACE", #endif @@ -21389,7 +18075,7 @@ static const char * const azCompileOpt[] = { ** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix ** is not required for a match. */ -SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ +SQLITE_API int SQLITE_STDCALL sqlite3_compileoption_used(const char *zOptName){ int i, n; #if SQLITE_ENABLE_API_ARMOR @@ -21417,7 +18103,7 @@ SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ ** Return the N-th compile-time option string. If N is out of range, ** return a NULL pointer. */ -SQLITE_API const char *sqlite3_compileoption_get(int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_compileoption_get(int N){ if( N>=0 && NaDb[] (or -1) */ - u8 nullRow; /* True if pointing to a row with no data */ - u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ - u8 isTable; /* True for rowid tables. False for indexes */ + BtCursor *pCursor; /* The cursor structure of the backend */ + Btree *pBt; /* Separate file holding temporary table */ + KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ + int seekResult; /* Result of previous sqlite3BtreeMoveto() */ + int pseudoTableReg; /* Register holding pseudotable content. */ + i16 nField; /* Number of fields in the header */ + u16 nHdrParsed; /* Number of header fields parsed so far */ #ifdef SQLITE_DEBUG u8 seekOp; /* Most recent seek operation on this cursor */ - u8 wrFlag; /* The wrFlag argument to sqlite3BtreeCursor() */ #endif + i8 iDb; /* Index of cursor database in db->aDb[] (or -1) */ + u8 nullRow; /* True if pointing to a row with no data */ + u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ Bool isEphemeral:1; /* True for an ephemeral table */ Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ - Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ + Bool isTable:1; /* True if a table requiring integer keys */ + Bool isOrdered:1; /* True if the underlying table is BTREE_UNORDERED */ Pgno pgnoRoot; /* Root page of the open btree cursor */ - i16 nField; /* Number of fields in the header */ - u16 nHdrParsed; /* Number of header fields parsed so far */ - union { - BtCursor *pCursor; /* CURTYPE_BTREE. Btree cursor */ - sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB. Vtab cursor */ - int pseudoTableReg; /* CURTYPE_PSEUDO. Reg holding content. */ - VdbeSorter *pSorter; /* CURTYPE_SORTER. Sorter object */ - } uc; - Btree *pBt; /* Separate file holding temporary table */ - KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ - int seekResult; /* Result of previous sqlite3BtreeMoveto() */ + sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ - VdbeCursor *pAltCursor; /* Associated index cursor from which to read */ - int *aAltMap; /* Mapping from table to index column numbers */ -#ifdef SQLITE_ENABLE_COLUMN_USED_MASK - u64 maskUsed; /* Mask of columns used by this cursor */ -#endif + VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ /* Cached information about the header for the data record that the ** cursor is currently pointing to. Only valid if cacheStatus matches @@ -21573,12 +18237,7 @@ struct VdbeCursor { ** static element declared in the structure. nField total array slots for ** aType[] and nField+1 array slots for aOffset[] */ }; - - -/* -** A value for VdbeCursor.cacheStatus that means the cache is always invalid. -*/ -#define CACHE_STALE 0 +typedef struct VdbeCursor VdbeCursor; /* ** When a sub-program is executed (OP_Program), a structure of this type @@ -21608,14 +18267,15 @@ struct VdbeFrame { Op *aOp; /* Program instructions for parent frame */ i64 *anExec; /* Event counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ + u8 *aOnceFlag; /* Array of OP_Once flags for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ void *token; /* Copy of SubProgram.token */ i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ - AuxData *pAuxData; /* Linked list of auxdata allocations */ int nCursor; /* Number of entries in apCsr */ int pc; /* Program Counter in parent (calling) frame */ int nOp; /* Size of aOp array */ int nMem; /* Number of entries in aMem */ + int nOnceFlag; /* Number of entries in aOnceFlag */ int nChildMem; /* Number of memory cells for child frame */ int nChildCsr; /* Number of cursors for child frame */ int nChange; /* Statement changes (Vdbe.nChange) */ @@ -21624,6 +18284,11 @@ struct VdbeFrame { #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) +/* +** A value for VdbeCursor.cacheValid that means the cache is always invalid. +*/ +#define CACHE_STALE 0 + /* ** Internally, the vdbe manipulates nearly all SQL values as Mem ** structures. Each Mem struct may cache multiple representations (string, @@ -21640,7 +18305,6 @@ struct Mem { } u; u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ - u8 eSubtype; /* Subtype for this value */ int n; /* Number of characters in string value, excluding '\0' */ char *z; /* String or BLOB value */ /* ShallowCopy only needs to copy the information above */ @@ -21655,12 +18319,6 @@ struct Mem { #endif }; -/* -** Size of struct Mem not including the Mem.zMalloc member or anything that -** follows. -*/ -#define MEMCELLSIZE offsetof(Mem,zMalloc) - /* One or more of the following flags are set to indicate the validOK ** representations of the value stored in the Mem struct. ** @@ -21683,7 +18341,7 @@ struct Mem { #define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ #define MEM_Undefined 0x0080 /* Value is undefined */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ -#define MEM_TypeMask 0x81ff /* Mask of type bits */ +#define MEM_TypeMask 0x01ff /* Mask of type bits */ /* Whenever Mem contains a valid string or blob representation, one of @@ -21697,18 +18355,11 @@ struct Mem { #define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ #define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ #define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ -#define MEM_Subtype 0x8000 /* Mem.eSubtype is valid */ #ifdef SQLITE_OMIT_INCRBLOB #undef MEM_Zero #define MEM_Zero 0x0000 #endif -/* Return TRUE if Mem X contains dynamically allocated content - anything -** that needs to be deallocated to avoid a leak. -*/ -#define VdbeMemDynamic(X) \ - (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) - /* ** Clear any existing type flags from a Mem and replace them with f */ @@ -21752,16 +18403,26 @@ struct AuxData { ** (Mem) which are only defined there. */ struct sqlite3_context { - Mem *pOut; /* The return value is stored here */ - FuncDef *pFunc; /* Pointer to function information */ - Mem *pMem; /* Memory cell used to store aggregate context */ - Vdbe *pVdbe; /* The VM that owns this context */ - int iOp; /* Instruction number of OP_Function */ - int isError; /* Error code returned by the function. */ - u8 skipFlag; /* Skip accumulator loading if true */ - u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ - u8 argc; /* Number of arguments */ - sqlite3_value *argv[1]; /* Argument set */ + Mem *pOut; /* The return value is stored here */ + FuncDef *pFunc; /* Pointer to function information */ + Mem *pMem; /* Memory cell used to store aggregate context */ + Vdbe *pVdbe; /* The VM that owns this context */ + int iOp; /* Instruction number of OP_Function */ + int isError; /* Error code returned by the function. */ + u8 skipFlag; /* Skip accumulator loading if true */ + u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ +}; + +/* +** An Explain object accumulates indented output which is helpful +** in describing recursive data structures. +*/ +struct Explain { + Vdbe *pVdbe; /* Attach the explanation to this Vdbe */ + StrAccum str; /* The string being accumulated */ + int nIndent; /* Number of elements in aIndent */ + u16 aIndent[100]; /* Levels of indentation */ + char zBase[100]; /* Initial space */ }; /* A bitfield type for use inside of structures. Always follow with :N where @@ -21788,57 +18449,53 @@ struct ScanStatus { */ struct Vdbe { sqlite3 *db; /* The database connection that owns this statement */ - Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ - Parse *pParse; /* Parsing context used to create this Vdbe */ - ynVar nVar; /* Number of entries in aVar[] */ - ynVar nzVar; /* Number of entries in azVar[] */ - u32 magic; /* Magic number for sanity checking */ - int nMem; /* Number of memory locations currently allocated */ - int nCursor; /* Number of slots in apCsr[] */ - u32 cacheCtr; /* VdbeCursor row cache generation counter */ - int pc; /* The program counter */ - int rc; /* Value to return */ - int nChange; /* Number of db changes made since last reset */ - int iStatement; /* Statement number (or 0 if has not opened stmt) */ - i64 iCurrentTime; /* Value of julianday('now') for this statement */ - i64 nFkConstraint; /* Number of imm. FK constraints this VM */ - i64 nStmtDefCons; /* Number of def. constraints when stmt started */ - i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ - - /* When allocating a new Vdbe object, all of the fields below should be - ** initialized to zero or NULL */ - Op *aOp; /* Space to hold the virtual machine's program */ Mem *aMem; /* The memory locations */ Mem **apArg; /* Arguments to currently executing user function */ Mem *aColName; /* Column names to return */ Mem *pResultSet; /* Pointer to an array of results */ + Parse *pParse; /* Parsing context used to create this Vdbe */ + int nMem; /* Number of memory locations currently allocated */ + int nOp; /* Number of instructions in the program */ + int nCursor; /* Number of slots in apCsr[] */ + u32 magic; /* Magic number for sanity checking */ char *zErrMsg; /* Error message written here */ + Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ VdbeCursor **apCsr; /* One element of this array for each open cursor */ Mem *aVar; /* Values for the OP_Variable opcode. */ char **azVar; /* Name of variables */ -#ifndef SQLITE_OMIT_TRACE - i64 startTime; /* Time when query started - used for profiling */ -#endif - int nOp; /* Number of instructions in the program */ + ynVar nVar; /* Number of entries in aVar[] */ + ynVar nzVar; /* Number of entries in azVar[] */ + u32 cacheCtr; /* VdbeCursor row cache generation counter */ + int pc; /* The program counter */ + int rc; /* Value to return */ #ifdef SQLITE_DEBUG int rcApp; /* errcode set by sqlite3_result_error_code() */ #endif u16 nResColumn; /* Number of columns in one row of the result set */ u8 errorAction; /* Recovery action to do in case of an error */ u8 minWriteFileFormat; /* Minimum file format for writable database files */ - bft expired:1; /* True if the VM needs to be recompiled */ - bft doingRerun:1; /* True if rerunning after an auto-reprepare */ bft explain:2; /* True if EXPLAIN present on SQL command */ bft changeCntOn:1; /* True to update the change-counter */ + bft expired:1; /* True if the VM needs to be recompiled */ bft runOnlyOnce:1; /* Automatically expire on reset */ bft usesStmtJournal:1; /* True if uses a statement journal */ bft readOnly:1; /* True for statements that do not write */ bft bIsReader:1; /* True for statements that read */ bft isPrepareV2:1; /* True if prepared with prepare_v2() */ + bft doingRerun:1; /* True if rerunning after an auto-reprepare */ + int nChange; /* Number of db changes made since last reset */ yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ yDbMask lockMask; /* Subset of btreeMask that requires a lock */ + int iStatement; /* Statement number (or 0 if has not opened stmt) */ u32 aCounter[5]; /* Counters used by sqlite3_stmt_status() */ +#ifndef SQLITE_OMIT_TRACE + i64 startTime; /* Time when query started - used for profiling */ +#endif + i64 iCurrentTime; /* Value of julianday('now') for this statement */ + i64 nFkConstraint; /* Number of imm. FK constraints this VM */ + i64 nStmtDefCons; /* Number of def. constraints when stmt started */ + i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ char *zSql; /* Text of the SQL statement that generated this */ void *pFree; /* Free this when deleting the vdbe */ VdbeFrame *pFrame; /* Parent frame */ @@ -21846,6 +18503,8 @@ struct Vdbe { int nFrame; /* Number of frames in pFrame list */ u32 expmask; /* Binding to these vars invalidates VM */ SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ + int nOnceFlag; /* Size of array aOnceFlag[] */ + u8 *aOnceFlag; /* Flags for OP_Once */ AuxData *pAuxData; /* Linked list of auxdata allocations */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS i64 *anExec; /* Number of times each op has been executed */ @@ -21857,48 +18516,26 @@ struct Vdbe { /* ** The following are allowed values for Vdbe.magic */ -#define VDBE_MAGIC_INIT 0x16bceaa5 /* Building a VDBE program */ -#define VDBE_MAGIC_RUN 0x2df20da3 /* VDBE is ready to execute */ -#define VDBE_MAGIC_HALT 0x319c2973 /* VDBE has completed execution */ -#define VDBE_MAGIC_RESET 0x48fa9f76 /* Reset and ready to run again */ -#define VDBE_MAGIC_DEAD 0x5606c3c8 /* The VDBE has been deallocated */ - -/* -** Structure used to store the context required by the -** sqlite3_preupdate_*() API functions. -*/ -struct PreUpdate { - Vdbe *v; - VdbeCursor *pCsr; /* Cursor to read old values from */ - int op; /* One of SQLITE_INSERT, UPDATE, DELETE */ - u8 *aRecord; /* old.* database record */ - KeyInfo keyinfo; - UnpackedRecord *pUnpacked; /* Unpacked version of aRecord[] */ - UnpackedRecord *pNewUnpacked; /* Unpacked version of new.* record */ - int iNewReg; /* Register for new.* values */ - i64 iKey1; /* First key value passed to hook */ - i64 iKey2; /* Second key value passed to hook */ - Mem *aNew; /* Array of new.* values */ - Table *pTab; /* Schema object being upated */ -}; +#define VDBE_MAGIC_INIT 0x26bceaa5 /* Building a VDBE program */ +#define VDBE_MAGIC_RUN 0xbdf20da3 /* VDBE is ready to execute */ +#define VDBE_MAGIC_HALT 0x519c2973 /* VDBE has completed execution */ +#define VDBE_MAGIC_DEAD 0xb606c3c8 /* The VDBE has been deallocated */ /* ** Function prototypes */ -SQLITE_PRIVATE void sqlite3VdbeError(Vdbe*, const char *, ...); SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); void sqliteVdbePopStack(Vdbe*,int); -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor**, int*); +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor*); SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor*); #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, Op*); #endif SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); -SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8); -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int, u32*); +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int); SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32); SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); -SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3*, AuxData**, int, int); +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(sqlite3*,VdbeCursor*,UnpackedRecord*,int*); @@ -21934,6 +18571,8 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem*,u8,u8); SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,int,Mem*); SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); +#define VdbeMemDynamic(X) \ + (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); SQLITE_PRIVATE const char *sqlite3OpcodeName(int); SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); @@ -21941,9 +18580,6 @@ SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int n); SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK -SQLITE_PRIVATE void sqlite3VdbePreUpdateHook(Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int); -#endif SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p); SQLITE_PRIVATE int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); @@ -21955,15 +18591,11 @@ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *, int *); SQLITE_PRIVATE int sqlite3VdbeSorterWrite(const VdbeCursor *, Mem *); SQLITE_PRIVATE int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); -#if !defined(SQLITE_OMIT_SHARED_CACHE) -SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe*); -#else -# define sqlite3VdbeEnter(X) -#endif - #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe*); #else +# define sqlite3VdbeEnter(X) # define sqlite3VdbeLeave(X) #endif @@ -21993,7 +18625,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); #define ExpandBlob(P) SQLITE_OK #endif -#endif /* !defined(SQLITE_VDBEINT_H) */ +#endif /* !defined(_VDBEINT_H_) */ /************** End of vdbeInt.h *********************************************/ /************** Continuing where we left off in status.c *********************/ @@ -22001,15 +18633,15 @@ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); /* ** Variables in which to record status information. */ +typedef struct sqlite3StatType sqlite3StatType; +static SQLITE_WSD struct sqlite3StatType { #if SQLITE_PTRSIZE>4 -typedef sqlite3_int64 sqlite3StatValueType; + sqlite3_int64 nowValue[10]; /* Current value */ + sqlite3_int64 mxValue[10]; /* Maximum value */ #else -typedef u32 sqlite3StatValueType; + u32 nowValue[10]; /* Current value */ + u32 mxValue[10]; /* Maximum value */ #endif -typedef struct sqlite3StatType sqlite3StatType; -static SQLITE_WSD struct sqlite3StatType { - sqlite3StatValueType nowValue[10]; /* Current value */ - sqlite3StatValueType mxValue[10]; /* Maximum value */ } sqlite3Stat = { {0,}, {0,} }; /* @@ -22090,31 +18722,25 @@ SQLITE_PRIVATE void sqlite3StatusDown(int op, int N){ } /* -** Adjust the highwater mark if necessary. -** The caller must hold the appropriate mutex. +** Set the value of a status to X. The highwater mark is adjusted if +** necessary. The caller must hold the appropriate mutex. */ -SQLITE_PRIVATE void sqlite3StatusHighwater(int op, int X){ - sqlite3StatValueType newValue; +SQLITE_PRIVATE void sqlite3StatusSet(int op, int X){ wsdStatInit; - assert( X>=0 ); - newValue = (sqlite3StatValueType)X; assert( op>=0 && op=0 && opwsdStat.mxValue[op] ){ - wsdStat.mxValue[op] = newValue; + wsdStat.nowValue[op] = X; + if( wsdStat.nowValue[op]>wsdStat.mxValue[op] ){ + wsdStat.mxValue[op] = wsdStat.nowValue[op]; } } /* ** Query status information. */ -SQLITE_API int sqlite3_status64( +SQLITE_API int SQLITE_STDCALL sqlite3_status64( int op, sqlite3_int64 *pCurrent, sqlite3_int64 *pHighwater, @@ -22139,8 +18765,8 @@ SQLITE_API int sqlite3_status64( (void)pMutex; /* Prevent warning when SQLITE_THREADSAFE=0 */ return SQLITE_OK; } -SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ - sqlite3_int64 iCur = 0, iHwtr = 0; +SQLITE_API int SQLITE_STDCALL sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ + sqlite3_int64 iCur, iHwtr; int rc; #ifdef SQLITE_ENABLE_API_ARMOR if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; @@ -22156,7 +18782,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF /* ** Query status information for a single database connection */ -SQLITE_API int sqlite3_db_status( +SQLITE_API int SQLITE_STDCALL sqlite3_db_status( sqlite3 *db, /* The database connection whose status is desired */ int op, /* Status verb */ int *pCurrent, /* Write current value here */ @@ -22201,7 +18827,6 @@ SQLITE_API int sqlite3_db_status( ** by all pagers associated with the given database connection. The ** highwater mark is meaningless and is returned as zero. */ - case SQLITE_DBSTATUS_CACHE_USED_SHARED: case SQLITE_DBSTATUS_CACHE_USED: { int totalUsed = 0; int i; @@ -22210,11 +18835,7 @@ SQLITE_API int sqlite3_db_status( Btree *pBt = db->aDb[i].pBt; if( pBt ){ Pager *pPager = sqlite3BtreePager(pBt); - int nByte = sqlite3PagerMemUsed(pPager); - if( op==SQLITE_DBSTATUS_CACHE_USED_SHARED ){ - nByte = nByte / sqlite3BtreeConnectionCount(pBt); - } - totalUsed += nByte; + totalUsed += sqlite3PagerMemUsed(pPager); } } sqlite3BtreeLeaveAll(db); @@ -22245,10 +18866,10 @@ SQLITE_API int sqlite3_db_status( + pSchema->idxHash.count + pSchema->fkeyHash.count ); - nByte += sqlite3_msize(pSchema->tblHash.ht); - nByte += sqlite3_msize(pSchema->trigHash.ht); - nByte += sqlite3_msize(pSchema->idxHash.ht); - nByte += sqlite3_msize(pSchema->fkeyHash.ht); + nByte += sqlite3MallocSize(pSchema->tblHash.ht); + nByte += sqlite3MallocSize(pSchema->trigHash.ht); + nByte += sqlite3MallocSize(pSchema->idxHash.ht); + nByte += sqlite3MallocSize(pSchema->fkeyHash.ht); for(p=sqliteHashFirst(&pSchema->trigHash); p; p=sqliteHashNext(p)){ sqlite3DeleteTrigger(db, (Trigger*)sqliteHashData(p)); @@ -22379,22 +19000,12 @@ SQLITE_API int sqlite3_db_status( ** Willmann-Bell, Inc ** Richmond, Virginia (USA) */ -/* #include "sqliteInt.h" */ /* #include */ /* #include */ #include #ifndef SQLITE_OMIT_DATETIME_FUNCS -/* -** The MSVC CRT on Windows CE may not have a localtime() function. -** So declare a substitute. The substitute function itself is -** defined in "os_win.c". -*/ -#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ - (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) -struct tm *__cdecl localtime(const time_t *); -#endif /* ** A structure for holding a single date and time. @@ -22410,54 +19021,38 @@ struct DateTime { char validHMS; /* True (1) if h,m,s are valid */ char validJD; /* True (1) if iJD is valid */ char validTZ; /* True (1) if tz is valid */ - char tzSet; /* Timezone was set explicitly */ }; /* -** Convert zDate into one or more integers according to the conversion -** specifier zFormat. -** -** zFormat[] contains 4 characters for each integer converted, except for -** the last integer which is specified by three characters. The meaning -** of a four-character format specifiers ABCD is: +** Convert zDate into one or more integers. Additional arguments +** come in groups of 5 as follows: ** -** A: number of digits to convert. Always "2" or "4". -** B: minimum value. Always "0" or "1". -** C: maximum value, decoded as: -** a: 12 -** b: 14 -** c: 24 -** d: 31 -** e: 59 -** f: 9999 -** D: the separator character, or \000 to indicate this is the -** last number to convert. -** -** Example: To translate an ISO-8601 date YYYY-MM-DD, the format would -** be "40f-21a-20c". The "40f-" indicates the 4-digit year followed by "-". -** The "21a-" indicates the 2-digit month followed by "-". The "20c" indicates -** the 2-digit day which is the last integer in the set. +** N number of digits in the integer +** min minimum allowed value of the integer +** max maximum allowed value of the integer +** nextC first character after the integer +** pVal where to write the integers value. ** +** Conversions continue until one with nextC==0 is encountered. ** The function returns the number of successful conversions. */ -static int getDigits(const char *zDate, const char *zFormat, ...){ - /* The aMx[] array translates the 3rd character of each format - ** spec into a max size: a b c d e f */ - static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 }; +static int getDigits(const char *zDate, ...){ va_list ap; + int val; + int N; + int min; + int max; + int nextC; + int *pVal; int cnt = 0; - char nextC; - va_start(ap, zFormat); + va_start(ap, zDate); do{ - char N = zFormat[0] - '0'; - char min = zFormat[1] - '0'; - int val = 0; - u16 max; - - assert( zFormat[2]>='a' && zFormat[2]<='f' ); - max = aMx[zFormat[2] - 'a']; - nextC = zFormat[3]; + N = va_arg(ap, int); + min = va_arg(ap, int); + max = va_arg(ap, int); + nextC = va_arg(ap, int); + pVal = va_arg(ap, int*); val = 0; while( N-- ){ if( !sqlite3Isdigit(*zDate) ){ @@ -22466,13 +19061,12 @@ static int getDigits(const char *zDate, const char *zFormat, ...){ val = val*10 + *zDate - '0'; zDate++; } - if( val<(int)min || val>(int)max || (nextC!=0 && nextC!=*zDate) ){ + if( valmax || (nextC!=0 && nextC!=*zDate) ){ goto end_getDigits; } - *va_arg(ap,int*) = val; + *pVal = val; zDate++; cnt++; - zFormat += 4; }while( nextC ); end_getDigits: va_end(ap); @@ -22513,14 +19107,13 @@ static int parseTimezone(const char *zDate, DateTime *p){ return c!=0; } zDate++; - if( getDigits(zDate, "20b:20e", &nHr, &nMn)!=2 ){ + if( getDigits(zDate, 2, 0, 14, ':', &nHr, 2, 0, 59, 0, &nMn)!=2 ){ return 1; } zDate += 5; p->tz = sgn*(nMn + nHr*60); zulu_time: while( sqlite3Isspace(*zDate) ){ zDate++; } - p->tzSet = 1; return *zDate!=0; } @@ -22534,13 +19127,13 @@ static int parseTimezone(const char *zDate, DateTime *p){ static int parseHhMmSs(const char *zDate, DateTime *p){ int h, m, s; double ms = 0.0; - if( getDigits(zDate, "20c:20e", &h, &m)!=2 ){ + if( getDigits(zDate, 2, 0, 24, ':', &h, 2, 0, 59, 0, &m)!=2 ){ return 1; } zDate += 5; if( *zDate==':' ){ zDate++; - if( getDigits(zDate, "20e", &s)!=1 ){ + if( getDigits(zDate, 2, 0, 59, 0, &s)!=1 ){ return 1; } zDate += 2; @@ -22628,7 +19221,7 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ }else{ neg = 0; } - if( getDigits(zDate, "40f-21a-21d", &Y, &M, &D)!=3 ){ + if( getDigits(zDate,4,0,9999,'-',&Y,2,1,12,'-',&M,2,1,31,0,&D)!=3 ){ return 1; } zDate += 10; @@ -22718,7 +19311,7 @@ static void computeYMD(DateTime *p){ A = Z + 1 + A - (A/4); B = A + 1524; C = (int)((B - 122.1)/365.25); - D = (36525*(C&32767))/100; + D = (36525*C)/100; E = (int)((B-D)/30.6001); X1 = (int)(30.6001*E); p->D = B - D - X1; @@ -22763,7 +19356,6 @@ static void clearYMD_HMS_TZ(DateTime *p){ p->validTZ = 0; } -#ifndef SQLITE_OMIT_LOCALTIME /* ** On recent Windows platforms, the localtime_s() function is available ** as part of the "Secure CRT". It is essentially equivalent to @@ -22782,6 +19374,7 @@ static void clearYMD_HMS_TZ(DateTime *p){ #define HAVE_LOCALTIME_S 1 #endif +#ifndef SQLITE_OMIT_LOCALTIME /* ** The following routine implements the rough equivalent of localtime_r() ** using whatever operating-system specific localtime facility that @@ -22953,18 +19546,13 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ } #ifndef SQLITE_OMIT_LOCALTIME else if( strcmp(z, "utc")==0 ){ - if( p->tzSet==0 ){ - sqlite3_int64 c1; - computeJD(p); - c1 = localtimeOffset(p, pCtx, &rc); - if( rc==SQLITE_OK ){ - p->iJD -= c1; - clearYMD_HMS_TZ(p); - p->iJD += c1 - localtimeOffset(p, pCtx, &rc); - } - p->tzSet = 1; - }else{ - rc = SQLITE_OK; + sqlite3_int64 c1; + computeJD(p); + c1 = localtimeOffset(p, pCtx, &rc); + if( rc==SQLITE_OK ){ + p->iJD -= c1; + clearYMD_HMS_TZ(p); + p->iJD += c1 - localtimeOffset(p, pCtx, &rc); } } #endif @@ -23312,7 +19900,7 @@ static void strftimeFunc( sqlite3_result_error_toobig(context); return; }else{ - z = sqlite3DbMallocRawNN(db, (int)n); + z = sqlite3DbMallocRaw(db, (int)n); if( z==0 ){ sqlite3_result_error_nomem(context); return; @@ -23448,6 +20036,7 @@ static void currentTimeFunc( ){ time_t t; char *zFormat = (char *)sqlite3_user_data(context); + sqlite3 *db; sqlite3_int64 iT; struct tm *pTm; struct tm sNow; @@ -23480,23 +20069,29 @@ static void currentTimeFunc( ** external linkage. */ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ - static FuncDef aDateTimeFuncs[] = { + static SQLITE_WSD FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS - DFUNCTION(julianday, -1, 0, 0, juliandayFunc ), - DFUNCTION(date, -1, 0, 0, dateFunc ), - DFUNCTION(time, -1, 0, 0, timeFunc ), - DFUNCTION(datetime, -1, 0, 0, datetimeFunc ), - DFUNCTION(strftime, -1, 0, 0, strftimeFunc ), - DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), - DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), - DFUNCTION(current_date, 0, 0, 0, cdateFunc ), + FUNCTION(julianday, -1, 0, 0, juliandayFunc ), + FUNCTION(date, -1, 0, 0, dateFunc ), + FUNCTION(time, -1, 0, 0, timeFunc ), + FUNCTION(datetime, -1, 0, 0, datetimeFunc ), + FUNCTION(strftime, -1, 0, 0, strftimeFunc ), + FUNCTION(current_time, 0, 0, 0, ctimeFunc ), + FUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), + FUNCTION(current_date, 0, 0, 0, cdateFunc ), #else STR_FUNCTION(current_time, 0, "%H:%M:%S", 0, currentTimeFunc), STR_FUNCTION(current_date, 0, "%Y-%m-%d", 0, currentTimeFunc), STR_FUNCTION(current_timestamp, 0, "%Y-%m-%d %H:%M:%S", 0, currentTimeFunc), #endif }; - sqlite3InsertBuiltinFuncs(aDateTimeFuncs, ArraySize(aDateTimeFuncs)); + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aDateTimeFuncs); + + for(i=0; ipMethods ){ - pId->pMethods->xClose(pId); + rc = pId->pMethods->xClose(pId); pId->pMethods = 0; } + return rc; } SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file *id, void *pBuf, int amt, i64 offset){ DO_OS_MALLOC_TEST(id); @@ -23631,8 +20207,8 @@ SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ #ifdef SQLITE_TEST if( op!=SQLITE_FCNTL_COMMIT_PHASETWO ){ /* Faults are not injected into COMMIT_PHASETWO because, assuming SQLite - ** is using a regular VFS, it is called after the corresponding - ** transaction has been committed. Injecting a fault at this point + ** is using a regular VFS, it is called after the corresponding + ** transaction has been committed. Injecting a fault at this point ** confuses the test scripts - the COMMIT comand returns SQLITE_NOMEM ** but the transaction is committed anyway. ** @@ -23701,10 +20277,10 @@ SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ ** VFS methods. */ SQLITE_PRIVATE int sqlite3OsOpen( - sqlite3_vfs *pVfs, - const char *zPath, - sqlite3_file *pFile, - int flags, + sqlite3_vfs *pVfs, + const char *zPath, + sqlite3_file *pFile, + int flags, int *pFlagsOut ){ int rc; @@ -23723,18 +20299,18 @@ SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dir return pVfs->xDelete(pVfs, zPath, dirSync); } SQLITE_PRIVATE int sqlite3OsAccess( - sqlite3_vfs *pVfs, - const char *zPath, - int flags, + sqlite3_vfs *pVfs, + const char *zPath, + int flags, int *pResOut ){ DO_OS_MALLOC_TEST(0); return pVfs->xAccess(pVfs, zPath, flags, pResOut); } SQLITE_PRIVATE int sqlite3OsFullPathname( - sqlite3_vfs *pVfs, - const char *zPath, - int nPathOut, + sqlite3_vfs *pVfs, + const char *zPath, + int nPathOut, char *zPathOut ){ DO_OS_MALLOC_TEST(0); @@ -23761,9 +20337,6 @@ SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufO SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ return pVfs->xSleep(pVfs, nMicro); } -SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs *pVfs){ - return pVfs->xGetLastError ? pVfs->xGetLastError(pVfs, 0, 0) : 0; -} SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ int rc; /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() @@ -23783,13 +20356,13 @@ SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p } SQLITE_PRIVATE int sqlite3OsOpenMalloc( - sqlite3_vfs *pVfs, - const char *zFile, - sqlite3_file **ppFile, + sqlite3_vfs *pVfs, + const char *zFile, + sqlite3_file **ppFile, int flags, int *pOutFlags ){ - int rc; + int rc = SQLITE_NOMEM; sqlite3_file *pFile; pFile = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile); if( pFile ){ @@ -23799,15 +20372,15 @@ SQLITE_PRIVATE int sqlite3OsOpenMalloc( }else{ *ppFile = pFile; } - }else{ - rc = SQLITE_NOMEM_BKPT; } return rc; } -SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *pFile){ +SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *pFile){ + int rc = SQLITE_OK; assert( pFile ); - sqlite3OsClose(pFile); + rc = sqlite3OsClose(pFile); sqlite3_free(pFile); + return rc; } /* @@ -23818,7 +20391,7 @@ SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *pFile){ */ SQLITE_PRIVATE int sqlite3OsInit(void){ void *p = sqlite3_malloc(10); - if( p==0 ) return SQLITE_NOMEM_BKPT; + if( p==0 ) return SQLITE_NOMEM; sqlite3_free(p); return sqlite3_os_init(); } @@ -23833,7 +20406,7 @@ static sqlite3_vfs * SQLITE_WSD vfsList = 0; ** Locate a VFS by name. If no name is given, simply return the ** first VFS on the list. */ -SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfs){ +SQLITE_API sqlite3_vfs *SQLITE_STDCALL sqlite3_vfs_find(const char *zVfs){ sqlite3_vfs *pVfs = 0; #if SQLITE_THREADSAFE sqlite3_mutex *mutex; @@ -23879,7 +20452,7 @@ static void vfsUnlink(sqlite3_vfs *pVfs){ ** VFS multiple times. The new VFS becomes the default if makeDflt is ** true. */ -SQLITE_API int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ +SQLITE_API int SQLITE_STDCALL sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ MUTEX_LOGIC(sqlite3_mutex *mutex;) #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); @@ -23907,7 +20480,7 @@ SQLITE_API int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ /* ** Unregister a VFS so that it is no longer accessible. */ -SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ +SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); #endif @@ -23945,7 +20518,6 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ ** during a hash table resize is a benign fault. */ -/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_BUILTIN_TEST @@ -24027,7 +20599,6 @@ SQLITE_PRIVATE void sqlite3EndBenignMalloc(void){ ** are merely placeholders. Real drivers must be substituted using ** sqlite3_config() before SQLite will operate. */ -/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is the default. It is @@ -24114,7 +20685,6 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ ** be necessary when compiling for Delphi, ** for example. */ -/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is the default. It is @@ -24245,11 +20815,10 @@ static void sqlite3MemFree(void *pPrior){ */ static int sqlite3MemSize(void *pPrior){ #ifdef SQLITE_MALLOCSIZE - assert( pPrior!=0 ); - return (int)SQLITE_MALLOCSIZE(pPrior); + return pPrior ? (int)SQLITE_MALLOCSIZE(pPrior) : 0; #else sqlite3_int64 *p; - assert( pPrior!=0 ); + if( pPrior==0 ) return 0; p = (sqlite3_int64*)pPrior; p--; return (int)p[0]; @@ -24391,7 +20960,6 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ ** This file contains implementations of the low-level memory allocation ** routines specified in the sqlite3_mem_methods object. */ -/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is used only if the @@ -24926,7 +21494,6 @@ SQLITE_PRIVATE int sqlite3MemdebugMallocCount(){ ** This version of the memory allocation subsystem is included ** in the build only if SQLITE_ENABLE_MEMSYS3 is defined. */ -/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is only built into the library @@ -25379,7 +21946,7 @@ static void memsys3FreeUnsafe(void *pOld){ */ static int memsys3Size(void *p){ Mem3Block *pBlock; - assert( p!=0 ); + if( p==0 ) return 0; pBlock = (Mem3Block*)p; assert( (pBlock[-1].u.hdr.size4x&1)!=0 ); return (pBlock[-1].u.hdr.size4x&~3)*2 - 4; @@ -25618,7 +22185,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ ** ** This memory allocator uses the following algorithm: ** -** 1. All memory allocation sizes are rounded up to a power of 2. +** 1. All memory allocations sizes are rounded up to a power of 2. ** ** 2. If two adjacent free blocks are the halves of a larger block, ** then the two blocks are coalesced into the single larger block. @@ -25641,7 +22208,6 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ ** The sqlite3_status() logic tracks the maximum values of n and M so ** that an application can, at any time, verify this constraint. */ -/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is used only when @@ -25695,7 +22261,6 @@ static SQLITE_WSD struct Mem5Global { */ sqlite3_mutex *mutex; -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Performance statistics */ @@ -25707,12 +22272,11 @@ static SQLITE_WSD struct Mem5Global { u32 maxOut; /* Maximum instantaneous currentOut */ u32 maxCount; /* Maximum instantaneous currentCount */ u32 maxRequest; /* Largest allocation (exclusive of internal frag) */ -#endif /* ** Lists of free blocks. aiFreelist[0] is a list of free blocks of ** size mem5.szAtom. aiFreelist[1] holds blocks of size szAtom*2. - ** aiFreelist[2] holds free blocks of size szAtom*4. And so forth. + ** and so forth. */ int aiFreelist[LOGMAX+1]; @@ -25778,7 +22342,9 @@ static void memsys5Link(int i, int iLogsize){ } /* -** Obtain or release the mutex needed to access global data structures. +** If the STATIC_MEM mutex is not already held, obtain it now. The mutex +** will already be held (obtained by code in malloc.c) if +** sqlite3GlobalConfig.bMemStat is true. */ static void memsys5Enter(void){ sqlite3_mutex_enter(mem5.mutex); @@ -25788,15 +22354,17 @@ static void memsys5Leave(void){ } /* -** Return the size of an outstanding allocation, in bytes. -** This only works for chunks that are currently checked out. +** Return the size of an outstanding allocation, in bytes. The +** size returned omits the 8-byte header overhead. This only +** works for chunks that are currently checked out. */ static int memsys5Size(void *p){ - int iSize, i; - assert( p!=0 ); - i = (int)(((u8 *)p-mem5.zPool)/mem5.szAtom); - assert( i>=0 && i=0 && i0 ); - /* No more than 1GiB per allocation */ - if( nByte > 0x40000000 ) return 0; - -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* Keep track of the maximum allocation request. Even unfulfilled ** requests are counted */ if( (u32)nByte>mem5.maxRequest ){ mem5.maxRequest = nByte; } -#endif + /* Abort if the requested allocation size is larger than the largest + ** power of two that we can represent using 32-bit signed integers. + */ + if( nByte > 0x40000000 ){ + return 0; + } /* Round nByte up to the next valid power of two */ - for(iFullSz=mem5.szAtom,iLogsize=0; iFullSz0 ); assert( mem5.currentOut>=(size*mem5.szAtom) ); mem5.currentCount--; mem5.currentOut -= size*mem5.szAtom; assert( mem5.currentOut>0 || mem5.currentCount==0 ); assert( mem5.currentCount>0 || mem5.currentOut==0 ); -#endif mem5.aCtrl[iBlock] = CTRL_FREE | iLogsize; while( ALWAYS(iLogsize>iLogsize) & 1 ){ iBuddy = iBlock - size; - assert( iBuddy>=0 ); }else{ iBuddy = iBlock + size; - if( iBuddy>=mem5.nBlock ) break; } + assert( iBuddy>=0 ); + if( (iBuddy+(1<mem5.nBlock ) break; if( mem5.aCtrl[iBuddy]!=(CTRL_FREE | iLogsize) ) break; memsys5Unlink(iBuddy, iLogsize); iLogsize++; @@ -25994,11 +22558,13 @@ static void *memsys5Realloc(void *pPrior, int nBytes){ if( nBytes<=nOld ){ return pPrior; } - p = memsys5Malloc(nBytes); + memsys5Enter(); + p = memsys5MallocUnsafe(nBytes); if( p ){ memcpy(p, pPrior, nOld); - memsys5Free(pPrior); + memsys5FreeUnsafe(pPrior); } + memsys5Leave(); return p; } @@ -26185,7 +22751,6 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ ** ** This file contains code that is common across all mutex implementations. */ -/* #include "sqliteInt.h" */ #if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT) /* @@ -26194,7 +22759,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ ** allocate a mutex while the system is uninitialized. */ static SQLITE_WSD int mutexIsInit = 0; -#endif /* SQLITE_DEBUG && !defined(SQLITE_MUTEX_OMIT) */ +#endif /* SQLITE_DEBUG */ #ifndef SQLITE_MUTEX_OMIT @@ -26217,18 +22782,11 @@ SQLITE_PRIVATE int sqlite3MutexInit(void){ }else{ pFrom = sqlite3NoopMutex(); } - pTo->xMutexInit = pFrom->xMutexInit; - pTo->xMutexEnd = pFrom->xMutexEnd; - pTo->xMutexFree = pFrom->xMutexFree; - pTo->xMutexEnter = pFrom->xMutexEnter; - pTo->xMutexTry = pFrom->xMutexTry; - pTo->xMutexLeave = pFrom->xMutexLeave; - pTo->xMutexHeld = pFrom->xMutexHeld; - pTo->xMutexNotheld = pFrom->xMutexNotheld; - sqlite3MemoryBarrier(); + memcpy(pTo, pFrom, offsetof(sqlite3_mutex_methods, xMutexAlloc)); + memcpy(&pTo->xMutexFree, &pFrom->xMutexFree, + sizeof(*pTo) - offsetof(sqlite3_mutex_methods, xMutexFree)); pTo->xMutexAlloc = pFrom->xMutexAlloc; } - assert( sqlite3GlobalConfig.mutex.xMutexInit ); rc = sqlite3GlobalConfig.mutex.xMutexInit(); #ifdef SQLITE_DEBUG @@ -26258,12 +22816,11 @@ SQLITE_PRIVATE int sqlite3MutexEnd(void){ /* ** Retrieve a pointer to a static mutex or allocate a new dynamic one. */ -SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int id){ +SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_mutex_alloc(int id){ #ifndef SQLITE_OMIT_AUTOINIT if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; #endif - assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -26272,16 +22829,14 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int id){ return 0; } assert( GLOBAL(int, mutexIsInit) ); - assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } /* ** Free a dynamic mutex. */ -SQLITE_API void sqlite3_mutex_free(sqlite3_mutex *p){ +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex *p){ if( p ){ - assert( sqlite3GlobalConfig.mutex.xMutexFree ); sqlite3GlobalConfig.mutex.xMutexFree(p); } } @@ -26290,9 +22845,8 @@ SQLITE_API void sqlite3_mutex_free(sqlite3_mutex *p){ ** Obtain the mutex p. If some other thread already has the mutex, block ** until it can be obtained. */ -SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex *p){ +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex *p){ if( p ){ - assert( sqlite3GlobalConfig.mutex.xMutexEnter ); sqlite3GlobalConfig.mutex.xMutexEnter(p); } } @@ -26301,10 +22855,9 @@ SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex *p){ ** Obtain the mutex p. If successful, return SQLITE_OK. Otherwise, if another ** thread holds the mutex and it cannot be obtained, return SQLITE_BUSY. */ -SQLITE_API int sqlite3_mutex_try(sqlite3_mutex *p){ +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex *p){ int rc = SQLITE_OK; if( p ){ - assert( sqlite3GlobalConfig.mutex.xMutexTry ); return sqlite3GlobalConfig.mutex.xMutexTry(p); } return rc; @@ -26316,9 +22869,8 @@ SQLITE_API int sqlite3_mutex_try(sqlite3_mutex *p){ ** is not currently entered. If a NULL pointer is passed as an argument ** this function is a no-op. */ -SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex *p){ +SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex *p){ if( p ){ - assert( sqlite3GlobalConfig.mutex.xMutexLeave ); sqlite3GlobalConfig.mutex.xMutexLeave(p); } } @@ -26328,12 +22880,10 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex *p){ ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are ** intended for use inside assert() statements. */ -SQLITE_API int sqlite3_mutex_held(sqlite3_mutex *p){ - assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_held(sqlite3_mutex *p){ return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } -SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex *p){ - assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); +SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex *p){ return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } #endif @@ -26369,7 +22919,6 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex *p){ ** that does error checking on mutexes to make sure they are being ** called correctly. */ -/* #include "sqliteInt.h" */ #ifndef SQLITE_MUTEX_OMIT @@ -26451,7 +23000,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; } ** that means that a mutex could not be allocated. */ static sqlite3_mutex *debugMutexAlloc(int id){ - static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_VFS3 - 1]; + static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_APP3 - 1]; sqlite3_debug_mutex *pNew = 0; switch( id ){ case SQLITE_MUTEX_FAST: @@ -26573,7 +23122,6 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ************************************************************************* ** This file contains the C functions that implement mutexes for pthreads */ -/* #include "sqliteInt.h" */ /* ** The code in this file is only used if we are compiling threadsafe @@ -26612,9 +23160,7 @@ struct sqlite3_mutex { #endif }; #if SQLITE_MUTEX_NREF -#define SQLITE3_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER,0,0,(pthread_t)0,0} -#elif defined(SQLITE_ENABLE_API_ARMOR) -#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0 } +#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0, 0, (pthread_t)0, 0 } #else #define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER } #endif @@ -26644,19 +23190,6 @@ static int pthreadMutexNotheld(sqlite3_mutex *p){ } #endif -/* -** Try to provide a memory barrier operation, needed for initialization -** and also for the implementation of xShmBarrier in the VFS in cases -** where SQLite is compiled without mutexes. -*/ -SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ -#if defined(SQLITE_MEMORY_BARRIER) - SQLITE_MEMORY_BARRIER; -#elif defined(__GNUC__) && GCC_VERSION>=4001000 - __sync_synchronize(); -#endif -} - /* ** Initialize and deinitialize the mutex subsystem. */ @@ -26682,9 +23215,6 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 **
  • SQLITE_MUTEX_STATIC_APP3 -**
  • SQLITE_MUTEX_STATIC_VFS1 -**
  • SQLITE_MUTEX_STATIC_VFS2 -**
  • SQLITE_MUTEX_STATIC_VFS3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -26721,9 +23251,6 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; sqlite3_mutex *p; @@ -26963,7 +23490,6 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ************************************************************************* ** This file contains the C functions that implement mutexes for Win32. */ -/* #include "sqliteInt.h" */ #if SQLITE_OS_WIN /* @@ -27008,8 +23534,8 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing +/* +** hwtime.h contains inline assembler code for implementing ** high-performance timing routines. */ /************** Include hwtime.h in the middle of os_common.h ****************/ @@ -27029,8 +23555,8 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** This file contains inline asm code for retrieving "high-performance" ** counters for x86 class CPUs. */ -#ifndef SQLITE_HWTIME_H -#define SQLITE_HWTIME_H +#ifndef _HWTIME_H_ +#define _HWTIME_H_ /* ** The following routine only works on pentium-class (or newer) processors. @@ -27098,7 +23624,7 @@ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } #endif -#endif /* !defined(SQLITE_HWTIME_H) */ +#endif /* !defined(_HWTIME_H_) */ /************** End of hwtime.h **********************************************/ /************** Continuing where we left off in os_common.h ******************/ @@ -27119,14 +23645,14 @@ static sqlite_uint64 g_elapsed; ** of code will give us the ability to simulate a disk I/O error. This ** is used for testing the I/O recovery logic. */ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_io_error_hit; -SQLITE_API extern int sqlite3_io_error_hardhit; -SQLITE_API extern int sqlite3_io_error_pending; -SQLITE_API extern int sqlite3_io_error_persist; -SQLITE_API extern int sqlite3_io_error_benign; -SQLITE_API extern int sqlite3_diskfull_pending; -SQLITE_API extern int sqlite3_diskfull; +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ +SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ +SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ +SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ +SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ +SQLITE_API int sqlite3_diskfull_pending = 0; +SQLITE_API int sqlite3_diskfull = 0; #define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) #define SimulateIOError(CODE) \ if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ @@ -27152,17 +23678,17 @@ static void local_ioerr(){ #define SimulateIOErrorBenign(X) #define SimulateIOError(A) #define SimulateDiskfullError(A) -#endif /* defined(SQLITE_TEST) */ +#endif /* ** When testing, keep a count of the number of open files. */ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_open_file_count; +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_open_file_count = 0; #define OpenCounter(X) sqlite3_open_file_count+=(X) #else #define OpenCounter(X) -#endif /* defined(SQLITE_TEST) */ +#endif #endif /* !defined(_OS_COMMON_H_) */ @@ -27188,8 +23714,8 @@ SQLITE_API extern int sqlite3_open_file_count; ** ** This file contains code that is specific to Windows. */ -#ifndef SQLITE_OS_WIN_H -#define SQLITE_OS_WIN_H +#ifndef _OS_WIN_H_ +#define _OS_WIN_H_ /* ** Include the primary Windows SDK header file. @@ -27261,7 +23787,7 @@ SQLITE_API extern int sqlite3_open_file_count; # define SQLITE_OS_WIN_THREADS 0 #endif -#endif /* SQLITE_OS_WIN_H */ +#endif /* _OS_WIN_H_ */ /************** End of os_win.h **********************************************/ /************** Continuing where we left off in mutex_w32.c ******************/ @@ -27319,24 +23845,6 @@ static int winMutexNotheld(sqlite3_mutex *p){ } #endif -/* -** Try to provide a memory barrier operation, needed for initialization -** and also for the xShmBarrier method of the VFS in cases when SQLite is -** compiled without mutexes (SQLITE_THREADSAFE=0). -*/ -SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ -#if defined(SQLITE_MEMORY_BARRIER) - SQLITE_MEMORY_BARRIER; -#elif defined(__GNUC__) - __sync_synchronize(); -#elif !defined(SQLITE_DISABLE_INTRINSIC) && \ - defined(_MSC_VER) && _MSC_VER>=1300 - _ReadWriteBarrier(); -#elif defined(MemoryBarrier) - MemoryBarrier(); -#endif -} - /* ** Initialize and deinitialize the mutex subsystem. */ @@ -27349,9 +23857,6 @@ static sqlite3_mutex winMutex_staticMutexes[] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; @@ -27364,8 +23869,8 @@ static int winMutex_isNt = -1; /* <0 means "need to query" */ */ static LONG SQLITE_WIN32_VOLATILE winMutex_lock = 0; -SQLITE_API int sqlite3_win32_is_nt(void); /* os_win.c */ -SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ +SQLITE_API int SQLITE_STDCALL sqlite3_win32_is_nt(void); /* os_win.c */ +SQLITE_API void SQLITE_STDCALL sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ static int winMutexInit(void){ /* The first to increment to 1 does actual initialization */ @@ -27423,9 +23928,6 @@ static int winMutexEnd(void){ **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 **
  • SQLITE_MUTEX_STATIC_APP3 -**
  • SQLITE_MUTEX_STATIC_VFS1 -**
  • SQLITE_MUTEX_STATIC_VFS2 -**
  • SQLITE_MUTEX_STATIC_VFS3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -27657,7 +24159,6 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** ** Memory allocation functions used throughout sqlite. */ -/* #include "sqliteInt.h" */ /* #include */ /* @@ -27665,7 +24166,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** held by SQLite. An example of non-essential memory is memory used to ** cache database pages that are not currently in use. */ -SQLITE_API int sqlite3_release_memory(int n){ +SQLITE_API int SQLITE_STDCALL sqlite3_release_memory(int n){ #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT return sqlite3PcacheReleaseMemory(n); #else @@ -27690,7 +24191,16 @@ typedef struct ScratchFreeslot { */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ - sqlite3_int64 alarmThreshold; /* The soft heap limit */ + + /* + ** The alarm callback and its arguments. The mem0.mutex lock will + ** be held while the callback is running. Recursive calls into + ** the memory subsystem are allowed, but no new callbacks will be + ** issued. + */ + sqlite3_int64 alarmThreshold; + void (*alarmCallback)(void*, sqlite3_int64,int); + void *alarmArg; /* ** Pointers to the end of sqlite3GlobalConfig.pScratch memory @@ -27707,7 +24217,7 @@ static SQLITE_WSD struct Mem0Global { ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; -} mem0 = { 0, 0, 0, 0, 0, 0 }; +} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; #define mem0 GLOBAL(struct Mem0Global, mem0) @@ -27718,51 +24228,78 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void){ return mem0.mutex; } -#ifndef SQLITE_OMIT_DEPRECATED /* -** Deprecated external interface. It used to set an alarm callback -** that was invoked when memory usage grew too large. Now it is a -** no-op. +** This routine runs when the memory allocator sees that the +** total memory allocation is about to exceed the soft heap +** limit. */ -SQLITE_API int sqlite3_memory_alarm( +static void softHeapLimitEnforcer( + void *NotUsed, + sqlite3_int64 NotUsed2, + int allocSize +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + sqlite3_release_memory(allocSize); +} + +/* +** Change the alarm callback +*/ +static int sqlite3MemoryAlarm( void(*xCallback)(void *pArg, sqlite3_int64 used,int N), void *pArg, sqlite3_int64 iThreshold ){ - (void)xCallback; - (void)pArg; - (void)iThreshold; + sqlite3_int64 nUsed; + sqlite3_mutex_enter(mem0.mutex); + mem0.alarmCallback = xCallback; + mem0.alarmArg = pArg; + mem0.alarmThreshold = iThreshold; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + mem0.nearlyFull = (iThreshold>0 && iThreshold<=nUsed); + sqlite3_mutex_leave(mem0.mutex); return SQLITE_OK; } + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Deprecated external interface. Internal/core SQLite code +** should call sqlite3MemoryAlarm. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_memory_alarm( + void(*xCallback)(void *pArg, sqlite3_int64 used,int N), + void *pArg, + sqlite3_int64 iThreshold +){ + return sqlite3MemoryAlarm(xCallback, pArg, iThreshold); +} #endif /* ** Set the soft heap-size limit for the library. Passing a zero or ** negative value indicates no limit. */ -SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_soft_heap_limit64(sqlite3_int64 n){ sqlite3_int64 priorLimit; sqlite3_int64 excess; - sqlite3_int64 nUsed; #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); if( rc ) return -1; #endif sqlite3_mutex_enter(mem0.mutex); priorLimit = mem0.alarmThreshold; - if( n<0 ){ - sqlite3_mutex_leave(mem0.mutex); - return priorLimit; - } - mem0.alarmThreshold = n; - nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - mem0.nearlyFull = (n>0 && n<=nUsed); sqlite3_mutex_leave(mem0.mutex); + if( n<0 ) return priorLimit; + if( n>0 ){ + sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, n); + }else{ + sqlite3MemoryAlarm(0, 0, 0); + } excess = sqlite3_memory_used() - n; if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); return priorLimit; } -SQLITE_API void sqlite3_soft_heap_limit(int n){ +SQLITE_API void SQLITE_STDCALL sqlite3_soft_heap_limit(int n){ if( n<0 ) n = 0; sqlite3_soft_heap_limit64(n); } @@ -27776,7 +24313,9 @@ SQLITE_PRIVATE int sqlite3MallocInit(void){ sqlite3MemSetDefault(); } memset(&mem0, 0, sizeof(mem0)); - mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + if( sqlite3GlobalConfig.bCoreMutex ){ + mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100 && sqlite3GlobalConfig.nScratch>0 ){ int i, n, sz; @@ -27800,9 +24339,10 @@ SQLITE_PRIVATE int sqlite3MallocInit(void){ sqlite3GlobalConfig.nScratch = 0; } if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512 - || sqlite3GlobalConfig.nPage<=0 ){ + || sqlite3GlobalConfig.nPage<1 ){ sqlite3GlobalConfig.pPage = 0; sqlite3GlobalConfig.szPage = 0; + sqlite3GlobalConfig.nPage = 0; } rc = sqlite3GlobalConfig.m.xInit(sqlite3GlobalConfig.m.pAppData); if( rc!=SQLITE_OK ) memset(&mem0, 0, sizeof(mem0)); @@ -27831,9 +24371,11 @@ SQLITE_PRIVATE void sqlite3MallocEnd(void){ /* ** Return the amount of memory currently checked out. */ -SQLITE_API sqlite3_int64 sqlite3_memory_used(void){ - sqlite3_int64 res, mx; - sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, 0); +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_used(void){ + int n, mx; + sqlite3_int64 res; + sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, 0); + res = (sqlite3_int64)n; /* Work around bug in Borland C. Ticket #3216 */ return res; } @@ -27842,20 +24384,31 @@ SQLITE_API sqlite3_int64 sqlite3_memory_used(void){ ** checked out since either the beginning of this process ** or since the most recent reset. */ -SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag){ - sqlite3_int64 res, mx; - sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, resetFlag); - return mx; +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_highwater(int resetFlag){ + int n, mx; + sqlite3_int64 res; + sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, resetFlag); + res = (sqlite3_int64)mx; /* Work around bug in Borland C. Ticket #3216 */ + return res; } /* ** Trigger the alarm */ static void sqlite3MallocAlarm(int nByte){ - if( mem0.alarmThreshold<=0 ) return; + void (*xCallback)(void*,sqlite3_int64,int); + sqlite3_int64 nowUsed; + void *pArg; + if( mem0.alarmCallback==0 ) return; + xCallback = mem0.alarmCallback; + nowUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + pArg = mem0.alarmArg; + mem0.alarmCallback = 0; sqlite3_mutex_leave(mem0.mutex); - sqlite3_release_memory(nByte); + xCallback(pArg, nowUsed, nByte); sqlite3_mutex_enter(mem0.mutex); + mem0.alarmCallback = xCallback; + mem0.alarmArg = pArg; } /* @@ -27867,8 +24420,8 @@ static int mallocWithAlarm(int n, void **pp){ void *p; assert( sqlite3_mutex_held(mem0.mutex) ); nFull = sqlite3GlobalConfig.m.xRoundup(n); - sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); - if( mem0.alarmThreshold>0 ){ + sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); + if( mem0.alarmCallback!=0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; @@ -27879,7 +24432,7 @@ static int mallocWithAlarm(int n, void **pp){ } p = sqlite3GlobalConfig.m.xMalloc(nFull); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( p==0 && mem0.alarmThreshold>0 ){ + if( p==0 && mem0.alarmCallback ){ sqlite3MallocAlarm(nFull); p = sqlite3GlobalConfig.m.xMalloc(nFull); } @@ -27922,13 +24475,13 @@ SQLITE_PRIVATE void *sqlite3Malloc(u64 n){ ** First make sure the memory subsystem is initialized, then do the ** allocation. */ -SQLITE_API void *sqlite3_malloc(int n){ +SQLITE_API void *SQLITE_STDCALL sqlite3_malloc(int n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif return n<=0 ? 0 : sqlite3Malloc(n); } -SQLITE_API void *sqlite3_malloc64(sqlite3_uint64 n){ +SQLITE_API void *SQLITE_STDCALL sqlite3_malloc64(sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -27959,7 +24512,7 @@ SQLITE_PRIVATE void *sqlite3ScratchMalloc(int n){ assert( n>0 ); sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusHighwater(SQLITE_STATUS_SCRATCH_SIZE, n); + sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ p = mem0.pScratchFree; mem0.pScratchFree = mem0.pScratchFree->pNext; @@ -28003,7 +24556,7 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void *p){ scratchAllocOut--; #endif - if( SQLITE_WITHIN(p, sqlite3GlobalConfig.pScratch, mem0.pScratchEnd) ){ + if( p>=sqlite3GlobalConfig.pScratch && plookaside.pStart, db->lookaside.pEnd); + return p>=db->lookaside.pStart && plookaside.pEnd; } #else #define isLookaside(A,B) 0 @@ -28054,33 +24607,31 @@ SQLITE_PRIVATE int sqlite3MallocSize(void *p){ return sqlite3GlobalConfig.m.xSize(p); } SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ - assert( p!=0 ); - if( db==0 || !isLookaside(db,p) ){ -#if SQLITE_DEBUG - if( db==0 ){ - assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + return sqlite3MallocSize(p); + }else{ + assert( sqlite3_mutex_held(db->mutex) ); + if( isLookaside(db, p) ){ + return db->lookaside.sz; }else{ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + return sqlite3GlobalConfig.m.xSize(p); } -#endif - return sqlite3GlobalConfig.m.xSize(p); - }else{ - assert( sqlite3_mutex_held(db->mutex) ); - return db->lookaside.sz; } } -SQLITE_API sqlite3_uint64 sqlite3_msize(void *p){ +SQLITE_API sqlite3_uint64 SQLITE_STDCALL sqlite3_msize(void *p){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - return p ? sqlite3GlobalConfig.m.xSize(p) : 0; + return (sqlite3_uint64)sqlite3GlobalConfig.m.xSize(p); } /* ** Free memory previously obtained from sqlite3Malloc(). */ -SQLITE_API void sqlite3_free(void *p){ +SQLITE_API void SQLITE_STDCALL sqlite3_free(void *p){ if( p==0 ) return; /* IMP: R-49053-54554 */ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); @@ -28162,14 +24713,14 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ pNew = pOld; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); + sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); nDiff = nNew - nOld; if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= mem0.alarmThreshold-nDiff ){ sqlite3MallocAlarm(nDiff); } pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); - if( pNew==0 && mem0.alarmThreshold>0 ){ + if( pNew==0 && mem0.alarmCallback ){ sqlite3MallocAlarm((int)nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } @@ -28189,14 +24740,14 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ ** The public interface to sqlite3Realloc. Make sure that the memory ** subsystem is initialized prior to invoking sqliteRealloc. */ -SQLITE_API void *sqlite3_realloc(void *pOld, int n){ +SQLITE_API void *SQLITE_STDCALL sqlite3_realloc(void *pOld, int n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif if( n<0 ) n = 0; /* IMP: R-26507-47431 */ return sqlite3Realloc(pOld, n); } -SQLITE_API void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ +SQLITE_API void *SQLITE_STDCALL sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -28220,31 +24771,16 @@ SQLITE_PRIVATE void *sqlite3MallocZero(u64 n){ ** the mallocFailed flag in the connection pointer. */ SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3 *db, u64 n){ - void *p; - testcase( db==0 ); - p = sqlite3DbMallocRaw(db, n); - if( p ) memset(p, 0, (size_t)n); - return p; -} - - -/* Finish the work of sqlite3DbMallocRawNN for the unusual and -** slower case when the allocation cannot be fulfilled using lookaside. -*/ -static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){ - void *p; - assert( db!=0 ); - p = sqlite3Malloc(n); - if( !p ) sqlite3OomFault(db); - sqlite3MemdebugSetType(p, - (db->lookaside.bDisable==0) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); + void *p = sqlite3DbMallocRaw(db, n); + if( p ){ + memset(p, 0, (size_t)n); + } return p; } /* -** Allocate memory, either lookaside (if possible) or heap. -** If the allocation fails, set the mallocFailed flag in -** the connection pointer. +** Allocate and zero memory. If the allocation fails, make +** the mallocFailed flag in the connection pointer. ** ** If db!=0 and db->mallocFailed is true (indicating a prior malloc ** failure on the same database connection) then always return 0. @@ -28259,73 +24795,64 @@ static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){ ** ** In other words, if a subsequent malloc (ex: "b") worked, it is assumed ** that all prior mallocs (ex: "a") worked too. -** -** The sqlite3MallocRawNN() variant guarantees that the "db" parameter is -** not a NULL pointer. */ SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){ void *p; - if( db ) return sqlite3DbMallocRawNN(db, n); - p = sqlite3Malloc(n); - sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - return p; -} -SQLITE_PRIVATE void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ + assert( db==0 || sqlite3_mutex_held(db->mutex) ); + assert( db==0 || db->pnBytesFreed==0 ); #ifndef SQLITE_OMIT_LOOKASIDE - LookasideSlot *pBuf; - assert( db!=0 ); - assert( sqlite3_mutex_held(db->mutex) ); - assert( db->pnBytesFreed==0 ); - if( db->lookaside.bDisable==0 ){ - assert( db->mallocFailed==0 ); - if( n>db->lookaside.sz ){ - db->lookaside.anStat[1]++; - }else if( (pBuf = db->lookaside.pFree)==0 ){ - db->lookaside.anStat[2]++; - }else{ - db->lookaside.pFree = pBuf->pNext; - db->lookaside.nOut++; - db->lookaside.anStat[0]++; - if( db->lookaside.nOut>db->lookaside.mxOut ){ - db->lookaside.mxOut = db->lookaside.nOut; + if( db ){ + LookasideSlot *pBuf; + if( db->mallocFailed ){ + return 0; + } + if( db->lookaside.bEnabled ){ + if( n>db->lookaside.sz ){ + db->lookaside.anStat[1]++; + }else if( (pBuf = db->lookaside.pFree)==0 ){ + db->lookaside.anStat[2]++; + }else{ + db->lookaside.pFree = pBuf->pNext; + db->lookaside.nOut++; + db->lookaside.anStat[0]++; + if( db->lookaside.nOut>db->lookaside.mxOut ){ + db->lookaside.mxOut = db->lookaside.nOut; + } + return (void*)pBuf; } - return (void*)pBuf; } - }else if( db->mallocFailed ){ - return 0; } #else - assert( db!=0 ); - assert( sqlite3_mutex_held(db->mutex) ); - assert( db->pnBytesFreed==0 ); - if( db->mallocFailed ){ + if( db && db->mallocFailed ){ return 0; } #endif - return dbMallocRawFinish(db, n); + p = sqlite3Malloc(n); + if( !p && db ){ + db->mallocFailed = 1; + } + sqlite3MemdebugSetType(p, + (db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); + return p; } -/* Forward declaration */ -static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n); - /* ** Resize the block of memory pointed to by p to n bytes. If the ** resize fails, set the mallocFailed flag in the connection object. */ SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ - assert( db!=0 ); - if( p==0 ) return sqlite3DbMallocRawNN(db, n); - assert( sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db,p) && n<=db->lookaside.sz ) return p; - return dbReallocFinish(db, p, n); -} -static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ void *pNew = 0; assert( db!=0 ); - assert( p!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); if( db->mallocFailed==0 ){ + if( p==0 ){ + return sqlite3DbMallocRaw(db, n); + } if( isLookaside(db, p) ){ - pNew = sqlite3DbMallocRawNN(db, n); + if( n<=db->lookaside.sz ){ + return p; + } + pNew = sqlite3DbMallocRaw(db, n); if( pNew ){ memcpy(pNew, p, db->lookaside.sz); sqlite3DbFree(db, p); @@ -28336,10 +24863,10 @@ static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ sqlite3MemdebugSetType(p, MEMTYPE_HEAP); pNew = sqlite3_realloc64(p, n); if( !pNew ){ - sqlite3OomFault(db); + db->mallocFailed = 1; } sqlite3MemdebugSetType(pNew, - (db->lookaside.bDisable==0 ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); + (db->lookaside.bEnabled ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); } } return pNew; @@ -28381,12 +24908,11 @@ SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3 *db, const char *z){ } SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ char *zNew; - assert( db!=0 ); if( z==0 ){ return 0; } assert( (n&0x7fffffff)==n ); - zNew = sqlite3DbMallocRawNN(db, n+1); + zNew = sqlite3DbMallocRaw(db, n+1); if( zNew ){ memcpy(zNew, z, (size_t)n); zNew[n] = 0; @@ -28395,52 +24921,28 @@ SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ } /* -** Free any prior content in *pz and replace it with a copy of zNew. -*/ -SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){ - sqlite3DbFree(db, *pz); - *pz = sqlite3DbStrDup(db, zNew); -} - -/* -** Call this routine to record the fact that an OOM (out-of-memory) error -** has happened. This routine will set db->mallocFailed, and also -** temporarily disable the lookaside memory allocator and interrupt -** any running VDBEs. +** Create a string from the zFromat argument and the va_list that follows. +** Store the string in memory obtained from sqliteMalloc() and make *pz +** point to that string. */ -SQLITE_PRIVATE void sqlite3OomFault(sqlite3 *db){ - if( db->mallocFailed==0 && db->bBenignMalloc==0 ){ - db->mallocFailed = 1; - if( db->nVdbeExec>0 ){ - db->u1.isInterrupted = 1; - } - db->lookaside.bDisable++; - } -} +SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zFormat, ...){ + va_list ap; + char *z; -/* -** This routine reactivates the memory allocator and clears the -** db->mallocFailed flag as necessary. -** -** The memory allocator is not restarted if there are running -** VDBEs. -*/ -SQLITE_PRIVATE void sqlite3OomClear(sqlite3 *db){ - if( db->mallocFailed && db->nVdbeExec==0 ){ - db->mallocFailed = 0; - db->u1.isInterrupted = 0; - assert( db->lookaside.bDisable>0 ); - db->lookaside.bDisable--; - } + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + sqlite3DbFree(db, *pz); + *pz = z; } /* ** Take actions at the end of an API call to indicate an OOM error */ static SQLITE_NOINLINE int apiOomError(sqlite3 *db){ - sqlite3OomClear(db); + db->mallocFailed = 0; sqlite3Error(db, SQLITE_NOMEM); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } /* @@ -28452,16 +24954,17 @@ static SQLITE_NOINLINE int apiOomError(sqlite3 *db){ ** function. However, if a malloc() failure has occurred since the previous ** invocation SQLITE_NOMEM is returned instead. ** -** If an OOM as occurred, then the connection error-code (the value -** returned by sqlite3_errcode()) is set to SQLITE_NOMEM. +** If the first argument, db, is not NULL and a malloc() error has occurred, +** then the connection error-code (the value returned by sqlite3_errcode()) +** is set to SQLITE_NOMEM. */ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ - /* If the db handle must hold the connection handle mutex here. - ** Otherwise the read (and possible write) of db->mallocFailed + /* If the db handle is not NULL, then we must hold the connection handle + ** mutex here. Otherwise the read (and possible write) of db->mallocFailed ** is unsafe, as is the call to sqlite3Error(). */ - assert( db!=0 ); - assert( sqlite3_mutex_held(db->mutex) ); + assert( !db || sqlite3_mutex_held(db->mutex) ); + if( db==0 ) return rc & 0xff; if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){ return apiOomError(db); } @@ -28472,41 +24975,43 @@ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ /************** Begin file printf.c ******************************************/ /* ** The "printf" code that follows dates from the 1980's. It is in -** the public domain. +** the public domain. The original comments are included here for +** completeness. They are very out-of-date but might be useful as +** an historical reference. Most of the "enhancements" have been backed +** out so that the functionality is now the same as standard printf(). ** ************************************************************************** ** ** This file contains code for a set of "printf"-like routines. These ** routines format strings much like the printf() from the standard C ** library, though the implementation here has enhancements to support -** SQLite. +** SQLlite. */ -/* #include "sqliteInt.h" */ /* ** Conversion types fall into various categories as defined by the ** following enumeration. */ -#define etRADIX 0 /* Integer types. %d, %x, %o, and so forth */ -#define etFLOAT 1 /* Floating point. %f */ -#define etEXP 2 /* Exponentional notation. %e and %E */ -#define etGENERIC 3 /* Floating or exponential, depending on exponent. %g */ -#define etSIZE 4 /* Return number of characters processed so far. %n */ -#define etSTRING 5 /* Strings. %s */ -#define etDYNSTRING 6 /* Dynamically allocated strings. %z */ -#define etPERCENT 7 /* Percent symbol. %% */ -#define etCHARX 8 /* Characters. %c */ +#define etRADIX 1 /* Integer types. %d, %x, %o, and so forth */ +#define etFLOAT 2 /* Floating point. %f */ +#define etEXP 3 /* Exponentional notation. %e and %E */ +#define etGENERIC 4 /* Floating or exponential, depending on exponent. %g */ +#define etSIZE 5 /* Return number of characters processed so far. %n */ +#define etSTRING 6 /* Strings. %s */ +#define etDYNSTRING 7 /* Dynamically allocated strings. %z */ +#define etPERCENT 8 /* Percent symbol. %% */ +#define etCHARX 9 /* Characters. %c */ /* The rest are extensions, not normally found in printf() */ -#define etSQLESCAPE 9 /* Strings with '\'' doubled. %q */ -#define etSQLESCAPE2 10 /* Strings with '\'' doubled and enclosed in '', +#define etSQLESCAPE 10 /* Strings with '\'' doubled. %q */ +#define etSQLESCAPE2 11 /* Strings with '\'' doubled and enclosed in '', NULL pointers replaced by SQL NULL. %Q */ -#define etTOKEN 11 /* a pointer to a Token structure */ -#define etSRCLIST 12 /* a pointer to a SrcList */ -#define etPOINTER 13 /* The %p conversion */ -#define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ -#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etTOKEN 12 /* a pointer to a Token structure */ +#define etSRCLIST 13 /* a pointer to a SrcList */ +#define etPOINTER 14 /* The %p conversion */ +#define etSQLESCAPE3 15 /* %w -> Strings with '\"' doubled */ +#define etORDINAL 16 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ -#define etINVALID 16 /* Any unrecognized conversion type */ +#define etINVALID 0 /* Any unrecognized conversion type */ /* @@ -28643,6 +25148,7 @@ static char *getTextArg(PrintfArguments *p){ */ SQLITE_PRIVATE void sqlite3VXPrintf( StrAccum *pAccum, /* Accumulate results here */ + u32 bFlags, /* SQLITE_PRINTF_* flags */ const char *fmt, /* Format string */ va_list ap /* arguments */ ){ @@ -28661,7 +25167,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( etByte flag_long; /* True if "l" flag is present */ etByte flag_longlong; /* True if the "ll" flag is present */ etByte done; /* Loop termination flag */ - etByte xtype = etINVALID; /* Conversion paradigm */ + etByte xtype = 0; /* Conversion paradigm */ u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ u8 useIntern; /* Ok to use internal conversions (ex: %T) */ char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ @@ -28682,11 +25188,11 @@ SQLITE_PRIVATE void sqlite3VXPrintf( char buf[etBUFSIZE]; /* Conversion buffer */ bufpt = 0; - if( pAccum->printfFlags ){ - if( (bArgList = (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ + if( bFlags ){ + if( (bArgList = (bFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ pArgList = va_arg(ap, PrintfArguments*); } - useIntern = pAccum->printfFlags & SQLITE_PRINTF_INTERNAL; + useIntern = bFlags & SQLITE_PRINTF_INTERNAL; }else{ bArgList = useIntern = 0; } @@ -28741,12 +25247,6 @@ SQLITE_PRIVATE void sqlite3VXPrintf( testcase( wx>0x7fffffff ); width = wx & 0x7fffffff; } - assert( width>=0 ); -#ifdef SQLITE_PRINTF_PRECISION_LIMIT - if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ - width = SQLITE_PRINTF_PRECISION_LIMIT; - } -#endif /* Get the precision */ if( c=='.' ){ @@ -28773,14 +25273,6 @@ SQLITE_PRIVATE void sqlite3VXPrintf( }else{ precision = -1; } - assert( precision>=(-1) ); -#ifdef SQLITE_PRINTF_PRECISION_LIMIT - if( precision>SQLITE_PRINTF_PRECISION_LIMIT ){ - precision = SQLITE_PRINTF_PRECISION_LIMIT; - } -#endif - - /* Get the conversion type modifier */ if( c=='l' ){ flag_long = 1; @@ -28953,16 +25445,21 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( realvalue>0.0 ){ LONGDOUBLE_TYPE scale = 1.0; while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} - while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } + while( realvalue>=1e64*scale && exp<=350 ){ scale *= 1e64; exp+=64; } + while( realvalue>=1e8*scale && exp<=350 ){ scale *= 1e8; exp+=8; } while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } realvalue /= scale; while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } if( exp>350 ){ - bufpt = buf; - buf[0] = prefix; - memcpy(buf+(prefix!=0),"Inf",4); - length = 3+(prefix!=0); + if( prefix=='-' ){ + bufpt = "-Inf"; + }else if( prefix=='+' ){ + bufpt = "+Inf"; + }else{ + bufpt = "Inf"; + } + length = sqlite3Strlen30(bufpt); break; } } @@ -29111,13 +25608,12 @@ SQLITE_PRIVATE void sqlite3VXPrintf( case etDYNSTRING: if( bArgList ){ bufpt = getTextArg(pArgList); - xtype = etSTRING; }else{ bufpt = va_arg(ap,char*); } if( bufpt==0 ){ bufpt = ""; - }else if( xtype==etDYNSTRING ){ + }else if( xtype==etDYNSTRING && !bArgList ){ zExtra = bufpt; } if( precision>=0 ){ @@ -29126,9 +25622,9 @@ SQLITE_PRIVATE void sqlite3VXPrintf( length = sqlite3Strlen30(bufpt); } break; - case etSQLESCAPE: /* Escape ' characters */ - case etSQLESCAPE2: /* Escape ' and enclose in '...' */ - case etSQLESCAPE3: { /* Escape " characters */ + case etSQLESCAPE: + case etSQLESCAPE2: + case etSQLESCAPE3: { int i, j, k, n, isnull; int needQuote; char ch; @@ -29147,7 +25643,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( ch==q ) n++; } needQuote = !isnull && xtype==etSQLESCAPE2; - n += i + 3; + n += i + 1 + needQuote*2; if( n>etBUFSIZE ){ bufpt = zExtra = sqlite3Malloc( n ); if( bufpt==0 ){ @@ -29211,7 +25707,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); if( zExtra ){ - sqlite3DbFree(pAccum->db, zExtra); + sqlite3_free(zExtra); zExtra = 0; } }/* End for loop over the format string */ @@ -29237,9 +25733,8 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ setStrAccumError(p, STRACCUM_TOOBIG); return N; }else{ - char *zOld = isMalloced(p) ? p->zText : 0; + char *zOld = (p->zText==p->zBase ? 0 : p->zText); i64 szNew = p->nChar; - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); szNew += N + 1; if( szNew+p->nChar<=p->mxAlloc ){ /* Force exponential buffer size growth as long as it does not overflow, @@ -29260,10 +25755,9 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ } if( zNew ){ assert( p->zText!=0 || p->nChar==0 ); - if( !isMalloced(p) && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); + if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); p->zText = zNew; p->nAlloc = sqlite3DbMallocSize(p->db, zNew); - p->printfFlags |= SQLITE_PRINTF_MALLOCED; }else{ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_NOMEM); @@ -29281,7 +25775,6 @@ SQLITE_PRIVATE void sqlite3AppendChar(StrAccum *p, int N, char c){ if( p->nChar+(i64)N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ){ return; } - assert( (p->zText==p->zBase)==!isMalloced(p) ); while( (N--)>0 ) p->zText[p->nChar++] = c; } @@ -29299,7 +25792,6 @@ static void SQLITE_NOINLINE enlargeAndAppend(StrAccum *p, const char *z, int N){ memcpy(&p->zText[p->nChar], z, N); p->nChar += N; } - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); } /* @@ -29335,13 +25827,11 @@ SQLITE_PRIVATE void sqlite3StrAccumAppendAll(StrAccum *p, const char *z){ */ SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ - assert( (p->zText==p->zBase)==!isMalloced(p) ); p->zText[p->nChar] = 0; - if( p->mxAlloc>0 && !isMalloced(p) ){ + if( p->mxAlloc>0 && p->zText==p->zBase ){ p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); if( p->zText ){ memcpy(p->zText, p->zBase, p->nChar+1); - p->printfFlags |= SQLITE_PRINTF_MALLOCED; }else{ setStrAccumError(p, STRACCUM_NOMEM); } @@ -29354,10 +25844,8 @@ SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ ** Reset an StrAccum string. Reclaim all malloced memory. */ SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum *p){ - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); - if( isMalloced(p) ){ + if( p->zText!=p->zBase ){ sqlite3DbFree(p->db, p->zText); - p->printfFlags &= ~SQLITE_PRINTF_MALLOCED; } p->zText = 0; } @@ -29383,7 +25871,6 @@ SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum *p, sqlite3 *db, char *zBase, i p->nAlloc = n; p->mxAlloc = mx; p->accError = 0; - p->printfFlags = 0; } /* @@ -29397,11 +25884,10 @@ SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list a assert( db!=0 ); sqlite3StrAccumInit(&acc, db, zBase, sizeof(zBase), db->aLimit[SQLITE_LIMIT_LENGTH]); - acc.printfFlags = SQLITE_PRINTF_INTERNAL; - sqlite3VXPrintf(&acc, zFormat, ap); + sqlite3VXPrintf(&acc, SQLITE_PRINTF_INTERNAL, zFormat, ap); z = sqlite3StrAccumFinish(&acc); if( acc.accError==STRACCUM_NOMEM ){ - sqlite3OomFault(db); + db->mallocFailed = 1; } return z; } @@ -29419,11 +25905,29 @@ SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){ return z; } +/* +** Like sqlite3MPrintf(), but call sqlite3DbFree() on zStr after formatting +** the string and before returning. This routine is intended to be used +** to modify an existing string. For example: +** +** x = sqlite3MPrintf(db, x, "prefix %s suffix", x); +** +*/ +SQLITE_PRIVATE char *sqlite3MAppendf(sqlite3 *db, char *zStr, const char *zFormat, ...){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + sqlite3DbFree(db, zStr); + return z; +} + /* ** Print into memory obtained from sqlite3_malloc(). Omit the internal ** %-conversion extensions. */ -SQLITE_API char *sqlite3_vmprintf(const char *zFormat, va_list ap){ +SQLITE_API char *SQLITE_STDCALL sqlite3_vmprintf(const char *zFormat, va_list ap){ char *z; char zBase[SQLITE_PRINT_BUF_SIZE]; StrAccum acc; @@ -29438,7 +25942,7 @@ SQLITE_API char *sqlite3_vmprintf(const char *zFormat, va_list ap){ if( sqlite3_initialize() ) return 0; #endif sqlite3StrAccumInit(&acc, 0, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); - sqlite3VXPrintf(&acc, zFormat, ap); + sqlite3VXPrintf(&acc, 0, zFormat, ap); z = sqlite3StrAccumFinish(&acc); return z; } @@ -29447,7 +25951,7 @@ SQLITE_API char *sqlite3_vmprintf(const char *zFormat, va_list ap){ ** Print into memory obtained from sqlite3_malloc()(). Omit the internal ** %-conversion extensions. */ -SQLITE_API char *sqlite3_mprintf(const char *zFormat, ...){ +SQLITE_API char *SQLITE_CDECL sqlite3_mprintf(const char *zFormat, ...){ va_list ap; char *z; #ifndef SQLITE_OMIT_AUTOINIT @@ -29472,7 +25976,7 @@ SQLITE_API char *sqlite3_mprintf(const char *zFormat, ...){ ** ** sqlite3_vsnprintf() is the varargs version. */ -SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ +SQLITE_API char *SQLITE_STDCALL sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ StrAccum acc; if( n<=0 ) return zBuf; #ifdef SQLITE_ENABLE_API_ARMOR @@ -29483,10 +25987,10 @@ SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_li } #endif sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); - sqlite3VXPrintf(&acc, zFormat, ap); + sqlite3VXPrintf(&acc, 0, zFormat, ap); return sqlite3StrAccumFinish(&acc); } -SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ +SQLITE_API char *SQLITE_CDECL sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ char *z; va_list ap; va_start(ap,zFormat); @@ -29503,18 +26007,13 @@ SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ ** sqlite3_log() must render into a static buffer. It cannot dynamically ** allocate memory because it might be called while the memory allocator ** mutex is held. -** -** sqlite3VXPrintf() might ask for *temporary* memory allocations for -** certain format characters (%q) or for very large precisions or widths. -** Care must be taken that any sqlite3_log() calls that occur while the -** memory mutex is held do not use these mechanisms. */ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ StrAccum acc; /* String accumulator */ char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); - sqlite3VXPrintf(&acc, zFormat, ap); + sqlite3VXPrintf(&acc, 0, zFormat, ap); sqlite3GlobalConfig.xLog(sqlite3GlobalConfig.pLogArg, iErrCode, sqlite3StrAccumFinish(&acc)); } @@ -29522,7 +26021,7 @@ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ /* ** Format and write a message to the log if logging is enabled. */ -SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...){ +SQLITE_API void SQLITE_CDECL sqlite3_log(int iErrCode, const char *zFormat, ...){ va_list ap; /* Vararg list */ if( sqlite3GlobalConfig.xLog ){ va_start(ap, zFormat); @@ -29543,7 +26042,7 @@ SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ char zBuf[500]; sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); va_start(ap,zFormat); - sqlite3VXPrintf(&acc, zFormat, ap); + sqlite3VXPrintf(&acc, 0, zFormat, ap); va_end(ap); sqlite3StrAccumFinish(&acc); fprintf(stdout,"%s", zBuf); @@ -29551,47 +26050,22 @@ SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ } #endif - -/* -** variable-argument wrapper around sqlite3VXPrintf(). The bFlags argument -** can contain the bit SQLITE_PRINTF_INTERNAL enable internal formats. -*/ -SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, const char *zFormat, ...){ - va_list ap; - va_start(ap,zFormat); - sqlite3VXPrintf(p, zFormat, ap); - va_end(ap); -} - -/************** End of printf.c **********************************************/ -/************** Begin file treeview.c ****************************************/ -/* -** 2015-06-08 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* +#ifdef SQLITE_DEBUG +/************************************************************************* +** Routines for implementing the "TreeView" display of hierarchical +** data structures for debugging. ** -** This file contains C code to implement the TreeView debugging routines. -** These routines print a parse tree to standard output for debugging and -** analysis. +** The main entry points (coded elsewhere) are: +** sqlite3TreeViewExpr(0, pExpr, 0); +** sqlite3TreeViewExprList(0, pList, 0, 0); +** sqlite3TreeViewSelect(0, pSelect, 0); +** Insert calls to those routines while debugging in order to display +** a diagram of Expr, ExprList, and Select objects. ** -** The interfaces in this file is only available when compiling -** with SQLITE_DEBUG. -*/ -/* #include "sqliteInt.h" */ -#ifdef SQLITE_DEBUG - -/* -** Add a new subitem to the tree. The moreToFollow flag indicates that this -** is not the last item in the tree. */ -static TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ +/* Add a new subitem to the tree. The moreToFollow flag indicates that this +** is not the last item in the tree. */ +SQLITE_PRIVATE TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ if( p==0 ){ p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) return 0; @@ -29603,21 +26077,15 @@ static TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ if( p->iLevelbLine) ) p->bLine[p->iLevel] = moreToFollow; return p; } - -/* -** Finished with one layer of the tree -*/ -static void sqlite3TreeViewPop(TreeView *p){ +/* Finished with one layer of the tree */ +SQLITE_PRIVATE void sqlite3TreeViewPop(TreeView *p){ if( p==0 ) return; p->iLevel--; if( p->iLevel<0 ) sqlite3_free(p); } - -/* -** Generate a single line of output for the tree, with a prefix that contains -** all the appropriate tree lines -*/ -static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ +/* Generate a single line of output for the tree, with a prefix that contains +** all the appropriate tree lines */ +SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ va_list ap; int i; StrAccum acc; @@ -29630,459 +26098,31 @@ static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|-- " : "'-- ", 4); } va_start(ap, zFormat); - sqlite3VXPrintf(&acc, zFormat, ap); + sqlite3VXPrintf(&acc, 0, zFormat, ap); va_end(ap); if( zBuf[acc.nChar-1]!='\n' ) sqlite3StrAccumAppend(&acc, "\n", 1); sqlite3StrAccumFinish(&acc); fprintf(stdout,"%s", zBuf); fflush(stdout); } - -/* -** Shorthand for starting a new tree item that consists of a single label -*/ -static void sqlite3TreeViewItem(TreeView *p, const char *zLabel,u8 moreFollows){ - p = sqlite3TreeViewPush(p, moreFollows); +/* Shorthand for starting a new tree item that consists of a single label */ +SQLITE_PRIVATE void sqlite3TreeViewItem(TreeView *p, const char *zLabel, u8 moreToFollow){ + p = sqlite3TreeViewPush(p, moreToFollow); sqlite3TreeViewLine(p, "%s", zLabel); } +#endif /* SQLITE_DEBUG */ /* -** Generate a human-readable description of a WITH clause. -*/ -SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 moreToFollow){ - int i; - if( pWith==0 ) return; - if( pWith->nCte==0 ) return; - if( pWith->pOuter ){ - sqlite3TreeViewLine(pView, "WITH (0x%p, pOuter=0x%p)",pWith,pWith->pOuter); - }else{ - sqlite3TreeViewLine(pView, "WITH (0x%p)", pWith); - } - if( pWith->nCte>0 ){ - pView = sqlite3TreeViewPush(pView, 1); - for(i=0; inCte; i++){ - StrAccum x; - char zLine[1000]; - const struct Cte *pCte = &pWith->a[i]; - sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); - sqlite3XPrintf(&x, "%s", pCte->zName); - if( pCte->pCols && pCte->pCols->nExpr>0 ){ - char cSep = '('; - int j; - for(j=0; jpCols->nExpr; j++){ - sqlite3XPrintf(&x, "%c%s", cSep, pCte->pCols->a[j].zName); - cSep = ','; - } - sqlite3XPrintf(&x, ")"); - } - sqlite3XPrintf(&x, " AS"); - sqlite3StrAccumFinish(&x); - sqlite3TreeViewItem(pView, zLine, inCte-1); - sqlite3TreeViewSelect(pView, pCte->pSelect, 0); - sqlite3TreeViewPop(pView); - } - sqlite3TreeViewPop(pView); - } -} - - -/* -** Generate a human-readable description of a Select object. -*/ -SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ - int n = 0; - int cnt = 0; - pView = sqlite3TreeViewPush(pView, moreToFollow); - if( p->pWith ){ - sqlite3TreeViewWith(pView, p->pWith, 1); - cnt = 1; - sqlite3TreeViewPush(pView, 1); - } - do{ - sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x nSelectRow=%d", - ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags, - (int)p->nSelectRow - ); - if( cnt++ ) sqlite3TreeViewPop(pView); - if( p->pPrior ){ - n = 1000; - }else{ - n = 0; - if( p->pSrc && p->pSrc->nSrc ) n++; - if( p->pWhere ) n++; - if( p->pGroupBy ) n++; - if( p->pHaving ) n++; - if( p->pOrderBy ) n++; - if( p->pLimit ) n++; - if( p->pOffset ) n++; - } - sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); - if( p->pSrc && p->pSrc->nSrc ){ - int i; - pView = sqlite3TreeViewPush(pView, (n--)>0); - sqlite3TreeViewLine(pView, "FROM"); - for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - StrAccum x; - char zLine[100]; - sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); - sqlite3XPrintf(&x, "{%d,*}", pItem->iCursor); - if( pItem->zDatabase ){ - sqlite3XPrintf(&x, " %s.%s", pItem->zDatabase, pItem->zName); - }else if( pItem->zName ){ - sqlite3XPrintf(&x, " %s", pItem->zName); - } - if( pItem->pTab ){ - sqlite3XPrintf(&x, " tabname=%Q", pItem->pTab->zName); - } - if( pItem->zAlias ){ - sqlite3XPrintf(&x, " (AS %s)", pItem->zAlias); - } - if( pItem->fg.jointype & JT_LEFT ){ - sqlite3XPrintf(&x, " LEFT-JOIN"); - } - sqlite3StrAccumFinish(&x); - sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); - if( pItem->pSelect ){ - sqlite3TreeViewSelect(pView, pItem->pSelect, 0); - } - if( pItem->fg.isTabFunc ){ - sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); - } - sqlite3TreeViewPop(pView); - } - sqlite3TreeViewPop(pView); - } - if( p->pWhere ){ - sqlite3TreeViewItem(pView, "WHERE", (n--)>0); - sqlite3TreeViewExpr(pView, p->pWhere, 0); - sqlite3TreeViewPop(pView); - } - if( p->pGroupBy ){ - sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); - } - if( p->pHaving ){ - sqlite3TreeViewItem(pView, "HAVING", (n--)>0); - sqlite3TreeViewExpr(pView, p->pHaving, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOrderBy ){ - sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); - } - if( p->pLimit ){ - sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); - sqlite3TreeViewExpr(pView, p->pLimit, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOffset ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); - sqlite3TreeViewExpr(pView, p->pOffset, 0); - sqlite3TreeViewPop(pView); - } - if( p->pPrior ){ - const char *zOp = "UNION"; - switch( p->op ){ - case TK_ALL: zOp = "UNION ALL"; break; - case TK_INTERSECT: zOp = "INTERSECT"; break; - case TK_EXCEPT: zOp = "EXCEPT"; break; - } - sqlite3TreeViewItem(pView, zOp, 1); - } - p = p->pPrior; - }while( p!=0 ); - sqlite3TreeViewPop(pView); -} - -/* -** Generate a human-readable explanation of an expression tree. -*/ -SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ - const char *zBinOp = 0; /* Binary operator */ - const char *zUniOp = 0; /* Unary operator */ - char zFlgs[30]; - pView = sqlite3TreeViewPush(pView, moreToFollow); - if( pExpr==0 ){ - sqlite3TreeViewLine(pView, "nil"); - sqlite3TreeViewPop(pView); - return; - } - if( pExpr->flags ){ - sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x",pExpr->flags); - }else{ - zFlgs[0] = 0; - } - switch( pExpr->op ){ - case TK_AGG_COLUMN: { - sqlite3TreeViewLine(pView, "AGG{%d:%d}%s", - pExpr->iTable, pExpr->iColumn, zFlgs); - break; - } - case TK_COLUMN: { - if( pExpr->iTable<0 ){ - /* This only happens when coding check constraints */ - sqlite3TreeViewLine(pView, "COLUMN(%d)%s", pExpr->iColumn, zFlgs); - }else{ - sqlite3TreeViewLine(pView, "{%d:%d}%s", - pExpr->iTable, pExpr->iColumn, zFlgs); - } - break; - } - case TK_INTEGER: { - if( pExpr->flags & EP_IntValue ){ - sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue); - }else{ - sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken); - } - break; - } -#ifndef SQLITE_OMIT_FLOATING_POINT - case TK_FLOAT: { - sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); - break; - } -#endif - case TK_STRING: { - sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); - break; - } - case TK_NULL: { - sqlite3TreeViewLine(pView,"NULL"); - break; - } -#ifndef SQLITE_OMIT_BLOB_LITERAL - case TK_BLOB: { - sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); - break; - } -#endif - case TK_VARIABLE: { - sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", - pExpr->u.zToken, pExpr->iColumn); - break; - } - case TK_REGISTER: { - sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); - break; - } - case TK_ID: { - sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken); - break; - } -#ifndef SQLITE_OMIT_CAST - case TK_CAST: { - /* Expressions of the form: CAST(pLeft AS token) */ - sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - break; - } -#endif /* SQLITE_OMIT_CAST */ - case TK_LT: zBinOp = "LT"; break; - case TK_LE: zBinOp = "LE"; break; - case TK_GT: zBinOp = "GT"; break; - case TK_GE: zBinOp = "GE"; break; - case TK_NE: zBinOp = "NE"; break; - case TK_EQ: zBinOp = "EQ"; break; - case TK_IS: zBinOp = "IS"; break; - case TK_ISNOT: zBinOp = "ISNOT"; break; - case TK_AND: zBinOp = "AND"; break; - case TK_OR: zBinOp = "OR"; break; - case TK_PLUS: zBinOp = "ADD"; break; - case TK_STAR: zBinOp = "MUL"; break; - case TK_MINUS: zBinOp = "SUB"; break; - case TK_REM: zBinOp = "REM"; break; - case TK_BITAND: zBinOp = "BITAND"; break; - case TK_BITOR: zBinOp = "BITOR"; break; - case TK_SLASH: zBinOp = "DIV"; break; - case TK_LSHIFT: zBinOp = "LSHIFT"; break; - case TK_RSHIFT: zBinOp = "RSHIFT"; break; - case TK_CONCAT: zBinOp = "CONCAT"; break; - case TK_DOT: zBinOp = "DOT"; break; - - case TK_UMINUS: zUniOp = "UMINUS"; break; - case TK_UPLUS: zUniOp = "UPLUS"; break; - case TK_BITNOT: zUniOp = "BITNOT"; break; - case TK_NOT: zUniOp = "NOT"; break; - case TK_ISNULL: zUniOp = "ISNULL"; break; - case TK_NOTNULL: zUniOp = "NOTNULL"; break; - - case TK_SPAN: { - sqlite3TreeViewLine(pView, "SPAN %Q", pExpr->u.zToken); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - break; - } - - case TK_COLLATE: { - sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - break; - } - - case TK_AGG_FUNCTION: - case TK_FUNCTION: { - ExprList *pFarg; /* List of function arguments */ - if( ExprHasProperty(pExpr, EP_TokenOnly) ){ - pFarg = 0; - }else{ - pFarg = pExpr->x.pList; - } - if( pExpr->op==TK_AGG_FUNCTION ){ - sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", - pExpr->op2, pExpr->u.zToken); - }else{ - sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); - } - if( pFarg ){ - sqlite3TreeViewExprList(pView, pFarg, 0, 0); - } - break; - } -#ifndef SQLITE_OMIT_SUBQUERY - case TK_EXISTS: { - sqlite3TreeViewLine(pView, "EXISTS-expr"); - sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); - break; - } - case TK_SELECT: { - sqlite3TreeViewLine(pView, "SELECT-expr"); - sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); - break; - } - case TK_IN: { - sqlite3TreeViewLine(pView, "IN"); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); - }else{ - sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); - } - break; - } -#endif /* SQLITE_OMIT_SUBQUERY */ - - /* - ** x BETWEEN y AND z - ** - ** This is equivalent to - ** - ** x>=y AND x<=z - ** - ** X is stored in pExpr->pLeft. - ** Y is stored in pExpr->pList->a[0].pExpr. - ** Z is stored in pExpr->pList->a[1].pExpr. - */ - case TK_BETWEEN: { - Expr *pX = pExpr->pLeft; - Expr *pY = pExpr->x.pList->a[0].pExpr; - Expr *pZ = pExpr->x.pList->a[1].pExpr; - sqlite3TreeViewLine(pView, "BETWEEN"); - sqlite3TreeViewExpr(pView, pX, 1); - sqlite3TreeViewExpr(pView, pY, 1); - sqlite3TreeViewExpr(pView, pZ, 0); - break; - } - case TK_TRIGGER: { - /* If the opcode is TK_TRIGGER, then the expression is a reference - ** to a column in the new.* or old.* pseudo-tables available to - ** trigger programs. In this case Expr.iTable is set to 1 for the - ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn - ** is set to the column of the pseudo-table to read, or to -1 to - ** read the rowid field. - */ - sqlite3TreeViewLine(pView, "%s(%d)", - pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); - break; - } - case TK_CASE: { - sqlite3TreeViewLine(pView, "CASE"); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); - sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); - break; - } -#ifndef SQLITE_OMIT_TRIGGER - case TK_RAISE: { - const char *zType = "unk"; - switch( pExpr->affinity ){ - case OE_Rollback: zType = "rollback"; break; - case OE_Abort: zType = "abort"; break; - case OE_Fail: zType = "fail"; break; - case OE_Ignore: zType = "ignore"; break; - } - sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); - break; - } -#endif - case TK_MATCH: { - sqlite3TreeViewLine(pView, "MATCH {%d:%d}%s", - pExpr->iTable, pExpr->iColumn, zFlgs); - sqlite3TreeViewExpr(pView, pExpr->pRight, 0); - break; - } - case TK_VECTOR: { - sqlite3TreeViewBareExprList(pView, pExpr->x.pList, "VECTOR"); - break; - } - case TK_SELECT_COLUMN: { - sqlite3TreeViewLine(pView, "SELECT-COLUMN %d", pExpr->iColumn); - sqlite3TreeViewSelect(pView, pExpr->pLeft->x.pSelect, 0); - break; - } - default: { - sqlite3TreeViewLine(pView, "op=%d", pExpr->op); - break; - } - } - if( zBinOp ){ - sqlite3TreeViewLine(pView, "%s%s", zBinOp, zFlgs); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); - sqlite3TreeViewExpr(pView, pExpr->pRight, 0); - }else if( zUniOp ){ - sqlite3TreeViewLine(pView, "%s%s", zUniOp, zFlgs); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - } - sqlite3TreeViewPop(pView); -} - - -/* -** Generate a human-readable explanation of an expression list. +** variable-argument wrapper around sqlite3VXPrintf(). */ -SQLITE_PRIVATE void sqlite3TreeViewBareExprList( - TreeView *pView, - const ExprList *pList, - const char *zLabel -){ - if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; - if( pList==0 ){ - sqlite3TreeViewLine(pView, "%s (empty)", zLabel); - }else{ - int i; - sqlite3TreeViewLine(pView, "%s", zLabel); - for(i=0; inExpr; i++){ - int j = pList->a[i].u.x.iOrderByCol; - if( j ){ - sqlite3TreeViewPush(pView, 0); - sqlite3TreeViewLine(pView, "iOrderByCol=%d", j); - } - sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); - if( j ) sqlite3TreeViewPop(pView); - } - } -} -SQLITE_PRIVATE void sqlite3TreeViewExprList( - TreeView *pView, - const ExprList *pList, - u8 moreToFollow, - const char *zLabel -){ - pView = sqlite3TreeViewPush(pView, moreToFollow); - sqlite3TreeViewBareExprList(pView, pList, zLabel); - sqlite3TreeViewPop(pView); +SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, u32 bFlags, const char *zFormat, ...){ + va_list ap; + va_start(ap,zFormat); + sqlite3VXPrintf(p, bFlags, zFormat, ap); + va_end(ap); } -#endif /* SQLITE_DEBUG */ - -/************** End of treeview.c ********************************************/ +/************** End of printf.c **********************************************/ /************** Begin file random.c ******************************************/ /* ** 2001 September 15 @@ -30101,7 +26141,6 @@ SQLITE_PRIVATE void sqlite3TreeViewExprList( ** Random numbers are used by some of the database backends in order ** to generate random integer keys for tables or random filenames. */ -/* #include "sqliteInt.h" */ /* All threads share a single random number generator. @@ -30116,7 +26155,7 @@ static SQLITE_WSD struct sqlite3PrngType { /* ** Return N random bytes. */ -SQLITE_API void sqlite3_randomness(int N, void *pBuf){ +SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *pBuf){ unsigned char t; unsigned char *zBuf = pBuf; @@ -30248,9 +26287,7 @@ SQLITE_PRIVATE void sqlite3PrngRestoreState(void){ ** of multiple cores can do so, while also allowing applications to stay ** single-threaded if desired. */ -/* #include "sqliteInt.h" */ #if SQLITE_OS_WIN -/* # include "os_win.h" */ #endif #if SQLITE_MAX_WORKER_THREADS>0 @@ -30286,14 +26323,10 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); - if( p==0 ) return SQLITE_NOMEM_BKPT; + if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); p->xTask = xTask; p->pIn = pIn; - /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a - ** function that returns SQLITE_ERROR when passed the argument 200, that - ** forces worker threads to run sequentially and deterministically - ** for testing purposes. */ if( sqlite3FaultSim(200) ){ rc = 1; }else{ @@ -30312,7 +26345,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ int rc; assert( ppOut!=0 ); - if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; + if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->done ){ *ppOut = p->pOut; rc = SQLITE_OK; @@ -30377,13 +26410,8 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( assert( xTask!=0 ); *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); - if( p==0 ) return SQLITE_NOMEM_BKPT; - /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a - ** function that returns SQLITE_ERROR when passed the argument 200, that - ** forces worker threads to run sequentially and deterministically - ** (via the sqlite3FaultSim() term of the conditional) for testing - ** purposes. */ - if( sqlite3GlobalConfig.bCoreMutex==0 || sqlite3FaultSim(200) ){ + if( p==0 ) return SQLITE_NOMEM; + if( sqlite3GlobalConfig.bCoreMutex==0 ){ memset(p, 0, sizeof(*p)); }else{ p->xTask = xTask; @@ -30409,9 +26437,9 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ BOOL bRc; assert( ppOut!=0 ); - if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; + if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->xTask==0 ){ - /* assert( p->id==GetCurrentThreadId() ); */ + assert( p->id==GetCurrentThreadId() ); rc = WAIT_OBJECT_0; assert( p->tid==0 ); }else{ @@ -30457,7 +26485,7 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( assert( xTask!=0 ); *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); - if( p==0 ) return SQLITE_NOMEM_BKPT; + if( p==0 ) return SQLITE_NOMEM; if( (SQLITE_PTR_TO_INT(p)/17)&1 ){ p->xTask = xTask; p->pIn = pIn; @@ -30473,7 +26501,7 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); - if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; + if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->xTask ){ *ppOut = p->xTask(p->pIn); }else{ @@ -30484,7 +26512,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ #if defined(SQLITE_TEST) { void *pTstAlloc = sqlite3Malloc(10); - if (!pTstAlloc) return SQLITE_NOMEM_BKPT; + if (!pTstAlloc) return SQLITE_NOMEM; sqlite3_free(pTstAlloc); } #endif @@ -30533,17 +26561,15 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ ** 0xfe 0xff big-endian utf-16 follows ** */ -/* #include "sqliteInt.h" */ /* #include */ -/* #include "vdbeInt.h" */ -#if !defined(SQLITE_AMALGAMATION) && SQLITE_BYTEORDER==0 +#ifndef SQLITE_AMALGAMATION /* ** The following constant value is used by the SQLITE_BIGENDIAN and ** SQLITE_LITTLEENDIAN macros. */ SQLITE_PRIVATE const int sqlite3one = 1; -#endif /* SQLITE_AMALGAMATION && SQLITE_BYTEORDER==0 */ +#endif /* SQLITE_AMALGAMATION */ /* ** This lookup table is used to help decode the first byte of @@ -30731,7 +26757,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired rc = sqlite3VdbeMemMakeWriteable(pMem); if( rc!=SQLITE_OK ){ assert( rc==SQLITE_NOMEM ); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } zIn = (u8*)pMem->z; zTerm = &zIn[pMem->n&~1]; @@ -30773,7 +26799,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired zTerm = &zIn[pMem->n]; zOut = sqlite3DbMallocRaw(pMem->db, len); if( !zOut ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } z = zOut; @@ -30816,7 +26842,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired c = pMem->flags; sqlite3VdbeMemRelease(pMem); - pMem->flags = MEM_Str|MEM_Term|(c&(MEM_AffMask|MEM_Subtype)); + pMem->flags = MEM_Str|MEM_Term|(c&MEM_AffMask); pMem->enc = desiredEnc; pMem->z = (char*)zOut; pMem->zMalloc = pMem->z; @@ -31048,7 +27074,6 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ ** strings, and stuff like that. ** */ -/* #include "sqliteInt.h" */ /* #include */ #if HAVE_ISNAN || SQLITE_HAVE_ISNAN # include @@ -31138,53 +27163,19 @@ SQLITE_PRIVATE int sqlite3IsNaN(double x){ ** than 1GiB) the value returned might be less than the true string length. */ SQLITE_PRIVATE int sqlite3Strlen30(const char *z){ + const char *z2 = z; if( z==0 ) return 0; - return 0x3fffffff & (int)strlen(z); -} - -/* -** Return the declared type of a column. Or return zDflt if the column -** has no declared type. -** -** The column type is an extra string stored after the zero-terminator on -** the column name if and only if the COLFLAG_HASTYPE flag is set. -*/ -SQLITE_PRIVATE char *sqlite3ColumnType(Column *pCol, char *zDflt){ - if( (pCol->colFlags & COLFLAG_HASTYPE)==0 ) return zDflt; - return pCol->zName + strlen(pCol->zName) + 1; -} - -/* -** Helper function for sqlite3Error() - called rarely. Broken out into -** a separate routine to avoid unnecessary register saves on entry to -** sqlite3Error(). -*/ -static SQLITE_NOINLINE void sqlite3ErrorFinish(sqlite3 *db, int err_code){ - if( db->pErr ) sqlite3ValueSetNull(db->pErr); - sqlite3SystemError(db, err_code); + while( *z2 ){ z2++; } + return 0x3fffffff & (int)(z2 - z); } /* ** Set the current error code to err_code and clear any prior error message. -** Also set iSysErrno (by calling sqlite3System) if the err_code indicates -** that would be appropriate. */ SQLITE_PRIVATE void sqlite3Error(sqlite3 *db, int err_code){ assert( db!=0 ); db->errCode = err_code; - if( err_code || db->pErr ) sqlite3ErrorFinish(db, err_code); -} - -/* -** Load the sqlite3.iSysErrno field if that is an appropriate thing -** to do based on the SQLite error code in rc. -*/ -SQLITE_PRIVATE void sqlite3SystemError(sqlite3 *db, int rc){ - if( rc==SQLITE_IOERR_NOMEM ) return; - rc &= 0xff; - if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){ - db->iSysErrno = sqlite3OsGetLastError(db->pVfs); - } + if( db->pErr ) sqlite3ValueSetNull(db->pErr); } /* @@ -31211,7 +27202,6 @@ SQLITE_PRIVATE void sqlite3SystemError(sqlite3 *db, int rc){ SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *zFormat, ...){ assert( db!=0 ); db->errCode = err_code; - sqlite3SystemError(db, err_code); if( zFormat==0 ){ sqlite3Error(db, err_code); }else if( db->pErr || (db->pErr = sqlite3ValueNew(db))!=0 ){ @@ -31275,13 +27265,18 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ ** brackets from around identifiers. For example: "[a-b-c]" becomes ** "a-b-c". */ -SQLITE_PRIVATE void sqlite3Dequote(char *z){ +SQLITE_PRIVATE int sqlite3Dequote(char *z){ char quote; int i, j; - if( z==0 ) return; + if( z==0 ) return -1; quote = z[0]; - if( !sqlite3Isquote(quote) ) return; - if( quote=='[' ) quote = ']'; + switch( quote ){ + case '\'': break; + case '"': break; + case '`': break; /* For MySQL compatibility */ + case '[': quote = ']'; break; /* For MS SqlServer compatibility */ + default: return -1; + } for(i=1, j=0;; i++){ assert( z[i] ); if( z[i]==quote ){ @@ -31296,14 +27291,7 @@ SQLITE_PRIVATE void sqlite3Dequote(char *z){ } } z[j] = 0; -} - -/* -** Generate a Token object from a string -*/ -SQLITE_PRIVATE void sqlite3TokenInit(Token *p, char *z){ - p->z = z; - p->n = sqlite3Strlen30(z); + return j; } /* Convenient short-hand */ @@ -31319,28 +27307,19 @@ SQLITE_PRIVATE void sqlite3TokenInit(Token *p, char *z){ ** case-independent fashion, using the same definition of "case ** independence" that SQLite uses internally when comparing identifiers. */ -SQLITE_API int sqlite3_stricmp(const char *zLeft, const char *zRight){ +SQLITE_API int SQLITE_STDCALL sqlite3_stricmp(const char *zLeft, const char *zRight){ + register unsigned char *a, *b; if( zLeft==0 ){ return zRight ? -1 : 0; }else if( zRight==0 ){ return 1; } - return sqlite3StrICmp(zLeft, zRight); -} -SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ - unsigned char *a, *b; - int c; a = (unsigned char *)zLeft; b = (unsigned char *)zRight; - for(;;){ - c = (int)UpperToLower[*a] - (int)UpperToLower[*b]; - if( c || *a==0 ) break; - a++; - b++; - } - return c; + while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } + return UpperToLower[*a] - UpperToLower[*b]; } -SQLITE_API int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ +SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ register unsigned char *a, *b; if( zLeft==0 ){ return zRight ? -1 : 0; @@ -31388,7 +27367,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en int eValid = 1; /* True exponent is either not used or is well-formed */ double result; int nDigits = 0; - int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ + int nonNum = 0; assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ @@ -31401,7 +27380,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); for(i=3-enc; i=zEnd ) goto do_atof_calc; @@ -31447,12 +27428,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en if( *z=='e' || *z=='E' ){ z+=incr; eValid = 0; - - /* This branch is needed to avoid a (harmless) buffer overread. The - ** special comment alerts the mutation tester that the correct answer - ** is obtained even if the branch is omitted */ - if( z>=zEnd ) goto do_atof_calc; /*PREVENTS-HARMLESS-OVERREAD*/ - + if( z>=zEnd ) goto do_atof_calc; /* get sign of exponent */ if( *z=='-' ){ esign = -1; @@ -31469,7 +27445,9 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en } /* skip trailing spaces */ - while( z0 ){ /*OPTIMIZATION-IF-TRUE*/ - if( esign>0 ){ - if( s>=(LARGEST_INT64/10) ) break; /*OPTIMIZATION-IF-FALSE*/ - s *= 10; - }else{ - if( s%10!=0 ) break; /*OPTIMIZATION-IF-FALSE*/ - s /= 10; - } - e--; + /* attempt to reduce exponent */ + if( esign>0 ){ + while( s<(LARGEST_INT64/10) && e>0 ) e--,s*=10; + }else{ + while( !(s%10) && e>0 ) e--,s/=10; } /* adjust the sign of significand */ s = sign<0 ? -s : s; - if( e==0 ){ /*OPTIMIZATION-IF-TRUE*/ - result = (double)s; - }else{ + /* if exponent, scale significand as appropriate + ** and store in result. */ + if( e ){ LONGDOUBLE_TYPE scale = 1.0; /* attempt to handle extremely small/large numbers better */ - if( e>307 ){ /*OPTIMIZATION-IF-TRUE*/ - if( e<342 ){ /*OPTIMIZATION-IF-TRUE*/ - while( e%308 ) { scale *= 1.0e+1; e -= 1; } - if( esign<0 ){ - result = s / scale; - result /= 1.0e+308; - }else{ - result = s * scale; - result *= 1.0e+308; - } - }else{ assert( e>=342 ); - if( esign<0 ){ - result = 0.0*s; - }else{ - result = 1e308*1e308*s; /* Infinity */ - } + if( e>307 && e<342 ){ + while( e%308 ) { scale *= 1.0e+1; e -= 1; } + if( esign<0 ){ + result = s / scale; + result /= 1.0e+308; + }else{ + result = s * scale; + result *= 1.0e+308; + } + }else if( e>=342 ){ + if( esign<0 ){ + result = 0.0*s; + }else{ + result = 1e308*1e308*s; /* Infinity */ } }else{ /* 1.0e+22 is the largest power of 10 than can be @@ -31538,6 +27506,8 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en result = s * scale; } } + } else { + result = (double)s; } } @@ -31545,7 +27515,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en *pResult = result; /* return true if number and no extra non-whitespace chracters after */ - return z==zEnd && nDigits>0 && eValid && nonNum==0; + return z>=zEnd && nDigits>0 && eValid && nonNum==0; #else return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ @@ -31607,7 +27577,7 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc int neg = 0; /* assume positive */ int i; int c = 0; - int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ + int nonNum = 0; const char *zStart; const char *zEnd = zNum + length; assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); @@ -31618,7 +27588,7 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); for(i=3-enc; i19*incr /* Too many digits */ - || nonNum /* UTF16 with high-order bytes non-zero */ - ){ + if( (c!=0 && &zNum[i]19*incr || nonNum ){ /* zNum is empty or contains non-numeric text or is longer ** than 19 digits (thus guaranteeing that it is too large) */ return 1; @@ -31691,6 +27657,7 @@ SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') + && sqlite3Isxdigit(z[2]) ){ u64 u = 0; int i, k; @@ -31937,8 +27904,7 @@ SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ /* a: p0<<28 | p2<<14 | p4 (unmasked) */ if (!(a&0x80)) { - /* we can skip these cause they were (effectively) done above - ** while calculating s */ + /* we can skip these cause they were (effectively) done above in calc'ing s */ /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ /* b &= (0x7f<<14)|(0x7f); */ b = b<<7; @@ -32159,8 +28125,11 @@ SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ ** 64-bit integer. */ SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ - int i; - for(i=1; (v >>= 7)!=0; i++){ assert( i<10 ); } + int i = 0; + do{ + i++; + v >>= 7; + }while( v!=0 && ALWAYS(i<9) ); return i; } @@ -32169,42 +28138,14 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ ** Read or write a four-byte big-endian integer value. */ SQLITE_PRIVATE u32 sqlite3Get4byte(const u8 *p){ -#if SQLITE_BYTEORDER==4321 - u32 x; - memcpy(&x,p,4); - return x; -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(__GNUC__) && GCC_VERSION>=4003000 - u32 x; - memcpy(&x,p,4); - return __builtin_bswap32(x); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 - u32 x; - memcpy(&x,p,4); - return _byteswap_ulong(x); -#else testcase( p[0]&0x80 ); return ((unsigned)p[0]<<24) | (p[1]<<16) | (p[2]<<8) | p[3]; -#endif } SQLITE_PRIVATE void sqlite3Put4byte(unsigned char *p, u32 v){ -#if SQLITE_BYTEORDER==4321 - memcpy(p,&v,4); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(__GNUC__) && GCC_VERSION>=4003000 - u32 x = __builtin_bswap32(v); - memcpy(p,&x,4); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 - u32 x = _byteswap_ulong(v); - memcpy(p,&x,4); -#else p[0] = (u8)(v>>24); p[1] = (u8)(v>>16); p[2] = (u8)(v>>8); p[3] = (u8)v; -#endif } @@ -32236,7 +28177,7 @@ SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ char *zBlob; int i; - zBlob = (char *)sqlite3DbMallocRawNN(db, n/2 + 1); + zBlob = (char *)sqlite3DbMallocRaw(db, n/2 + 1); n--; if( zBlob ){ for(i=0; i0 ){ - if( iA>LARGEST_INT64/iB ) return 1; - if( iA0 ){ - if( iBLARGEST_INT64/-iB ) return 1; - } - } - *pA = iA*iB; + i64 iA1, iA0, iB1, iB0, r; + + iA1 = iA/TWOPOWER32; + iA0 = iA % TWOPOWER32; + iB1 = iB/TWOPOWER32; + iB0 = iB % TWOPOWER32; + if( iA1==0 ){ + if( iB1==0 ){ + *pA *= iB; + return 0; + } + r = iA0*iB1; + }else if( iB1==0 ){ + r = iA1*iB0; + }else{ + /* If both iA1 and iB1 are non-zero, overflow will result */ + return 1; + } + testcase( r==(-TWOPOWER31)-1 ); + testcase( r==(-TWOPOWER31) ); + testcase( r==TWOPOWER31 ); + testcase( r==TWOPOWER31-1 ); + if( r<(-TWOPOWER31) || r>=TWOPOWER31 ) return 1; + r *= TWOPOWER32; + if( sqlite3AddInt64(&r, iA0*iB0) ) return 1; + *pA = r; return 0; } @@ -32437,7 +28393,7 @@ SQLITE_PRIVATE LogEst sqlite3LogEst(u64 x){ if( x<2 ) return 0; while( x<8 ){ y -= 10; x <<= 1; } }else{ - while( x>255 ){ y += 40; x >>= 4; } /*OPTIMIZATION-IF-TRUE*/ + while( x>255 ){ y += 40; x >>= 4; } while( x>15 ){ y += 10; x >>= 1; } } return a[x&7] + y - 10; @@ -32460,32 +28416,21 @@ SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double x){ } #endif /* SQLITE_OMIT_VIRTUALTABLE */ -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) /* ** Convert a LogEst into an integer. -** -** Note that this routine is only used when one or more of various -** non-standard compile-time options is enabled. */ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ u64 n; + if( x<10 ) return 1; n = x%10; x /= 10; if( n>=5 ) n -= 2; else if( n>=1 ) n -= 1; -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) - if( x>60 ) return (u64)LARGEST_INT64; -#else - /* If only SQLITE_ENABLE_STAT3_OR_STAT4 is on, then the largest input - ** possible to this routine is 310, resulting in a maximum x of 31 */ - assert( x<=60 ); -#endif - return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x); + if( x>=3 ){ + return x>60 ? (u64)LARGEST_INT64 : (n+8)<<(x-3); + } + return (n+8)>>(3-x); } -#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */ /************** End of util.c ************************************************/ /************** Begin file hash.c ********************************************/ @@ -32503,7 +28448,6 @@ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ ** This is the implementation of generic hash-tables ** used in SQLite. */ -/* #include "sqliteInt.h" */ /* #include */ /* Turn bulk memory into a hash table object by initializing the @@ -32546,12 +28490,8 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash *pH){ static unsigned int strHash(const char *z){ unsigned int h = 0; unsigned char c; - while( (c = (unsigned char)*z++)!=0 ){ /*OPTIMIZATION-IF-TRUE*/ - /* Knuth multiplicative hashing. (Sorting & Searching, p. 510). - ** 0x9e3779b1 is 2654435761 which is the closest prime number to - ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */ - h += sqlite3UpperToLower[c]; - h *= 0x9e3779b1; + while( (c = (unsigned char)*z++)!=0 ){ + h = (h<<3) ^ h ^ sqlite3UpperToLower[c]; } return h; } @@ -32643,7 +28583,7 @@ static HashElem *findElementWithHash( int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ - if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ + if( pH->ht ){ struct _ht *pEntry; h = strHash(pKey) % pH->htsize; pEntry = &pH->ht[h]; @@ -32764,180 +28704,172 @@ SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ /************** End of hash.c ************************************************/ /************** Begin file opcodes.c *****************************************/ /* Automatically generated. Do not edit */ -/* See the tool/mkopcodec.tcl script for details. */ -#if !defined(SQLITE_OMIT_EXPLAIN) \ - || defined(VDBE_PROFILE) \ - || defined(SQLITE_DEBUG) +/* See the mkopcodec.awk script for details. */ +#if !defined(SQLITE_OMIT_EXPLAIN) || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) #if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) || defined(SQLITE_DEBUG) # define OpHelp(X) "\0" X #else # define OpHelp(X) #endif SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ - static const char *const azName[] = { - /* 0 */ "Savepoint" OpHelp(""), - /* 1 */ "AutoCommit" OpHelp(""), - /* 2 */ "Transaction" OpHelp(""), - /* 3 */ "SorterNext" OpHelp(""), - /* 4 */ "PrevIfOpen" OpHelp(""), - /* 5 */ "NextIfOpen" OpHelp(""), - /* 6 */ "Prev" OpHelp(""), - /* 7 */ "Next" OpHelp(""), - /* 8 */ "Checkpoint" OpHelp(""), - /* 9 */ "JournalMode" OpHelp(""), - /* 10 */ "Vacuum" OpHelp(""), - /* 11 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), - /* 12 */ "VUpdate" OpHelp("data=r[P3@P2]"), - /* 13 */ "Goto" OpHelp(""), - /* 14 */ "Gosub" OpHelp(""), - /* 15 */ "InitCoroutine" OpHelp(""), - /* 16 */ "Yield" OpHelp(""), - /* 17 */ "MustBeInt" OpHelp(""), - /* 18 */ "Jump" OpHelp(""), - /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "Once" OpHelp(""), - /* 21 */ "If" OpHelp(""), - /* 22 */ "IfNot" OpHelp(""), - /* 23 */ "SeekLT" OpHelp("key=r[P3@P4]"), - /* 24 */ "SeekLE" OpHelp("key=r[P3@P4]"), - /* 25 */ "SeekGE" OpHelp("key=r[P3@P4]"), - /* 26 */ "SeekGT" OpHelp("key=r[P3@P4]"), - /* 27 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), - /* 28 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 29 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 30 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 31 */ "Found" OpHelp("key=r[P3@P4]"), - /* 32 */ "SeekRowid" OpHelp("intkey=r[P3]"), - /* 33 */ "NotExists" OpHelp("intkey=r[P3]"), - /* 34 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), - /* 35 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), - /* 36 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), - /* 37 */ "Eq" OpHelp("IF r[P3]==r[P1]"), - /* 38 */ "Gt" OpHelp("IF r[P3]>r[P1]"), - /* 39 */ "Le" OpHelp("IF r[P3]<=r[P1]"), - /* 40 */ "Lt" OpHelp("IF r[P3]=r[P1]"), - /* 42 */ "ElseNotEq" OpHelp(""), - /* 43 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), - /* 44 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), - /* 45 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), - /* 47 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), - /* 48 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), - /* 49 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), - /* 50 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), - /* 51 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), - /* 52 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), - /* 53 */ "Last" OpHelp(""), - /* 54 */ "BitNot" OpHelp("r[P1]= ~r[P1]"), - /* 55 */ "SorterSort" OpHelp(""), - /* 56 */ "Sort" OpHelp(""), - /* 57 */ "Rewind" OpHelp(""), - /* 58 */ "IdxLE" OpHelp("key=r[P3@P4]"), - /* 59 */ "IdxGT" OpHelp("key=r[P3@P4]"), - /* 60 */ "IdxLT" OpHelp("key=r[P3@P4]"), - /* 61 */ "IdxGE" OpHelp("key=r[P3@P4]"), - /* 62 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), - /* 63 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), - /* 64 */ "Program" OpHelp(""), - /* 65 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), - /* 66 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), - /* 67 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]-=P3, goto P2"), - /* 68 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 69 */ "IncrVacuum" OpHelp(""), - /* 70 */ "VNext" OpHelp(""), - /* 71 */ "Init" OpHelp("Start at P2"), - /* 72 */ "Return" OpHelp(""), - /* 73 */ "EndCoroutine" OpHelp(""), - /* 74 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 75 */ "Halt" OpHelp(""), - /* 76 */ "Integer" OpHelp("r[P2]=P1"), - /* 77 */ "Int64" OpHelp("r[P2]=P4"), - /* 78 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 79 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 80 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 81 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 82 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), - /* 83 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 84 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 85 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 86 */ "IntCopy" OpHelp("r[P2]=r[P1]"), - /* 87 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 88 */ "CollSeq" OpHelp(""), - /* 89 */ "Function0" OpHelp("r[P3]=func(r[P2@P5])"), - /* 90 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), - /* 91 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 92 */ "RealAffinity" OpHelp(""), - /* 93 */ "Cast" OpHelp("affinity(r[P1])"), - /* 94 */ "Permutation" OpHelp(""), - /* 95 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), - /* 96 */ "Column" OpHelp("r[P3]=PX"), - /* 97 */ "String8" OpHelp("r[P2]='P4'"), - /* 98 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 99 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 100 */ "Count" OpHelp("r[P2]=count()"), - /* 101 */ "ReadCookie" OpHelp(""), - /* 102 */ "SetCookie" OpHelp(""), - /* 103 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), - /* 104 */ "OpenRead" OpHelp("root=P2 iDb=P3"), - /* 105 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), - /* 106 */ "OpenAutoindex" OpHelp("nColumn=P2"), - /* 107 */ "OpenEphemeral" OpHelp("nColumn=P2"), - /* 108 */ "SorterOpen" OpHelp(""), - /* 109 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), - /* 110 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), - /* 111 */ "Close" OpHelp(""), - /* 112 */ "ColumnsUsed" OpHelp(""), - /* 113 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), - /* 114 */ "NewRowid" OpHelp("r[P2]=rowid"), - /* 115 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), - /* 116 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), - /* 117 */ "Delete" OpHelp(""), - /* 118 */ "ResetCount" OpHelp(""), - /* 119 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), - /* 120 */ "SorterData" OpHelp("r[P2]=data"), - /* 121 */ "RowKey" OpHelp("r[P2]=key"), - /* 122 */ "RowData" OpHelp("r[P2]=data"), - /* 123 */ "Rowid" OpHelp("r[P2]=rowid"), - /* 124 */ "NullRow" OpHelp(""), - /* 125 */ "SorterInsert" OpHelp(""), - /* 126 */ "IdxInsert" OpHelp("key=r[P2]"), - /* 127 */ "IdxDelete" OpHelp("key=r[P2@P3]"), - /* 128 */ "Seek" OpHelp("Move P3 to P1.rowid"), - /* 129 */ "IdxRowid" OpHelp("r[P2]=rowid"), - /* 130 */ "Destroy" OpHelp(""), - /* 131 */ "Clear" OpHelp(""), - /* 132 */ "Real" OpHelp("r[P2]=P4"), - /* 133 */ "ResetSorter" OpHelp(""), - /* 134 */ "CreateIndex" OpHelp("r[P2]=root iDb=P1"), - /* 135 */ "CreateTable" OpHelp("r[P2]=root iDb=P1"), - /* 136 */ "ParseSchema" OpHelp(""), - /* 137 */ "LoadAnalysis" OpHelp(""), - /* 138 */ "DropTable" OpHelp(""), - /* 139 */ "DropIndex" OpHelp(""), - /* 140 */ "DropTrigger" OpHelp(""), - /* 141 */ "IntegrityCk" OpHelp(""), - /* 142 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), - /* 143 */ "Param" OpHelp(""), - /* 144 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), - /* 145 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), - /* 146 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), - /* 147 */ "AggStep0" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 148 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 149 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 150 */ "Expire" OpHelp(""), - /* 151 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 152 */ "VBegin" OpHelp(""), - /* 153 */ "VCreate" OpHelp(""), - /* 154 */ "VDestroy" OpHelp(""), - /* 155 */ "VOpen" OpHelp(""), - /* 156 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 157 */ "VRename" OpHelp(""), - /* 158 */ "Pagecount" OpHelp(""), - /* 159 */ "MaxPgcnt" OpHelp(""), - /* 160 */ "CursorHint" OpHelp(""), - /* 161 */ "Noop" OpHelp(""), - /* 162 */ "Explain" OpHelp(""), + static const char *const azName[] = { "?", + /* 1 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), + /* 2 */ "Savepoint" OpHelp(""), + /* 3 */ "AutoCommit" OpHelp(""), + /* 4 */ "Transaction" OpHelp(""), + /* 5 */ "SorterNext" OpHelp(""), + /* 6 */ "PrevIfOpen" OpHelp(""), + /* 7 */ "NextIfOpen" OpHelp(""), + /* 8 */ "Prev" OpHelp(""), + /* 9 */ "Next" OpHelp(""), + /* 10 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 11 */ "Checkpoint" OpHelp(""), + /* 12 */ "JournalMode" OpHelp(""), + /* 13 */ "Vacuum" OpHelp(""), + /* 14 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 15 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 16 */ "Goto" OpHelp(""), + /* 17 */ "Gosub" OpHelp(""), + /* 18 */ "Return" OpHelp(""), + /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), + /* 20 */ "InitCoroutine" OpHelp(""), + /* 21 */ "EndCoroutine" OpHelp(""), + /* 22 */ "Yield" OpHelp(""), + /* 23 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 24 */ "Halt" OpHelp(""), + /* 25 */ "Integer" OpHelp("r[P2]=P1"), + /* 26 */ "Int64" OpHelp("r[P2]=P4"), + /* 27 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 28 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 29 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 30 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 31 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 32 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 33 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 34 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 35 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 36 */ "CollSeq" OpHelp(""), + /* 37 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 38 */ "MustBeInt" OpHelp(""), + /* 39 */ "RealAffinity" OpHelp(""), + /* 40 */ "Cast" OpHelp("affinity(r[P1])"), + /* 41 */ "Permutation" OpHelp(""), + /* 42 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 43 */ "Jump" OpHelp(""), + /* 44 */ "Once" OpHelp(""), + /* 45 */ "If" OpHelp(""), + /* 46 */ "IfNot" OpHelp(""), + /* 47 */ "Column" OpHelp("r[P3]=PX"), + /* 48 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 49 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 50 */ "Count" OpHelp("r[P2]=count()"), + /* 51 */ "ReadCookie" OpHelp(""), + /* 52 */ "SetCookie" OpHelp(""), + /* 53 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 54 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 55 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 56 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 57 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 58 */ "SorterOpen" OpHelp(""), + /* 59 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 60 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 61 */ "Close" OpHelp(""), + /* 62 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 63 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 64 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 65 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 66 */ "Seek" OpHelp("intkey=r[P2]"), + /* 67 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 68 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 69 */ "Found" OpHelp("key=r[P3@P4]"), + /* 70 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 71 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), + /* 72 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), + /* 73 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 74 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 75 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 76 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), + /* 77 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), + /* 78 */ "Ne" OpHelp("if r[P1]!=r[P3] goto P2"), + /* 79 */ "Eq" OpHelp("if r[P1]==r[P3] goto P2"), + /* 80 */ "Gt" OpHelp("if r[P1]>r[P3] goto P2"), + /* 81 */ "Le" OpHelp("if r[P1]<=r[P3] goto P2"), + /* 82 */ "Lt" OpHelp("if r[P1]=r[P3] goto P2"), + /* 84 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), + /* 85 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 86 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 87 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), + /* 89 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 90 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 91 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 92 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 93 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 94 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), + /* 95 */ "Delete" OpHelp(""), + /* 96 */ "BitNot" OpHelp("r[P1]= ~r[P1]"), + /* 97 */ "String8" OpHelp("r[P2]='P4'"), + /* 98 */ "ResetCount" OpHelp(""), + /* 99 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 100 */ "SorterData" OpHelp("r[P2]=data"), + /* 101 */ "RowKey" OpHelp("r[P2]=key"), + /* 102 */ "RowData" OpHelp("r[P2]=data"), + /* 103 */ "Rowid" OpHelp("r[P2]=rowid"), + /* 104 */ "NullRow" OpHelp(""), + /* 105 */ "Last" OpHelp(""), + /* 106 */ "SorterSort" OpHelp(""), + /* 107 */ "Sort" OpHelp(""), + /* 108 */ "Rewind" OpHelp(""), + /* 109 */ "SorterInsert" OpHelp(""), + /* 110 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 111 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 112 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 113 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 114 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 115 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 116 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 117 */ "Destroy" OpHelp(""), + /* 118 */ "Clear" OpHelp(""), + /* 119 */ "ResetSorter" OpHelp(""), + /* 120 */ "CreateIndex" OpHelp("r[P2]=root iDb=P1"), + /* 121 */ "CreateTable" OpHelp("r[P2]=root iDb=P1"), + /* 122 */ "ParseSchema" OpHelp(""), + /* 123 */ "LoadAnalysis" OpHelp(""), + /* 124 */ "DropTable" OpHelp(""), + /* 125 */ "DropIndex" OpHelp(""), + /* 126 */ "DropTrigger" OpHelp(""), + /* 127 */ "IntegrityCk" OpHelp(""), + /* 128 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 129 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 130 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 131 */ "Program" OpHelp(""), + /* 132 */ "Param" OpHelp(""), + /* 133 */ "Real" OpHelp("r[P2]=P4"), + /* 134 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 135 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 136 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 137 */ "IfPos" OpHelp("if r[P1]>0 goto P2"), + /* 138 */ "IfNeg" OpHelp("r[P1]+=P3, if r[P1]<0 goto P2"), + /* 139 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]+=P3, goto P2"), + /* 140 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 141 */ "JumpZeroIncr" OpHelp("if (r[P1]++)==0 ) goto P2"), + /* 142 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 143 */ "IncrVacuum" OpHelp(""), + /* 144 */ "Expire" OpHelp(""), + /* 145 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 146 */ "VBegin" OpHelp(""), + /* 147 */ "VCreate" OpHelp(""), + /* 148 */ "VDestroy" OpHelp(""), + /* 149 */ "VOpen" OpHelp(""), + /* 150 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 151 */ "VNext" OpHelp(""), + /* 152 */ "VRename" OpHelp(""), + /* 153 */ "Pagecount" OpHelp(""), + /* 154 */ "MaxPgcnt" OpHelp(""), + /* 155 */ "Init" OpHelp("Start at P2"), + /* 156 */ "Noop" OpHelp(""), + /* 157 */ "Explain" OpHelp(""), }; return azName[i]; } @@ -32990,7 +28922,6 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ ** * Definitions of sqlite3_vfs objects for all locking methods ** plus implementations of sqlite3_os_init() and sqlite3_os_end(). */ -/* #include "sqliteInt.h" */ #if SQLITE_OS_UNIX /* This file is used on unix only */ /* @@ -33018,19 +28949,6 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ # endif #endif -/* Use pread() and pwrite() if they are available */ -#if defined(__APPLE__) -# define HAVE_PREAD 1 -# define HAVE_PWRITE 1 -#endif -#if defined(HAVE_PREAD64) && defined(HAVE_PWRITE64) -# undef USE_PREAD -# define USE_PREAD64 1 -#elif defined(HAVE_PREAD) && defined(HAVE_PWRITE) -# undef USE_PREAD64 -# define USE_PREAD 1 -#endif - /* ** standard include files. */ @@ -33109,11 +29027,6 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ */ #define MAX_PATHNAME 512 -/* -** Maximum supported symbolic links -*/ -#define SQLITE_MAX_SYMLINKS 100 - /* Always cast the getpid() return type for compatibility with ** kernel modules in VxWorks. */ #define osGetpid(X) (pid_t)getpid() @@ -33223,6 +29136,8 @@ static pid_t randomnessPid = 0; #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings issued */ +#define UNIXFILE_BLOCK 0x0200 /* Next SHM lock might block */ /* ** Include code that is common to all os_*.c files @@ -33266,8 +29181,8 @@ static pid_t randomnessPid = 0; */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing +/* +** hwtime.h contains inline assembler code for implementing ** high-performance timing routines. */ /************** Include hwtime.h in the middle of os_common.h ****************/ @@ -33287,8 +29202,8 @@ static pid_t randomnessPid = 0; ** This file contains inline asm code for retrieving "high-performance" ** counters for x86 class CPUs. */ -#ifndef SQLITE_HWTIME_H -#define SQLITE_HWTIME_H +#ifndef _HWTIME_H_ +#define _HWTIME_H_ /* ** The following routine only works on pentium-class (or newer) processors. @@ -33356,7 +29271,7 @@ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } #endif -#endif /* !defined(SQLITE_HWTIME_H) */ +#endif /* !defined(_HWTIME_H_) */ /************** End of hwtime.h **********************************************/ /************** Continuing where we left off in os_common.h ******************/ @@ -33377,14 +29292,14 @@ static sqlite_uint64 g_elapsed; ** of code will give us the ability to simulate a disk I/O error. This ** is used for testing the I/O recovery logic. */ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_io_error_hit; -SQLITE_API extern int sqlite3_io_error_hardhit; -SQLITE_API extern int sqlite3_io_error_pending; -SQLITE_API extern int sqlite3_io_error_persist; -SQLITE_API extern int sqlite3_io_error_benign; -SQLITE_API extern int sqlite3_diskfull_pending; -SQLITE_API extern int sqlite3_diskfull; +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ +SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ +SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ +SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ +SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ +SQLITE_API int sqlite3_diskfull_pending = 0; +SQLITE_API int sqlite3_diskfull = 0; #define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) #define SimulateIOError(CODE) \ if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ @@ -33410,17 +29325,17 @@ static void local_ioerr(){ #define SimulateIOErrorBenign(X) #define SimulateIOError(A) #define SimulateDiskfullError(A) -#endif /* defined(SQLITE_TEST) */ +#endif /* ** When testing, keep a count of the number of open files. */ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_open_file_count; +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_open_file_count = 0; #define OpenCounter(X) sqlite3_open_file_count+=(X) #else #define OpenCounter(X) -#endif /* defined(SQLITE_TEST) */ +#endif #endif /* !defined(_OS_COMMON_H_) */ @@ -33485,6 +29400,19 @@ static int posixOpen(const char *zFile, int flags, int mode){ return open(zFile, flags, mode); } +/* +** On some systems, calls to fchown() will trigger a message in a security +** log if they come from non-root processes. So avoid calling fchown() if +** we are not running as root. +*/ +static int posixFchown(int fd, uid_t uid, gid_t gid){ +#if OS_VXWORKS + return 0; +#else + return geteuid() ? 0 : fchown(fd,uid,gid); +#endif +} + /* Forward reference */ static int openDirectory(const char*, int*); static int unixGetpagesize(void); @@ -33550,7 +29478,7 @@ static struct unix_syscall { #else { "pread64", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osPread64 ((ssize_t(*)(int,void*,size_t,off64_t))aSyscall[10].pCurrent) +#define osPread64 ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[10].pCurrent) { "write", (sqlite3_syscall_ptr)write, 0 }, #define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) @@ -33568,10 +29496,10 @@ static struct unix_syscall { #else { "pwrite64", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off64_t))\ +#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off_t))\ aSyscall[13].pCurrent) - { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, + { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, #define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE @@ -33593,74 +29521,29 @@ static struct unix_syscall { { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) -#if defined(HAVE_FCHOWN) - { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, -#else - { "fchown", (sqlite3_syscall_ptr)0, 0 }, -#endif + { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) - { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, -#define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) - #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 - { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, -#else - { "mmap", (sqlite3_syscall_ptr)0, 0 }, -#endif -#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[22].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) -#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, -#else - { "munmap", (sqlite3_syscall_ptr)0, 0 }, -#endif -#define osMunmap ((void*(*)(void*,size_t))aSyscall[23].pCurrent) +#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) -#if HAVE_MREMAP && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) +#if HAVE_MREMAP { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, #else { "mremap", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[24].pCurrent) - -#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, -#else - { "getpagesize", (sqlite3_syscall_ptr)0, 0 }, -#endif -#define osGetpagesize ((int(*)(void))aSyscall[25].pCurrent) +#define osGetpagesize ((int(*)(void))aSyscall[24].pCurrent) -#if defined(HAVE_READLINK) - { "readlink", (sqlite3_syscall_ptr)readlink, 0 }, -#else - { "readlink", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) - -#if defined(HAVE_LSTAT) - { "lstat", (sqlite3_syscall_ptr)lstat, 0 }, -#else - { "lstat", (sqlite3_syscall_ptr)0, 0 }, -#endif -#define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) }; /* End of the overrideable system calls */ - -/* -** On some systems, calls to fchown() will trigger a message in a security -** log if they come from non-root processes. So avoid calling fchown() if -** we are not running as root. -*/ -static int robustFchown(int fd, uid_t uid, gid_t gid){ -#if defined(HAVE_FCHOWN) - return osGeteuid() ? 0 : osFchown(fd,uid,gid); -#else - return 0; -#endif -} - /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the ** "unix" VFSes. Return SQLITE_OK opon successfully updating the @@ -33822,14 +29705,14 @@ static int robust_open(const char *z, int f, mode_t m){ ** unixEnterLeave() */ static void unixEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } static void unixLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } #ifdef SQLITE_DEBUG static int unixMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } #endif @@ -33945,12 +29828,23 @@ static int robust_ftruncate(int h, sqlite3_int64 sz){ ** should handle ENOLCK, ENOTSUP, EOPNOTSUPP separately. */ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { - assert( (sqliteIOErr == SQLITE_IOERR_LOCK) || - (sqliteIOErr == SQLITE_IOERR_UNLOCK) || - (sqliteIOErr == SQLITE_IOERR_RDLOCK) || - (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ); switch (posixError) { - case EACCES: +#if 0 + /* At one point this code was not commented out. In theory, this branch + ** should never be hit, as this function should only be called after + ** a locking-related function (i.e. fcntl()) has returned non-zero with + ** the value of errno as the first argument. Since a system call has failed, + ** errno should be non-zero. + ** + ** Despite this, if errno really is zero, we still don't want to return + ** SQLITE_OK. The system call failed, and *some* SQLite error should be + ** propagated back to the caller. Commenting this branch out means errno==0 + ** will be handled by the "default:" case below. + */ + case 0: + return SQLITE_OK; +#endif + case EAGAIN: case ETIMEDOUT: case EBUSY: @@ -33960,9 +29854,41 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { * introspection, in which it actually means what it says */ return SQLITE_BUSY; + case EACCES: + /* EACCES is like EAGAIN during locking operations, but not any other time*/ + if( (sqliteIOErr == SQLITE_IOERR_LOCK) || + (sqliteIOErr == SQLITE_IOERR_UNLOCK) || + (sqliteIOErr == SQLITE_IOERR_RDLOCK) || + (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ){ + return SQLITE_BUSY; + } + /* else fall through */ case EPERM: return SQLITE_PERM; +#if EOPNOTSUPP!=ENOTSUP + case EOPNOTSUPP: + /* something went terribly awry, unless during file system support + * introspection, in which it actually means what it says */ +#endif +#ifdef ENOTSUP + case ENOTSUP: + /* invalid fd, unless during file system support introspection, in which + * it actually means what it says */ +#endif + case EIO: + case EBADF: + case EINVAL: + case ENOTCONN: + case ENODEV: + case ENXIO: + case ENOENT: +#ifdef ESTALE /* ESTALE is not defined on Interix systems */ + case ESTALE: +#endif + case ENOSYS: + /* these should force the client to close the file and reconnect */ + default: return sqliteIOErr; } @@ -34246,7 +30172,7 @@ static unixInodeInfo *inodeList = 0; /* ** -** This function - unixLogErrorAtLine(), is only ever called via the macro +** This function - unixLogError_x(), is only ever called via the macro ** unixLogError(). ** ** It is invoked after an error occurs in an OS function and errno has been @@ -34415,7 +30341,7 @@ static int findInodeInfo( rc = osFstat(fd, &statbuf); if( rc!=0 ){ storeLastErrno(pFile, errno); -#if defined(EOVERFLOW) && defined(SQLITE_DISABLE_LFS) +#ifdef EOVERFLOW if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; #endif return SQLITE_IOERR; @@ -34460,7 +30386,7 @@ static int findInodeInfo( if( pInode==0 ){ pInode = sqlite3_malloc64( sizeof(*pInode) ); if( pInode==0 ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } memset(pInode, 0, sizeof(*pInode)); memcpy(&pInode->fileId, &fileId, sizeof(fileId)); @@ -34502,25 +30428,30 @@ static int fileHasMoved(unixFile *pFile){ static void verifyDbFile(unixFile *pFile){ struct stat buf; int rc; - - /* These verifications occurs for the main database only */ - if( pFile->ctrlFlags & UNIXFILE_NOLOCK ) return; - + if( pFile->ctrlFlags & UNIXFILE_WARNED ){ + /* One or more of the following warnings have already been issued. Do not + ** repeat them so as not to clutter the error log */ + return; + } rc = osFstat(pFile->h, &buf); if( rc!=0 ){ sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; return; } - if( buf.st_nlink==0 ){ + if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; return; } if( buf.st_nlink>1 ){ sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; return; } if( fileHasMoved(pFile) ){ sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; return; } } @@ -34540,7 +30471,6 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); assert( pFile ); - assert( pFile->eFileLock<=SHARED_LOCK ); unixEnterMutex(); /* Because pFile->pInode is shared across threads */ /* Check if a thread in this process holds such a lock */ @@ -34597,7 +30527,9 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ unixInodeInfo *pInode = pFile->pInode; assert( unixMutexHeld() ); assert( pInode!=0 ); - if( (pFile->ctrlFlags & (UNIXFILE_EXCL|UNIXFILE_RDONLY))==UNIXFILE_EXCL ){ + if( ((pFile->ctrlFlags & UNIXFILE_EXCL)!=0 || pInode->bProcessLock) + && ((pFile->ctrlFlags & UNIXFILE_RDONLY)==0) + ){ if( pInode->bProcessLock==0 ){ struct flock lock; assert( pInode->nLock==0 ); @@ -34647,7 +30579,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** lock transitions in terms of the POSIX advisory shared and exclusive ** lock primitives (called read-locks and write-locks below, to avoid ** confusion with SQLite lock names). The algorithms are complicated - ** slightly in order to be compatible with Windows95 systems simultaneously + ** slightly in order to be compatible with windows systems simultaneously ** accessing the same database file, in case that is ever required. ** ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved @@ -34655,14 +30587,8 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** range', a range of 510 bytes at a well known offset. ** ** To obtain a SHARED lock, a read-lock is obtained on the 'pending - ** byte'. If this is successful, 'shared byte range' is read-locked - ** and the lock on the 'pending byte' released. (Legacy note: When - ** SQLite was first developed, Windows95 systems were still very common, - ** and Widnows95 lacks a shared-lock capability. So on Windows95, a - ** single randomly selected by from the 'shared byte range' is locked. - ** Windows95 is now pretty much extinct, but this work-around for the - ** lack of shared-locks on Windows95 lives on, for backwards - ** compatibility.) + ** byte'. If this is successful, a random byte from the 'shared byte + ** range' is read-locked and the lock on the 'pending byte' released. ** ** A process may only obtain a RESERVED lock after it has a SHARED lock. ** A RESERVED lock is implemented by grabbing a write-lock on the @@ -34681,6 +30607,11 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** range'. Since all other locks require a read-lock on one of the bytes ** within this range, this ensures that no other locks are held on the ** database. + ** + ** The reason a single byte cannot be used instead of the 'shared byte + ** range' is that some versions of windows do not support read-locks. By + ** locking a random byte from a range, concurrent SHARED locks may exist + ** even if the locking primitive used is always a write-lock. */ int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; @@ -34950,7 +30881,9 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ if( unixFileLock(pFile, &lock)==(-1) ){ tErrno = errno; rc = SQLITE_IOERR_UNLOCK; - storeLastErrno(pFile, tErrno); + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } goto end_unlock; } lock.l_type = F_RDLCK; @@ -34972,7 +30905,9 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ if( unixFileLock(pFile, &lock)==(-1) ){ tErrno = errno; rc = SQLITE_IOERR_UNLOCK; - storeLastErrno(pFile, tErrno); + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } goto end_unlock; } }else @@ -35223,7 +31158,17 @@ static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); assert( pFile ); - reserved = osAccess((const char*)pFile->lockingContext, 0)==0; + + /* Check if a thread in this process holds such a lock */ + if( pFile->eFileLock>SHARED_LOCK ){ + /* Either this connection or some other connection in the same process + ** holds a lock on the file. No need to check further. */ + reserved = 1; + }else{ + /* The lock is held if and only if the lockfile exists */ + const char *zLockFile = (const char*)pFile->lockingContext; + reserved = osAccess(zLockFile, 0)==0; + } OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); *pResOut = reserved; return rc; @@ -35285,7 +31230,7 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { rc = SQLITE_BUSY; } else { rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( rc!=SQLITE_BUSY ){ + if( IS_LOCK_ERROR(rc) ){ storeLastErrno(pFile, tErrno); } } @@ -35332,12 +31277,14 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { /* To fully unlock the database, delete the lock file */ assert( eFileLock==NO_LOCK ); rc = osRmdir(zLockFile); + if( rc<0 && errno==ENOTDIR ) rc = osUnlink(zLockFile); if( rc<0 ){ int tErrno = errno; - if( tErrno==ENOENT ){ - rc = SQLITE_OK; - }else{ + rc = 0; + if( ENOENT != tErrno ){ rc = SQLITE_IOERR_UNLOCK; + } + if( IS_LOCK_ERROR(rc) ){ storeLastErrno(pFile, tErrno); } return rc; @@ -35350,11 +31297,14 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { ** Close a file. Make sure the lock has been released before closing. */ static int dotlockClose(sqlite3_file *id) { - unixFile *pFile = (unixFile*)id; - assert( id!=0 ); - dotlockUnlock(id, NO_LOCK); - sqlite3_free(pFile->lockingContext); - return closeUnixFile(id); + int rc = SQLITE_OK; + if( id ){ + unixFile *pFile = (unixFile*)id; + dotlockUnlock(id, NO_LOCK); + sqlite3_free(pFile->lockingContext); + rc = closeUnixFile(id); + } + return rc; } /****************** End of the dot-file lock implementation ******************* ******************************************************************************/ @@ -35420,8 +31370,10 @@ static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ int tErrno = errno; /* unlock failed with an error */ lrc = SQLITE_IOERR_UNLOCK; - storeLastErrno(pFile, tErrno); - rc = lrc; + if( IS_LOCK_ERROR(lrc) ){ + storeLastErrno(pFile, tErrno); + rc = lrc; + } } } else { int tErrno = errno; @@ -35554,9 +31506,12 @@ static int flockUnlock(sqlite3_file *id, int eFileLock) { ** Close a file. */ static int flockClose(sqlite3_file *id) { - assert( id!=0 ); - flockUnlock(id, NO_LOCK); - return closeUnixFile(id); + int rc = SQLITE_OK; + if( id ){ + flockUnlock(id, NO_LOCK); + rc = closeUnixFile(id); + } + return rc; } #endif /* SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORK */ @@ -36181,22 +32136,23 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { */ static int afpClose(sqlite3_file *id) { int rc = SQLITE_OK; - unixFile *pFile = (unixFile*)id; - assert( id!=0 ); - afpUnlock(id, NO_LOCK); - unixEnterMutex(); - if( pFile->pInode && pFile->pInode->nLock ){ - /* If there are outstanding locks, do not actually close the file just - ** yet because that would clear those locks. Instead, add the file - ** descriptor to pInode->aPending. It will be automatically closed when - ** the last lock is cleared. - */ - setPendingFd(pFile); + if( id ){ + unixFile *pFile = (unixFile*)id; + afpUnlock(id, NO_LOCK); + unixEnterMutex(); + if( pFile->pInode && pFile->pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + setPendingFd(pFile); + } + releaseInodeInfo(pFile); + sqlite3_free(pFile->lockingContext); + rc = closeUnixFile(id); + unixLeaveMutex(); } - releaseInodeInfo(pFile); - sqlite3_free(pFile->lockingContext); - rc = closeUnixFile(id); - unixLeaveMutex(); return rc; } @@ -36266,6 +32222,7 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ TIMER_START; assert( cnt==(cnt&0x1ffff) ); assert( id->h>2 ); + cnt &= 0x1ffff; do{ #if defined(USE_PREAD) got = osPread(id->h, pBuf, cnt, offset); @@ -36275,9 +32232,13 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ SimulateIOError( got = -1 ); #else newOffset = lseek(id->h, offset, SEEK_SET); - SimulateIOError( newOffset = -1 ); - if( newOffset<0 ){ - storeLastErrno((unixFile*)id, errno); + SimulateIOError( newOffset-- ); + if( newOffset!=offset ){ + if( newOffset == -1 ){ + storeLastErrno((unixFile*)id, errno); + }else{ + storeLastErrno((unixFile*)id, 0); + } return -1; } got = osRead(id->h, pBuf, cnt); @@ -36376,7 +32337,6 @@ static int seekAndWriteFd( assert( nBuf==(nBuf&0x1ffff) ); assert( fd>2 ); - assert( piErrno!=0 ); nBuf &= 0x1ffff; TIMER_START; @@ -36387,10 +32347,11 @@ static int seekAndWriteFd( #else do{ i64 iSeek = lseek(fd, iOff, SEEK_SET); - SimulateIOError( iSeek = -1 ); - if( iSeek<0 ){ - rc = -1; - break; + SimulateIOError( iSeek-- ); + + if( iSeek!=iOff ){ + if( piErrno ) *piErrno = (iSeek==-1 ? errno : 0); + return -1; } rc = osWrite(fd, pBuf, nBuf); }while( rc<0 && errno==EINTR ); @@ -36399,7 +32360,7 @@ static int seekAndWriteFd( TIMER_END; OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); - if( rc<0 ) *piErrno = errno; + if( rc<0 && piErrno ) *piErrno = errno; return rc; } @@ -36462,7 +32423,7 @@ static int unixWrite( } #endif -#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 +#if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -36478,8 +32439,8 @@ static int unixWrite( } } #endif - - while( (wrote = seekAndWrite(pFile, offset, pBuf, amt))0 ){ + + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; offset += wrote; pBuf = &((char*)pBuf)[wrote]; @@ -36487,7 +32448,7 @@ static int unixWrite( SimulateIOError(( wrote=(-1), amt=1 )); SimulateDiskfullError(( wrote=0, amt=1 )); - if( amt>wrote ){ + if( amt>0 ){ if( wrote<0 && pFile->lastErrno!=ENOSPC ){ /* lastErrno set by seekAndWrite */ return SQLITE_IOERR_WRITE; @@ -36583,15 +32544,10 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ #endif /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a - ** no-op. But go ahead and call fstat() to validate the file - ** descriptor as we need a method to provoke a failure during - ** coverate testing. + ** no-op */ #ifdef SQLITE_NO_SYNC - { - struct stat buf; - rc = osFstat(fd, &buf); - } + rc = SQLITE_OK; #elif HAVE_FULLFSYNC if( fullSync ){ rc = osFcntl(fd, F_FULLFSYNC, 0); @@ -36657,20 +32613,16 @@ static int openDirectory(const char *zFilename, int *pFd){ char zDirname[MAX_PATHNAME+1]; sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); - for(ii=(int)strlen(zDirname); ii>0 && zDirname[ii]!='/'; ii--); + for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); if( ii>0 ){ zDirname[ii] = '\0'; - }else{ - if( zDirname[0]!='/' ) zDirname[0] = '.'; - zDirname[1] = 0; - } - fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); - if( fd>=0 ){ - OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); + if( fd>=0 ){ + OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); + } } *pFd = fd; - if( fd>=0 ) return SQLITE_OK; - return unixLogError(SQLITE_CANTOPEN_BKPT, "openDirectory", zDirname); + return (fd>=0?SQLITE_OK:unixLogError(SQLITE_CANTOPEN_BKPT, "open", zDirname)); } /* @@ -36723,11 +32675,10 @@ static int unixSync(sqlite3_file *id, int flags){ OSTRACE(("DIRSYNC %s (have_fullfsync=%d fullsync=%d)\n", pFile->zPath, HAVE_FULLFSYNC, isFullsync)); rc = osOpenDirectory(pFile->zPath, &dirfd); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && dirfd>=0 ){ full_fsync(dirfd, 0, 0); robust_close(pFile, dirfd, __LINE__); - }else{ - assert( rc==SQLITE_CANTOPEN ); + }else if( rc==SQLITE_CANTOPEN ){ rc = SQLITE_OK; } pFile->ctrlFlags &= ~UNIXFILE_DIRSYNC; @@ -36859,14 +32810,18 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ int nWrite = 0; /* Number of bytes written by seekAndWrite */ i64 iWrite; /* Next offset to write to */ - iWrite = (buf.st_size/nBlk)*nBlk + nBlk - 1; + iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; assert( iWrite>=buf.st_size ); + assert( (iWrite/nBlk)==((buf.st_size+nBlk-1)/nBlk) ); assert( ((iWrite+1)%nBlk)==0 ); - for(/*no-op*/; iWrite=nSize ) iWrite = nSize - 1; + for(/*no-op*/; iWritectrlFlags |= UNIXFILE_BLOCK; // Deferred feature */ + return SQLITE_OK; + } case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->eFileLock; return SQLITE_OK; @@ -37240,9 +33199,10 @@ static int unixShmSystemLock( assert( n==1 || lockType!=F_RDLCK ); /* Locks are within range */ - assert( n>=1 && n<=SQLITE_SHM_NLOCK ); + assert( n>=1 && nh>=0 ){ + int lkType; /* Initialize the locking parameters */ memset(&f, 0, sizeof(f)); f.l_type = lockType; @@ -37250,8 +33210,10 @@ static int unixShmSystemLock( f.l_start = ofst; f.l_len = n; - rc = osFcntl(pShmNode->h, F_SETLK, &f); + lkType = (pFile->ctrlFlags & UNIXFILE_BLOCK)!=0 ? F_SETLKW : F_SETLK; + rc = osFcntl(pShmNode->h, lkType, &f); rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; + pFile->ctrlFlags &= ~UNIXFILE_BLOCK; } /* Update the global lock state and do debug tracing */ @@ -37318,7 +33280,7 @@ static int unixShmRegionPerMap(void){ static void unixShmPurge(unixFile *pFd){ unixShmNode *p = pFd->pInode->pShmNode; assert( unixMutexHeld() ); - if( p && ALWAYS(p->nRef==0) ){ + if( p && p->nRef==0 ){ int nShmPerMap = unixShmRegionPerMap(); int i; assert( p->pInode==pFd->pInode ); @@ -37385,7 +33347,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ /* Allocate space for the new unixShm object. */ p = sqlite3_malloc64( sizeof(*p) ); - if( p==0 ) return SQLITE_NOMEM_BKPT; + if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); assert( pDbFd->pShm==0 ); @@ -37405,7 +33367,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** a new *-shm file is created, an attempt will be made to create it ** with the same permissions. */ - if( osFstat(pDbFd->h, &sStat) ){ + if( osFstat(pDbFd->h, &sStat) && pInode->bProcessLock==0 ){ rc = SQLITE_IOERR_FSTAT; goto shm_open_err; } @@ -37417,7 +33379,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ #endif pShmNode = sqlite3_malloc64( sizeof(*pShmNode) + nShmFilename ); if( pShmNode==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; goto shm_open_err; } memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); @@ -37433,12 +33395,10 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; - if( sqlite3GlobalConfig.bCoreMutex ){ - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_NOMEM_BKPT; - goto shm_open_err; - } + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_NOMEM; + goto shm_open_err; } if( pInode->bProcessLock==0 ){ @@ -37457,7 +33417,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** is owned by the same user that owns the original database. Otherwise, ** the original owner will not be able to connect. */ - robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); + osFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. @@ -37594,8 +33554,7 @@ static int unixShmMap( /* Write to the last byte of each newly allocated or extended page */ assert( (nByte % pgsz)==0 ); for(iPg=(sStat.st_size/pgsz); iPg<(nByte/pgsz); iPg++){ - int x = 0; - if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, &x)!=1 ){ + if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, 0)!=1 ){ const char *zFile = pShmNode->zFilename; rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); goto shmpage_out; @@ -37610,7 +33569,7 @@ static int unixShmMap( pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ - rc = SQLITE_IOERR_NOMEM_BKPT; + rc = SQLITE_IOERR_NOMEM; goto shmpage_out; } pShmNode->apRegion = apNew; @@ -37630,7 +33589,7 @@ static int unixShmMap( }else{ pMem = sqlite3_malloc64(szRegion); if( pMem==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; goto shmpage_out; } memset(pMem, 0, szRegion); @@ -37778,8 +33737,7 @@ static void unixShmBarrier( sqlite3_file *fd /* Database file holding the shared memory */ ){ UNUSED_PARAMETER(fd); - sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ - unixEnterMutex(); /* Also mutex, for redundancy */ + unixEnterMutex(); unixLeaveMutex(); } @@ -37889,9 +33847,7 @@ static void unixRemapfile( assert( pFd->mmapSizeActual>=pFd->mmapSize ); assert( MAP_FAILED!=0 ); -#ifdef SQLITE_MMAP_READWRITE if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; -#endif if( pOrig ){ #if HAVE_MREMAP @@ -37963,14 +33919,17 @@ static void unixRemapfile( ** recreated as a result of outstanding references) or an SQLite error ** code otherwise. */ -static int unixMapfile(unixFile *pFd, i64 nMap){ +static int unixMapfile(unixFile *pFd, i64 nByte){ + i64 nMap = nByte; + int rc; + assert( nMap>=0 || pFd->nFetchOut==0 ); - assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); if( pFd->nFetchOut>0 ) return SQLITE_OK; if( nMap<0 ){ struct stat statbuf; /* Low-level file information */ - if( osFstat(pFd->h, &statbuf) ){ + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ return SQLITE_IOERR_FSTAT; } nMap = statbuf.st_size; @@ -37979,9 +33938,12 @@ static int unixMapfile(unixFile *pFd, i64 nMap){ nMap = pFd->mmapSizeMax; } - assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); if( nMap!=pFd->mmapSize ){ - unixRemapfile(pFd, nMap); + if( nMap>0 ){ + unixRemapfile(pFd, nMap); + }else{ + unixUnmapfile(pFd); + } } return SQLITE_OK; @@ -38408,7 +34370,7 @@ static int fillInUnixFile( pNew->pId = vxworksFindFileId(zFilename); if( pNew->pId==0 ){ ctrlFlags |= UNIXFILE_NOLOCK; - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; } #endif @@ -38464,7 +34426,7 @@ static int fillInUnixFile( afpLockingContext *pCtx; pNew->lockingContext = pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; }else{ /* NB: zFilename exists and remains valid until the file is closed ** according to requirement F11141. So we do not need to make a @@ -38494,7 +34456,7 @@ static int fillInUnixFile( nFilename = (int)strlen(zFilename) + 6; zLockFile = (char *)sqlite3_malloc64(nFilename); if( zLockFile==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; }else{ sqlite3_snprintf(nFilename, zLockFile, "%s" DOTLOCK_SUFFIX, zFilename); } @@ -38517,7 +34479,7 @@ static int fillInUnixFile( if( zSemName[n]=='/' ) zSemName[n] = '_'; pNew->pInode->pSem = sem_open(zSemName, O_CREAT, 0666, 1); if( pNew->pInode->pSem == SEM_FAILED ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; pNew->pInode->aSemName[0] = '\0'; } } @@ -38550,31 +34512,29 @@ static int fillInUnixFile( */ static const char *unixTempFileDir(void){ static const char *azDirs[] = { + 0, 0, 0, "/var/tmp", "/usr/tmp", "/tmp", - "." + 0 /* List terminator */ }; - unsigned int i = 0; + unsigned int i; struct stat buf; - const char *zDir = sqlite3_temp_directory; - - if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); - if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); - while(1){ - if( zDir!=0 - && osStat(zDir, &buf)==0 - && S_ISDIR(buf.st_mode) - && osAccess(zDir, 03)==0 - ){ - return zDir; - } - if( i>=sizeof(azDirs)/sizeof(azDirs[0]) ) break; - zDir = azDirs[i++]; + const char *zDir = 0; + + azDirs[0] = sqlite3_temp_directory; + if( !azDirs[1] ) azDirs[1] = getenv("SQLITE_TMPDIR"); + if( !azDirs[2] ) azDirs[2] = getenv("TMPDIR"); + for(i=0; imxPathname bytes. */ static int unixGetTempname(int nBuf, char *zBuf){ + static const unsigned char zChars[] = + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789"; + unsigned int i, j; const char *zDir; - int iLimit = 0; /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. */ - zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); zDir = unixTempFileDir(); - if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; + if( zDir==0 ) zDir = "."; + + /* Check that the output buffer is large enough for the temporary file + ** name. If it is not, return SQLITE_ERROR. + */ + if( (strlen(zDir) + strlen(SQLITE_TEMP_FILE_PREFIX) + 18) >= (size_t)nBuf ){ + return SQLITE_ERROR; + } + do{ - u64 r; - sqlite3_randomness(sizeof(r), &r); - assert( nBuf>2 ); - zBuf[nBuf-2] = 0; - sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", - zDir, r, 0); - if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ) return SQLITE_ERROR; + sqlite3_snprintf(nBuf-18, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX, zDir); + j = (int)strlen(zBuf); + sqlite3_randomness(15, &zBuf[j]); + for(i=0; i<15; i++, j++){ + zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + } + zBuf[j] = 0; + zBuf[j+1] = 0; }while( osAccess(zBuf,0)==0 ); return SQLITE_OK; } @@ -38674,27 +34646,6 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ return pUnused; } -/* -** Find the mode, uid and gid of file zFile. -*/ -static int getFileMode( - const char *zFile, /* File name */ - mode_t *pMode, /* OUT: Permissions of zFile */ - uid_t *pUid, /* OUT: uid of zFile. */ - gid_t *pGid /* OUT: gid of zFile. */ -){ - struct stat sStat; /* Output of stat() on database file */ - int rc = SQLITE_OK; - if( 0==osStat(zFile, &sStat) ){ - *pMode = sStat.st_mode & 0777; - *pUid = sStat.st_uid; - *pGid = sStat.st_gid; - }else{ - rc = SQLITE_IOERR_FSTAT; - } - return rc; -} - /* ** This function is called by unixOpen() to determine the unix permissions ** to create new files with. If no error occurs, then SQLITE_OK is returned @@ -38730,6 +34681,7 @@ static int findCreateFileMode( if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ char zDb[MAX_PATHNAME+1]; /* Database file path */ int nDb; /* Number of valid bytes in zDb */ + struct stat sStat; /* Output of stat() on database file */ /* zPath is a path to a WAL or journal file. The following block derives ** the path to the associated database file from zPath. This block handles @@ -38744,34 +34696,28 @@ static int findCreateFileMode( ** used by the test_multiplex.c module. */ nDb = sqlite3Strlen30(zPath) - 1; +#ifdef SQLITE_ENABLE_8_3_NAMES + while( nDb>0 && sqlite3Isalnum(zPath[nDb]) ) nDb--; + if( nDb==0 || zPath[nDb]!='-' ) return SQLITE_OK; +#else while( zPath[nDb]!='-' ){ -#ifndef SQLITE_ENABLE_8_3_NAMES - /* In the normal case (8+3 filenames disabled) the journal filename - ** is guaranteed to contain a '-' character. */ assert( nDb>0 ); - assert( sqlite3Isalnum(zPath[nDb]) ); -#else - /* If 8+3 names are possible, then the journal file might not contain - ** a '-' character. So check for that case and return early. */ - if( nDb==0 || zPath[nDb]=='.' ) return SQLITE_OK; -#endif + assert( zPath[nDb]!='\n' ); nDb--; } +#endif memcpy(zDb, zPath, nDb); zDb[nDb] = '\0'; - rc = getFileMode(zDb, pMode, pUid, pGid); + if( 0==osStat(zDb, &sStat) ){ + *pMode = sStat.st_mode & 0777; + *pUid = sStat.st_uid; + *pGid = sStat.st_gid; + }else{ + rc = SQLITE_IOERR_FSTAT; + } }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ *pMode = 0600; - }else if( flags & SQLITE_OPEN_URI ){ - /* If this is a main database file and the file was opened using a URI - ** filename, check for the "modeof" parameter. If present, interpret - ** its value as a filename and try to copy the mode, uid and gid from - ** that file. */ - const char *z = sqlite3_uri_parameter(zPath, "modeof"); - if( z ){ - rc = getFileMode(z, pMode, pUid, pGid); - } } return rc; } @@ -38887,7 +34833,7 @@ static int unixOpen( }else{ pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } } p->pUnused = pUnused; @@ -38900,7 +34846,7 @@ static int unixOpen( }else if( !zName ){ /* If zName is NULL, the upper layer is requesting a temp file. */ assert(isDelete && !syncDir); - rc = unixGetTempname(pVfs->mxPathname, zTmpname); + rc = unixGetTempname(MAX_PATHNAME+2, zTmpname); if( rc!=SQLITE_OK ){ return rc; } @@ -38933,8 +34879,7 @@ static int unixOpen( } fd = robust_open(zName, openFlags, openMode); OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); - assert( !isExclusive || (openFlags & O_CREAT)!=0 ); - if( fd<0 && errno!=EISDIR && isReadWrite ){ + if( fd<0 && errno!=EISDIR && isReadWrite && !isExclusive ){ /* Failed to open the file for read/write access. Try read-only. */ flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); openFlags &= ~(O_RDWR|O_CREAT); @@ -38953,7 +34898,7 @@ static int unixOpen( ** the same as the original database. */ if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ - robustFchown(fd, uid, gid); + osFchown(fd, uid, gid); } } assert( fd>=0 ); @@ -38973,7 +34918,7 @@ static int unixOpen( zPath = sqlite3_mprintf("%s", zName); if( zPath==0 ){ robust_close(p, fd, __LINE__); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } #else osUnlink(zName); @@ -38984,6 +34929,9 @@ static int unixOpen( p->openFlags = openFlags; } #endif + + noLock = eType!=SQLITE_OPEN_MAIN_DB; + #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE if( fstatfs(fd, &fsInfo) == -1 ){ @@ -39002,7 +34950,6 @@ static int unixOpen( /* Set up appropriate ctrlFlags */ if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; - noLock = eType!=SQLITE_OPEN_MAIN_DB; if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; if( syncDir ) ctrlFlags |= UNIXFILE_DIRSYNC; if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; @@ -39079,12 +35026,16 @@ static int unixDelete( int fd; rc = osOpenDirectory(zPath, &fd); if( rc==SQLITE_OK ){ - if( full_fsync(fd,0,0) ){ +#if OS_VXWORKS + if( fsync(fd)==-1 ) +#else + if( fsync(fd) ) +#endif + { rc = unixLogError(SQLITE_IOERR_DIR_FSYNC, "fsync", zPath); } robust_close(0, fd, __LINE__); - }else{ - assert( rc==SQLITE_CANTOPEN ); + }else if( rc==SQLITE_CANTOPEN ){ rc = SQLITE_OK; } } @@ -39108,50 +35059,34 @@ static int unixAccess( int flags, /* What do we want to learn about the zPath file? */ int *pResOut /* Write result boolean here */ ){ + int amode = 0; UNUSED_PARAMETER(NotUsed); SimulateIOError( return SQLITE_IOERR_ACCESS; ); - assert( pResOut!=0 ); - - /* The spec says there are three possible values for flags. But only - ** two of them are actually used */ - assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE ); + switch( flags ){ + case SQLITE_ACCESS_EXISTS: + amode = F_OK; + break; + case SQLITE_ACCESS_READWRITE: + amode = W_OK|R_OK; + break; + case SQLITE_ACCESS_READ: + amode = R_OK; + break; - if( flags==SQLITE_ACCESS_EXISTS ){ - struct stat buf; - *pResOut = (0==osStat(zPath, &buf) && buf.st_size>0); - }else{ - *pResOut = osAccess(zPath, W_OK|R_OK)==0; + default: + assert(!"Invalid flags argument"); } - return SQLITE_OK; -} - -/* -** -*/ -static int mkFullPathname( - const char *zPath, /* Input path */ - char *zOut, /* Output buffer */ - int nOut /* Allocated size of buffer zOut */ -){ - int nPath = sqlite3Strlen30(zPath); - int iOff = 0; - if( zPath[0]!='/' ){ - if( osGetcwd(zOut, nOut-2)==0 ){ - return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); + *pResOut = (osAccess(zPath, amode)==0); + if( flags==SQLITE_ACCESS_EXISTS && *pResOut ){ + struct stat buf; + if( 0==osStat(zPath, &buf) && buf.st_size==0 ){ + *pResOut = 0; } - iOff = sqlite3Strlen30(zOut); - zOut[iOff++] = '/'; - } - if( (iOff+nPath+1)>nOut ){ - /* SQLite assumes that xFullPathname() nul-terminates the output buffer - ** even if it returns an error. */ - zOut[iOff] = '\0'; - return SQLITE_CANTOPEN_BKPT; } - sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); return SQLITE_OK; } + /* ** Turn a relative pathname into a full pathname. The relative path ** is stored as a nul-terminated string in the buffer pointed to by @@ -39167,17 +35102,6 @@ static int unixFullPathname( int nOut, /* Size of output buffer in bytes */ char *zOut /* Output buffer */ ){ -#if !defined(HAVE_READLINK) || !defined(HAVE_LSTAT) - return mkFullPathname(zPath, zOut, nOut); -#else - int rc = SQLITE_OK; - int nByte; - int nLink = 1; /* Number of symbolic links followed so far */ - const char *zIn = zPath; /* Input path for each iteration of loop */ - char *zDel = 0; - - assert( pVfs->mxPathname==MAX_PATHNAME ); - UNUSED_PARAMETER(pVfs); /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -39186,62 +35110,21 @@ static int unixFullPathname( */ SimulateIOError( return SQLITE_ERROR ); - do { - - /* Call stat() on path zIn. Set bLink to true if the path is a symbolic - ** link, or false otherwise. */ - int bLink = 0; - struct stat buf; - if( osLstat(zIn, &buf)!=0 ){ - if( errno!=ENOENT ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "lstat", zIn); - } - }else{ - bLink = S_ISLNK(buf.st_mode); - } - - if( bLink ){ - if( zDel==0 ){ - zDel = sqlite3_malloc(nOut); - if( zDel==0 ) rc = SQLITE_NOMEM_BKPT; - }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ - rc = SQLITE_CANTOPEN_BKPT; - } - - if( rc==SQLITE_OK ){ - nByte = osReadlink(zIn, zDel, nOut-1); - if( nByte<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zIn); - }else{ - if( zDel[0]!='/' ){ - int n; - for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); - if( nByte+n+1>nOut ){ - rc = SQLITE_CANTOPEN_BKPT; - }else{ - memmove(&zDel[n], zDel, nByte+1); - memcpy(zDel, zIn, n); - nByte += n; - } - } - zDel[nByte] = '\0'; - } - } - - zIn = zDel; - } + assert( pVfs->mxPathname==MAX_PATHNAME ); + UNUSED_PARAMETER(pVfs); - assert( rc!=SQLITE_OK || zIn!=zOut || zIn[0]=='/' ); - if( rc==SQLITE_OK && zIn!=zOut ){ - rc = mkFullPathname(zIn, zOut, nOut); + zOut[nOut-1] = '\0'; + if( zPath[0]=='/' ){ + sqlite3_snprintf(nOut, zOut, "%s", zPath); + }else{ + int nCwd; + if( osGetcwd(zOut, nOut-1)==0 ){ + return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); } - if( bLink==0 ) break; - zIn = zOut; - }while( rc==SQLITE_OK ); - - sqlite3_free(zDel); - return rc; -#endif /* HAVE_READLINK && HAVE_LSTAT */ + nCwd = (int)strlen(zOut); + sqlite3_snprintf(nOut-nCwd, &zOut[nCwd], "/%s", zPath); + } + return SQLITE_OK; } @@ -39410,8 +35293,11 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_nsec/1000000; #else struct timeval sNow; - (void)gettimeofday(&sNow, 0); /* Cannot fail given valid arguments */ - *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; + if( gettimeofday(&sNow, 0)==0 ){ + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; + }else{ + rc = SQLITE_ERROR; + } #endif #ifdef SQLITE_TEST @@ -39423,7 +35309,6 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ return rc; } -#ifndef SQLITE_OMIT_DEPRECATED /* ** Find the current time (in Universal Coordinated Time). Write the ** current time and date as a Julian Day number into *prNow and @@ -39437,21 +35322,19 @@ static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ *prNow = i/86400000.0; return rc; } -#else -# define unixCurrentTime 0 -#endif /* -** The xGetLastError() method is designed to return a better -** low-level error message when operating-system problems come up -** during SQLite operation. Only the integer return code is currently -** used. +** We added the xGetLastError() method with the intention of providing +** better low-level error messages when operating-system problems come up +** during SQLite operation. But so far, none of that has been implemented +** in the core. So this routine is never called. For now, it is merely +** a place-holder. */ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ UNUSED_PARAMETER(NotUsed); UNUSED_PARAMETER(NotUsed2); UNUSED_PARAMETER(NotUsed3); - return errno; + return 0; } @@ -39706,7 +35589,7 @@ static int proxyCreateLockPath(const char *lockPath){ } buf[i] = lockPath[i]; } - OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n",lockPath,osGetpid(0))); + OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n", lockPath, osGetpid(0))); return 0; } @@ -39742,7 +35625,7 @@ static int proxyCreateUnixFile( }else{ pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } } if( fd<0 ){ @@ -39775,7 +35658,7 @@ static int proxyCreateUnixFile( pNew = (unixFile *)sqlite3_malloc64(sizeof(*pNew)); if( pNew==NULL ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; goto end_create_proxy; } memset(pNew, 0, sizeof(unixFile)); @@ -40118,7 +36001,7 @@ static int proxyTakeConch(unixFile *pFile){ writeSize = PROXY_PATHINDEX + strlen(&writeBuffer[PROXY_PATHINDEX]); robust_ftruncate(conchFile->h, writeSize); rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); - full_fsync(conchFile->h,0,0); + fsync(conchFile->h); /* If we created a new conch file (not just updated the contents of a ** valid conch file), try to match the permissions of the database */ @@ -40188,7 +36071,7 @@ static int proxyTakeConch(unixFile *pFile){ if( tempLockPath ){ pCtx->lockProxyPath = sqlite3DbStrDup(0, tempLockPath); if( !pCtx->lockProxyPath ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; } } } @@ -40253,7 +36136,7 @@ static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ ** the name of the original database file. */ *pConchPath = conchPath = (char *)sqlite3_malloc64(len + 8); if( conchPath==0 ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } memcpy(conchPath, dbPath, len+1); @@ -40369,7 +36252,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } memset(pCtx, 0, sizeof(*pCtx)); @@ -40405,7 +36288,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { if( rc==SQLITE_OK ){ pCtx->dbPath = sqlite3DbStrDup(0, dbPath); if( pCtx->dbPath==NULL ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; } } if( rc==SQLITE_OK ){ @@ -40591,7 +36474,7 @@ static int proxyUnlock(sqlite3_file *id, int eFileLock) { ** Close a file that uses proxy locks. */ static int proxyClose(sqlite3_file *id) { - if( ALWAYS(id) ){ + if( id ){ unixFile *pFile = (unixFile*)id; proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; unixFile *lockProxy = pCtx->lockProxy; @@ -40652,7 +36535,7 @@ static int proxyClose(sqlite3_file *id) { ** necessarily been initialized when this routine is called, and so they ** should not be used. */ -SQLITE_API int sqlite3_os_init(void){ +SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void){ /* ** The following macro defines an initializer for an sqlite3_vfs object. ** The name of the VFS is NAME. The pAppData is a pointer to a pointer @@ -40735,7 +36618,7 @@ SQLITE_API int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==28 ); + assert( ArraySize(aSyscall)==25 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ @@ -40751,7 +36634,7 @@ SQLITE_API int sqlite3_os_init(void){ ** to release dynamically allocated objects. But not on unix. ** This routine is a no-op for unix. */ -SQLITE_API int sqlite3_os_end(void){ +SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void){ return SQLITE_OK; } @@ -40773,7 +36656,6 @@ SQLITE_API int sqlite3_os_end(void){ ** ** This file contains code that is specific to Windows. */ -/* #include "sqliteInt.h" */ #if SQLITE_OS_WIN /* This file is used for Windows only */ /* @@ -40818,8 +36700,8 @@ SQLITE_API int sqlite3_os_end(void){ */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing +/* +** hwtime.h contains inline assembler code for implementing ** high-performance timing routines. */ /************** Include hwtime.h in the middle of os_common.h ****************/ @@ -40839,8 +36721,8 @@ SQLITE_API int sqlite3_os_end(void){ ** This file contains inline asm code for retrieving "high-performance" ** counters for x86 class CPUs. */ -#ifndef SQLITE_HWTIME_H -#define SQLITE_HWTIME_H +#ifndef _HWTIME_H_ +#define _HWTIME_H_ /* ** The following routine only works on pentium-class (or newer) processors. @@ -40908,7 +36790,7 @@ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } #endif -#endif /* !defined(SQLITE_HWTIME_H) */ +#endif /* !defined(_HWTIME_H_) */ /************** End of hwtime.h **********************************************/ /************** Continuing where we left off in os_common.h ******************/ @@ -40929,14 +36811,14 @@ static sqlite_uint64 g_elapsed; ** of code will give us the ability to simulate a disk I/O error. This ** is used for testing the I/O recovery logic. */ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_io_error_hit; -SQLITE_API extern int sqlite3_io_error_hardhit; -SQLITE_API extern int sqlite3_io_error_pending; -SQLITE_API extern int sqlite3_io_error_persist; -SQLITE_API extern int sqlite3_io_error_benign; -SQLITE_API extern int sqlite3_diskfull_pending; -SQLITE_API extern int sqlite3_diskfull; +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ +SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ +SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ +SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ +SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ +SQLITE_API int sqlite3_diskfull_pending = 0; +SQLITE_API int sqlite3_diskfull = 0; #define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) #define SimulateIOError(CODE) \ if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ @@ -40962,17 +36844,17 @@ static void local_ioerr(){ #define SimulateIOErrorBenign(X) #define SimulateIOError(A) #define SimulateDiskfullError(A) -#endif /* defined(SQLITE_TEST) */ +#endif /* ** When testing, keep a count of the number of open files. */ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_open_file_count; +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_open_file_count = 0; #define OpenCounter(X) sqlite3_open_file_count+=(X) #else #define OpenCounter(X) -#endif /* defined(SQLITE_TEST) */ +#endif #endif /* !defined(_OS_COMMON_H_) */ @@ -40982,7 +36864,6 @@ SQLITE_API extern int sqlite3_open_file_count; /* ** Include the header file for the Windows VFS. */ -/* #include "os_win.h" */ /* ** Compiling and using WAL mode requires several APIs that are only @@ -41035,10 +36916,6 @@ SQLITE_API extern int sqlite3_open_file_count; # define NTDDI_WINBLUE 0x06030000 #endif -#ifndef NTDDI_WINTHRESHOLD -# define NTDDI_WINTHRESHOLD 0x06040000 -#endif - /* ** Check to see if the GetVersionEx[AW] functions are deprecated on the ** target system. GetVersionEx was first deprecated in Win8.1. @@ -41051,19 +36928,6 @@ SQLITE_API extern int sqlite3_open_file_count; # endif #endif -/* -** Check to see if the CreateFileMappingA function is supported on the -** target system. It is unavailable when using "mincore.lib" on Win10. -** When compiling for Windows 10, always assume "mincore.lib" is in use. -*/ -#ifndef SQLITE_WIN32_CREATEFILEMAPPINGA -# if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WINTHRESHOLD -# define SQLITE_WIN32_CREATEFILEMAPPINGA 0 -# else -# define SQLITE_WIN32_CREATEFILEMAPPINGA 1 -# endif -#endif - /* ** This constant should already be defined (in the "WinDef.h" SDK file). */ @@ -41249,17 +37113,6 @@ struct winFile { #endif }; -/* -** The winVfsAppData structure is used for the pAppData member for all of the -** Win32 VFS variants. -*/ -typedef struct winVfsAppData winVfsAppData; -struct winVfsAppData { - const sqlite3_io_methods *pMethod; /* The file I/O methods to use. */ - void *pAppData; /* The extra pAppData, if any. */ - BOOL bNoLock; /* Non-zero if locking is disabled. */ -}; - /* ** Allowed values for winFile.ctrlFlags */ @@ -41314,23 +37167,11 @@ struct winVfsAppData { # define SQLITE_WIN32_HEAP_CREATE (TRUE) #endif -/* - * This is cache size used in the calculation of the initial size of the - * Win32-specific heap. It cannot be negative. - */ -#ifndef SQLITE_WIN32_CACHE_SIZE -# if SQLITE_DEFAULT_CACHE_SIZE>=0 -# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) -# else -# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) -# endif -#endif - /* * The initial size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_INIT_SIZE -# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_DEFAULT_CACHE_SIZE) * \ (SQLITE_DEFAULT_PAGE_SIZE) + 4194304) #endif @@ -41493,9 +37334,8 @@ static struct win_syscall { #define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ - (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) && \ - SQLITE_WIN32_CREATEFILEMAPPINGA +#if (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, #else { "CreateFileMappingA", (SYSCALL)0, 0 }, @@ -41725,7 +37565,8 @@ static struct win_syscall { #define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) -#if defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_GETVERSIONEX +#if defined(SQLITE_WIN32_HAS_ANSI) && defined(SQLITE_WIN32_GETVERSIONEX) && \ + SQLITE_WIN32_GETVERSIONEX { "GetVersionExA", (SYSCALL)GetVersionExA, 0 }, #else { "GetVersionExA", (SYSCALL)0, 0 }, @@ -41735,7 +37576,7 @@ static struct win_syscall { LPOSVERSIONINFOA))aSyscall[34].pCurrent) #if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - SQLITE_WIN32_GETVERSIONEX + defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX { "GetVersionExW", (SYSCALL)GetVersionExW, 0 }, #else { "GetVersionExW", (SYSCALL)0, 0 }, @@ -42186,7 +38027,7 @@ static const char *winNextSystemCall(sqlite3_vfs *p, const char *zName){ ** "pnLargest" argument, if non-zero, will be used to return the size of the ** largest committed free block in the heap, in bytes. */ -SQLITE_API int sqlite3_win32_compact_heap(LPUINT pnLargest){ +SQLITE_API int SQLITE_STDCALL sqlite3_win32_compact_heap(LPUINT pnLargest){ int rc = SQLITE_OK; UINT nLargest = 0; HANDLE hHeap; @@ -42204,7 +38045,7 @@ SQLITE_API int sqlite3_win32_compact_heap(LPUINT pnLargest){ if( lastErrno==NO_ERROR ){ sqlite3_log(SQLITE_NOMEM, "failed to HeapCompact (no space), heap=%p", (void*)hHeap); - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; }else{ sqlite3_log(SQLITE_ERROR, "failed to HeapCompact (%lu), heap=%p", osGetLastError(), (void*)hHeap); @@ -42226,12 +38067,12 @@ SQLITE_API int sqlite3_win32_compact_heap(LPUINT pnLargest){ ** the sqlite3_memory_used() function does not return zero, SQLITE_BUSY will ** be returned and no changes will be made to the Win32 native heap. */ -SQLITE_API int sqlite3_win32_reset_heap(){ +SQLITE_API int SQLITE_STDCALL sqlite3_win32_reset_heap(){ int rc; MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ MUTEX_LOGIC( sqlite3_mutex *pMem; ) /* The memsys static mutex */ - MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) - MUTEX_LOGIC( pMem = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); ) + MUTEX_LOGIC( pMaster = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); ) + MUTEX_LOGIC( pMem = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM); ) sqlite3_mutex_enter(pMaster); sqlite3_mutex_enter(pMem); winMemAssertMagic(); @@ -42271,17 +38112,11 @@ SQLITE_API int sqlite3_win32_reset_heap(){ ** (if available). */ -SQLITE_API void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ +SQLITE_API void SQLITE_STDCALL sqlite3_win32_write_debug(const char *zBuf, int nBuf){ char zDbgBuf[SQLITE_WIN32_DBG_BUF_SIZE]; int nMin = MIN(nBuf, (SQLITE_WIN32_DBG_BUF_SIZE - 1)); /* may be negative. */ if( nMin<-1 ) nMin = -1; /* all negative values become -1. */ assert( nMin==-1 || nMin==0 || nMin0 ){ memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); @@ -42317,7 +38152,7 @@ SQLITE_API void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ static HANDLE sleepObj = NULL; #endif -SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds){ +SQLITE_API void SQLITE_STDCALL sqlite3_win32_sleep(DWORD milliseconds){ #if SQLITE_OS_WINRT if ( sleepObj==NULL ){ sleepObj = osCreateEventExW(NULL, NULL, CREATE_EVENT_MANUAL_RESET, @@ -42352,7 +38187,7 @@ SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject){ ** the LockFileEx() API. */ -#if !SQLITE_WIN32_GETVERSIONEX +#if !defined(SQLITE_WIN32_GETVERSIONEX) || !SQLITE_WIN32_GETVERSIONEX # define osIsNT() (1) #elif SQLITE_OS_WINCE || SQLITE_OS_WINRT || !defined(SQLITE_WIN32_HAS_ANSI) # define osIsNT() (1) @@ -42366,14 +38201,14 @@ SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject){ ** This function determines if the machine is running a version of Windows ** based on the NT kernel. */ -SQLITE_API int sqlite3_win32_is_nt(void){ +SQLITE_API int SQLITE_STDCALL sqlite3_win32_is_nt(void){ #if SQLITE_OS_WINRT /* ** NOTE: The WinRT sub-platform is always assumed to be based on the NT ** kernel. */ return 1; -#elif SQLITE_WIN32_GETVERSIONEX +#elif defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ #if defined(SQLITE_WIN32_HAS_ANSI) OSVERSIONINFOA sInfo; @@ -42530,7 +38365,7 @@ static int winMemInit(void *pAppData){ "failed to HeapCreate (%lu), flags=%u, initSize=%lu, maxSize=%lu", osGetLastError(), SQLITE_WIN32_HEAP_FLAGS, dwInitialSize, dwMaximumSize); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } pWinMemData->bOwned = TRUE; assert( pWinMemData->bOwned ); @@ -42540,7 +38375,7 @@ static int winMemInit(void *pAppData){ if( !pWinMemData->hHeap ){ sqlite3_log(SQLITE_NOMEM, "failed to GetProcessHeap (%lu)", osGetLastError()); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } pWinMemData->bOwned = FALSE; assert( !pWinMemData->bOwned ); @@ -42607,244 +38442,147 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ #endif /* SQLITE_WIN32_MALLOC */ /* -** Convert a UTF-8 string to Microsoft Unicode. +** Convert a UTF-8 string to Microsoft Unicode (UTF-16?). ** -** Space to hold the returned string is obtained from sqlite3_malloc(). +** Space to hold the returned string is obtained from malloc. */ -static LPWSTR winUtf8ToUnicode(const char *zText){ +static LPWSTR winUtf8ToUnicode(const char *zFilename){ int nChar; - LPWSTR zWideText; + LPWSTR zWideFilename; - nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0); + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); if( nChar==0 ){ return 0; } - zWideText = sqlite3MallocZero( nChar*sizeof(WCHAR) ); - if( zWideText==0 ){ + zWideFilename = sqlite3MallocZero( nChar*sizeof(zWideFilename[0]) ); + if( zWideFilename==0 ){ return 0; } - nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText, + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, nChar); if( nChar==0 ){ - sqlite3_free(zWideText); - zWideText = 0; + sqlite3_free(zWideFilename); + zWideFilename = 0; } - return zWideText; + return zWideFilename; } /* -** Convert a Microsoft Unicode string to UTF-8. -** -** Space to hold the returned string is obtained from sqlite3_malloc(). +** Convert Microsoft Unicode to UTF-8. Space to hold the returned string is +** obtained from sqlite3_malloc(). */ -static char *winUnicodeToUtf8(LPCWSTR zWideText){ +static char *winUnicodeToUtf8(LPCWSTR zWideFilename){ int nByte; - char *zText; + char *zFilename; - nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0); + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, 0, 0, 0, 0); if( nByte == 0 ){ return 0; } - zText = sqlite3MallocZero( nByte ); - if( zText==0 ){ + zFilename = sqlite3MallocZero( nByte ); + if( zFilename==0 ){ return 0; } - nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte, + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, zFilename, nByte, 0, 0); if( nByte == 0 ){ - sqlite3_free(zText); - zText = 0; + sqlite3_free(zFilename); + zFilename = 0; } - return zText; + return zFilename; } /* -** Convert an ANSI string to Microsoft Unicode, using the ANSI or OEM -** code page. +** Convert an ANSI string to Microsoft Unicode, based on the +** current codepage settings for file apis. ** -** Space to hold the returned string is obtained from sqlite3_malloc(). +** Space to hold the returned string is obtained +** from sqlite3_malloc. */ -static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){ +static LPWSTR winMbcsToUnicode(const char *zFilename){ int nByte; - LPWSTR zMbcsText; - int codepage = useAnsi ? CP_ACP : CP_OEMCP; + LPWSTR zMbcsFilename; + int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; - nByte = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, + nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, NULL, 0)*sizeof(WCHAR); if( nByte==0 ){ return 0; } - zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) ); - if( zMbcsText==0 ){ + zMbcsFilename = sqlite3MallocZero( nByte*sizeof(zMbcsFilename[0]) ); + if( zMbcsFilename==0 ){ return 0; } - nByte = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, + nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, zMbcsFilename, nByte); if( nByte==0 ){ - sqlite3_free(zMbcsText); - zMbcsText = 0; + sqlite3_free(zMbcsFilename); + zMbcsFilename = 0; } - return zMbcsText; + return zMbcsFilename; } /* -** Convert a Microsoft Unicode string to a multi-byte character string, -** using the ANSI or OEM code page. +** Convert Microsoft Unicode to multi-byte character string, based on the +** user's ANSI codepage. ** -** Space to hold the returned string is obtained from sqlite3_malloc(). +** Space to hold the returned string is obtained from +** sqlite3_malloc(). */ -static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){ +static char *winUnicodeToMbcs(LPCWSTR zWideFilename){ int nByte; - char *zText; - int codepage = useAnsi ? CP_ACP : CP_OEMCP; + char *zFilename; + int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; - nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, 0, 0, 0, 0); + nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, 0, 0, 0, 0); if( nByte == 0 ){ return 0; } - zText = sqlite3MallocZero( nByte ); - if( zText==0 ){ + zFilename = sqlite3MallocZero( nByte ); + if( zFilename==0 ){ return 0; } - nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, zText, + nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, zFilename, nByte, 0, 0); if( nByte == 0 ){ - sqlite3_free(zText); - zText = 0; + sqlite3_free(zFilename); + zFilename = 0; } - return zText; + return zFilename; } /* -** Convert a multi-byte character string to UTF-8. -** -** Space to hold the returned string is obtained from sqlite3_malloc(). +** Convert multibyte character string to UTF-8. Space to hold the +** returned string is obtained from sqlite3_malloc(). */ -static char *winMbcsToUtf8(const char *zText, int useAnsi){ - char *zTextUtf8; +SQLITE_API char *SQLITE_STDCALL sqlite3_win32_mbcs_to_utf8(const char *zFilename){ + char *zFilenameUtf8; LPWSTR zTmpWide; - zTmpWide = winMbcsToUnicode(zText, useAnsi); + zTmpWide = winMbcsToUnicode(zFilename); if( zTmpWide==0 ){ return 0; } - zTextUtf8 = winUnicodeToUtf8(zTmpWide); + zFilenameUtf8 = winUnicodeToUtf8(zTmpWide); sqlite3_free(zTmpWide); - return zTextUtf8; + return zFilenameUtf8; } /* -** Convert a UTF-8 string to a multi-byte character string. -** -** Space to hold the returned string is obtained from sqlite3_malloc(). +** Convert UTF-8 to multibyte character string. Space to hold the +** returned string is obtained from sqlite3_malloc(). */ -static char *winUtf8ToMbcs(const char *zText, int useAnsi){ - char *zTextMbcs; +SQLITE_API char *SQLITE_STDCALL sqlite3_win32_utf8_to_mbcs(const char *zFilename){ + char *zFilenameMbcs; LPWSTR zTmpWide; - zTmpWide = winUtf8ToUnicode(zText); + zTmpWide = winUtf8ToUnicode(zFilename); if( zTmpWide==0 ){ return 0; } - zTextMbcs = winUnicodeToMbcs(zTmpWide, useAnsi); + zFilenameMbcs = winUnicodeToMbcs(zTmpWide); sqlite3_free(zTmpWide); - return zTextMbcs; -} - -/* -** This is a public wrapper for the winUtf8ToUnicode() function. -*/ -SQLITE_API LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !zText ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif -#ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return 0; -#endif - return winUtf8ToUnicode(zText); -} - -/* -** This is a public wrapper for the winUnicodeToUtf8() function. -*/ -SQLITE_API char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !zWideText ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif -#ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return 0; -#endif - return winUnicodeToUtf8(zWideText); -} - -/* -** This is a public wrapper for the winMbcsToUtf8() function. -*/ -SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zText){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !zText ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif -#ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return 0; -#endif - return winMbcsToUtf8(zText, osAreFileApisANSI()); -} - -/* -** This is a public wrapper for the winMbcsToUtf8() function. -*/ -SQLITE_API char *sqlite3_win32_mbcs_to_utf8_v2(const char *zText, int useAnsi){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !zText ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif -#ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return 0; -#endif - return winMbcsToUtf8(zText, useAnsi); -} - -/* -** This is a public wrapper for the winUtf8ToMbcs() function. -*/ -SQLITE_API char *sqlite3_win32_utf8_to_mbcs(const char *zText){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !zText ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif -#ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return 0; -#endif - return winUtf8ToMbcs(zText, osAreFileApisANSI()); -} - -/* -** This is a public wrapper for the winUtf8ToMbcs() function. -*/ -SQLITE_API char *sqlite3_win32_utf8_to_mbcs_v2(const char *zText, int useAnsi){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !zText ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif -#ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return 0; -#endif - return winUtf8ToMbcs(zText, useAnsi); + return zFilenameMbcs; } /* @@ -42854,7 +38592,7 @@ SQLITE_API char *sqlite3_win32_utf8_to_mbcs_v2(const char *zText, int useAnsi){ ** argument is the name of the directory to use. The return value will be ** SQLITE_OK if successful. */ -SQLITE_API int sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){ +SQLITE_API int SQLITE_STDCALL sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){ char **ppDirectory = 0; #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); @@ -42874,7 +38612,7 @@ SQLITE_API int sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){ if( zValue && zValue[0] ){ zValueUtf8 = winUnicodeToUtf8(zValue); if ( zValueUtf8==0 ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } } sqlite3_free(*ppDirectory); @@ -42946,7 +38684,7 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ if( dwLen > 0 ){ /* allocate a buffer and convert to UTF8 */ sqlite3BeginBenignMalloc(); - zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); + zOut = sqlite3_win32_mbcs_to_utf8(zTemp); sqlite3EndBenignMalloc(); /* free the system buffer allocated by FormatMessage */ osLocalFree(zTemp); @@ -43088,17 +38826,16 @@ static void winLogIoerr(int nRetry, int lineno){ } } -/* -** This #if does not rely on the SQLITE_OS_WINCE define because the -** corresponding section in "date.c" cannot use it. +#if SQLITE_OS_WINCE +/************************************************************************* +** This section contains code for WinCE only. */ -#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ - (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) +#if !defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API /* -** The MSVC CRT on Windows CE may not have a localtime() function. -** So define a substitute. +** The MSVC CRT on Windows CE may not have a localtime() function. So +** create a substitute. */ -/* # include */ +/* #include */ struct tm *__cdecl localtime(const time_t *t) { static struct tm y; @@ -43122,10 +38859,6 @@ struct tm *__cdecl localtime(const time_t *t) } #endif -#if SQLITE_OS_WINCE -/************************************************************************* -** This section contains code for WinCE only. -*/ #define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] /* @@ -43156,7 +38889,7 @@ static int winceCreateLock(const char *zFilename, winFile *pFile){ zName = winUtf8ToUnicode(zFilename); if( zName==0 ){ /* out of memory */ - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } /* Initialize the local lockdata */ @@ -43581,12 +39314,7 @@ static int winClose(sqlite3_file *id){ }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (sqlite3_win32_sleep(100), 1) ); #if SQLITE_OS_WINCE #define WINCE_DELETION_ATTEMPTS 3 - { - winVfsAppData *pAppData = (winVfsAppData*)pFile->pVfs->pAppData; - if( pAppData==NULL || !pAppData->bNoLock ){ - winceDestroyLock(pFile); - } - } + winceDestroyLock(pFile); if( pFile->zDeleteOnClose ){ int cnt = 0; while( @@ -43714,7 +39442,7 @@ static int winWrite( "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile, pFile->h, pBuf, amt, offset, pFile->locktype)); -#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 +#if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -44127,12 +39855,6 @@ static int winLock(sqlite3_file *id, int locktype){ return SQLITE_OK; } - /* Do not allow any kind of write-lock on a read-only database - */ - if( (pFile->ctrlFlags & WINFILE_RDONLY)!=0 && locktype>=RESERVED_LOCK ){ - return SQLITE_IOERR_LOCK; - } - /* Make sure the locking sequence is correct */ assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); @@ -44144,8 +39866,9 @@ static int winLock(sqlite3_file *id, int locktype){ ** the PENDING_LOCK byte is temporary. */ newLocktype = pFile->locktype; - if( pFile->locktype==NO_LOCK - || (locktype==EXCLUSIVE_LOCK && pFile->locktype<=RESERVED_LOCK) + if( (pFile->locktype==NO_LOCK) + || ( (locktype==EXCLUSIVE_LOCK) + && (pFile->locktype==RESERVED_LOCK)) ){ int cnt = 3; while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, @@ -44261,7 +39984,7 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ res = 1; OSTRACE(("TEST-WR-LOCK file=%p, result=%d (local)\n", pFile->h, res)); }else{ - res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE,0,1,0); + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); if( res ){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } @@ -44318,44 +40041,6 @@ static int winUnlock(sqlite3_file *id, int locktype){ return rc; } -/****************************************************************************** -****************************** No-op Locking ********************************** -** -** Of the various locking implementations available, this is by far the -** simplest: locking is ignored. No attempt is made to lock the database -** file for reading or writing. -** -** This locking mode is appropriate for use on read-only databases -** (ex: databases that are burned into CD-ROM, for example.) It can -** also be used if the application employs some external mechanism to -** prevent simultaneous access of the same database by two or more -** database connections. But there is a serious risk of database -** corruption if this locking mode is used in situations where multiple -** database connections are accessing the same database file at the same -** time and one or more of those connections are writing. -*/ - -static int winNolockLock(sqlite3_file *id, int locktype){ - UNUSED_PARAMETER(id); - UNUSED_PARAMETER(locktype); - return SQLITE_OK; -} - -static int winNolockCheckReservedLock(sqlite3_file *id, int *pResOut){ - UNUSED_PARAMETER(id); - UNUSED_PARAMETER(pResOut); - return SQLITE_OK; -} - -static int winNolockUnlock(sqlite3_file *id, int locktype){ - UNUSED_PARAMETER(id); - UNUSED_PARAMETER(locktype); - return SQLITE_OK; -} - -/******************* End of the no-op lock implementation ********************* -******************************************************************************/ - /* ** If *pArg is initially negative then this is a query. Set *pArg to ** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. @@ -44389,7 +40074,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } - case SQLITE_FCNTL_LAST_ERRNO: { + case SQLITE_LAST_ERRNO: { *(int*)pArg = (int)pFile->lastErrno; OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; @@ -44447,12 +40132,6 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } - case SQLITE_FCNTL_WIN32_GET_HANDLE: { - LPHANDLE phFile = (LPHANDLE)pArg; - *phFile = pFile->h; - OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); - return SQLITE_OK; - } #ifdef SQLITE_TEST case SQLITE_FCNTL_WIN32_SET_HANDLE: { LPHANDLE phFile = (LPHANDLE)pArg; @@ -44545,14 +40224,14 @@ static SYSTEM_INFO winSysInfo; ** winShmLeaveMutex() */ static void winShmEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } static void winShmLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } #ifndef NDEBUG static int winShmMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } #endif @@ -44640,12 +40319,12 @@ struct winShm { /* ** Apply advisory locks for all n bytes beginning at ofst. */ -#define WINSHM_UNLCK 1 -#define WINSHM_RDLCK 2 -#define WINSHM_WRLCK 3 +#define _SHM_UNLCK 1 +#define _SHM_RDLCK 2 +#define _SHM_WRLCK 3 static int winShmSystemLock( winShmNode *pFile, /* Apply locks to this open shared-memory segment */ - int lockType, /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ + int lockType, /* _SHM_UNLCK, _SHM_RDLCK, or _SHM_WRLCK */ int ofst, /* Offset to first byte to be locked/unlocked */ int nByte /* Number of bytes to lock or unlock */ ){ @@ -44658,12 +40337,12 @@ static int winShmSystemLock( pFile->hFile.h, lockType, ofst, nByte)); /* Release/Acquire the system-level lock */ - if( lockType==WINSHM_UNLCK ){ + if( lockType==_SHM_UNLCK ){ rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); }else{ /* Initialize the locking parameters */ DWORD dwFlags = LOCKFILE_FAIL_IMMEDIATELY; - if( lockType == WINSHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; + if( lockType == _SHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); } @@ -44675,7 +40354,7 @@ static int winShmSystemLock( } OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", - pFile->hFile.h, (lockType == WINSHM_UNLCK) ? "winUnlockFile" : + pFile->hFile.h, (lockType == _SHM_UNLCK) ? "winUnlockFile" : "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); return rc; @@ -44753,12 +40432,12 @@ static int winOpenSharedMemory(winFile *pDbFd){ ** allocate space for a new winShmNode and filename. */ p = sqlite3MallocZero( sizeof(*p) ); - if( p==0 ) return SQLITE_IOERR_NOMEM_BKPT; + if( p==0 ) return SQLITE_IOERR_NOMEM; nName = sqlite3Strlen30(pDbFd->zPath); pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); if( pNew==0 ){ sqlite3_free(p); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } pNew->zFilename = (char*)&pNew[1]; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); @@ -44783,12 +40462,10 @@ static int winOpenSharedMemory(winFile *pDbFd){ pShmNode->pNext = winShmNodeList; winShmNodeList = pShmNode; - if( sqlite3GlobalConfig.bCoreMutex ){ - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_IOERR_NOMEM_BKPT; - goto shm_open_err; - } + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_IOERR_NOMEM; + goto shm_open_err; } rc = winOpen(pDbFd->pVfs, @@ -44803,7 +40480,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. */ - if( winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ + if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); if( rc!=SQLITE_OK ){ rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), @@ -44811,8 +40488,8 @@ static int winOpenSharedMemory(winFile *pDbFd){ } } if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); + winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); + rc = winShmSystemLock(pShmNode, _SHM_RDLCK, WIN_SHM_DMS, 1); } if( rc ) goto shm_open_err; } @@ -44841,7 +40518,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Jump here on any error */ shm_open_err: - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ sqlite3_free(p); sqlite3_free(pNew); @@ -44930,7 +40607,7 @@ static int winShmLock( /* Unlock the system-level locks */ if( (mask & allMask)==0 ){ - rc = winShmSystemLock(pShmNode, WINSHM_UNLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, _SHM_UNLCK, ofst+WIN_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -44958,7 +40635,7 @@ static int winShmLock( /* Get shared locks at the system level, if necessary */ if( rc==SQLITE_OK ){ if( (allShared & mask)==0 ){ - rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, _SHM_RDLCK, ofst+WIN_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -44983,7 +40660,7 @@ static int winShmLock( ** also mark the local connection as being locked. */ if( rc==SQLITE_OK ){ - rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, _SHM_WRLCK, ofst+WIN_SHM_BASE, n); if( rc==SQLITE_OK ){ assert( (p->sharedMask & mask)==0 ); p->exclMask |= mask; @@ -45007,8 +40684,8 @@ static void winShmBarrier( sqlite3_file *fd /* Database holding the shared memory */ ){ UNUSED_PARAMETER(fd); - sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ - winShmEnterMutex(); /* Also mutex, for redundancy */ + /* MemoryBarrier(); // does not work -- do not know why not */ + winShmEnterMutex(); winShmLeaveMutex(); } @@ -45092,7 +40769,7 @@ static int winShmMap( pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) ); if( !apNew ){ - rc = SQLITE_IOERR_NOMEM_BKPT; + rc = SQLITE_IOERR_NOMEM; goto shmpage_out; } pShmNode->aRegion = apNew; @@ -45109,7 +40786,7 @@ static int winShmMap( hMap = osCreateFileMappingW(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); -#elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA +#elif defined(SQLITE_WIN32_HAS_ANSI) hMap = osCreateFileMappingA(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); @@ -45253,19 +40930,17 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ DWORD flags = FILE_MAP_READ; winUnmapfile(pFd); -#ifdef SQLITE_MMAP_READWRITE if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ protect = PAGE_READWRITE; flags |= FILE_MAP_WRITE; } -#endif #if SQLITE_OS_WINRT pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); -#elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA +#elif defined(SQLITE_WIN32_HAS_ANSI) pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); @@ -45426,44 +41101,6 @@ static const sqlite3_io_methods winIoMethod = { winUnfetch /* xUnfetch */ }; -/* -** This vector defines all the methods that can operate on an -** sqlite3_file for win32 without performing any locking. -*/ -static const sqlite3_io_methods winIoNolockMethod = { - 3, /* iVersion */ - winClose, /* xClose */ - winRead, /* xRead */ - winWrite, /* xWrite */ - winTruncate, /* xTruncate */ - winSync, /* xSync */ - winFileSize, /* xFileSize */ - winNolockLock, /* xLock */ - winNolockUnlock, /* xUnlock */ - winNolockCheckReservedLock, /* xCheckReservedLock */ - winFileControl, /* xFileControl */ - winSectorSize, /* xSectorSize */ - winDeviceCharacteristics, /* xDeviceCharacteristics */ - winShmMap, /* xShmMap */ - winShmLock, /* xShmLock */ - winShmBarrier, /* xShmBarrier */ - winShmUnmap, /* xShmUnmap */ - winFetch, /* xFetch */ - winUnfetch /* xUnfetch */ -}; - -static winVfsAppData winAppData = { - &winIoMethod, /* pMethod */ - 0, /* pAppData */ - 0 /* bNoLock */ -}; - -static winVfsAppData winNolockAppData = { - &winIoNolockMethod, /* pMethod */ - 0, /* pAppData */ - 1 /* bNoLock */ -}; - /**************************************************************************** **************************** sqlite3_vfs methods **************************** ** @@ -45484,7 +41121,7 @@ static char *winConvertToUtf8Filename(const void *zFilename){ } #ifdef SQLITE_WIN32_HAS_ANSI else{ - zConverted = winMbcsToUtf8(zFilename, osAreFileApisANSI()); + zConverted = sqlite3_win32_mbcs_to_utf8(zFilename); } #endif /* caller will handle out of memory */ @@ -45505,7 +41142,7 @@ static void *winConvertFromUtf8Filename(const char *zFilename){ } #ifdef SQLITE_WIN32_HAS_ANSI else{ - zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); + zConverted = sqlite3_win32_utf8_to_mbcs(zFilename); } #endif /* caller will handle out of memory */ @@ -45560,7 +41197,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ zBuf = sqlite3MallocZero( nBuf ); if( !zBuf ){ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } /* Figure out the effective temporary directory. First, check if one @@ -45618,7 +41255,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zConverted ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } if( winIsDir(zConverted) ){ sqlite3_snprintf(nMax, zBuf, "%s", zDir); @@ -45631,7 +41268,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zConverted ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } if( cygwin_conv_path( osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A, zDir, @@ -45652,7 +41289,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ sqlite3_free(zConverted); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } sqlite3_snprintf(nMax, zBuf, "%s", zUtf8); sqlite3_free(zUtf8); @@ -45670,7 +41307,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zWidePath ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } if( osGetTempPathW(nMax, zWidePath)==0 ){ sqlite3_free(zWidePath); @@ -45688,7 +41325,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ sqlite3_free(zWidePath); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } } #ifdef SQLITE_WIN32_HAS_ANSI @@ -45698,7 +41335,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zMbcsPath ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } if( osGetTempPathA(nMax, zMbcsPath)==0 ){ sqlite3_free(zBuf); @@ -45706,14 +41343,14 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ return winLogError(SQLITE_IOERR_GETTEMPPATH, osGetLastError(), "winGetTempname3", 0); } - zUtf8 = winMbcsToUtf8(zMbcsPath, osAreFileApisANSI()); + zUtf8 = sqlite3_win32_mbcs_to_utf8(zMbcsPath); if( zUtf8 ){ sqlite3_snprintf(nMax, zBuf, "%s", zUtf8); sqlite3_free(zUtf8); }else{ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } } #endif /* SQLITE_WIN32_HAS_ANSI */ @@ -45796,7 +41433,7 @@ static int winIsDir(const void *zConverted){ ** Open a file. */ static int winOpen( - sqlite3_vfs *pVfs, /* Used to get maximum path length and AppData */ + sqlite3_vfs *pVfs, /* Used to get maximum path name length */ const char *zName, /* Name of the file (UTF-8) */ sqlite3_file *id, /* Write the SQLite file handle here */ int flags, /* Open mode flags */ @@ -45811,7 +41448,6 @@ static int winOpen( #if SQLITE_OS_WINCE int isTemp = 0; #endif - winVfsAppData *pAppData; winFile *pFile = (winFile*)id; void *zConverted; /* Filename in OS encoding */ const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ @@ -45906,7 +41542,7 @@ static int winOpen( if( zConverted==0 ){ sqlite3_free(zTmpname); OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } if( winIsDir(zConverted) ){ @@ -46033,20 +41669,15 @@ static int winOpen( "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); - pAppData = (winVfsAppData*)pVfs->pAppData; - #if SQLITE_OS_WINCE - { - if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB - && ((pAppData==NULL) || !pAppData->bNoLock) - && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK - ){ - osCloseHandle(h); - sqlite3_free(zConverted); - sqlite3_free(zTmpname); - OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); - return rc; - } + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB + && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK + ){ + osCloseHandle(h); + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); + return rc; } if( isTemp ){ pFile->zDeleteOnClose = zConverted; @@ -46057,7 +41688,7 @@ static int winOpen( } sqlite3_free(zTmpname); - pFile->pMethod = pAppData ? pAppData->pMethod : &winIoMethod; + pFile->pMethod = &winIoMethod; pFile->pVfs = pVfs; pFile->h = h; if( isReadonly ){ @@ -46111,7 +41742,7 @@ static int winDelete( zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("DELETE name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } if( osIsNT() ){ do { @@ -46219,7 +41850,7 @@ static int winAccess( zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } if( osIsNT() ){ int cnt = 0; @@ -46332,18 +41963,6 @@ static int winFullPathname( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) - DWORD nByte; - void *zConverted; - char *zOut; -#endif - - /* If this path name begins with "/X:", where "X" is any alphabetic - ** character, discard the initial "/" from the pathname. - */ - if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ - zRelative++; - } #if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); @@ -46358,7 +41977,7 @@ static int winFullPathname( */ char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); if( !zOut ){ - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } if( cygwin_conv_path( (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A) | @@ -46370,7 +41989,7 @@ static int winFullPathname( char *zUtf8 = winConvertToUtf8Filename(zOut); if( !zUtf8 ){ sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", sqlite3_data_directory, winGetDirSep(), zUtf8); @@ -46380,7 +41999,7 @@ static int winFullPathname( }else{ char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); if( !zOut ){ - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } if( cygwin_conv_path( (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A), @@ -46392,7 +42011,7 @@ static int winFullPathname( char *zUtf8 = winConvertToUtf8Filename(zOut); if( !zUtf8 ){ sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zUtf8); sqlite3_free(zUtf8); @@ -46422,6 +42041,17 @@ static int winFullPathname( #endif #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) + DWORD nByte; + void *zConverted; + char *zOut; + + /* If this path name begins with "/X:", where "X" is any alphabetic + ** character, discard the initial "/" from the pathname. + */ + if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ + zRelative++; + } + /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. This function could fail if, for example, the @@ -46441,7 +42071,7 @@ static int winFullPathname( } zConverted = winConvertFromUtf8Filename(zRelative); if( zConverted==0 ){ - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } if( osIsNT() ){ LPWSTR zTemp; @@ -46455,7 +42085,7 @@ static int winFullPathname( zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } nByte = osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); if( nByte==0 ){ @@ -46481,7 +42111,7 @@ static int winFullPathname( zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); if( nByte==0 ){ @@ -46491,7 +42121,7 @@ static int winFullPathname( "winFullPathname4", zRelative); } sqlite3_free(zConverted); - zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); + zOut = sqlite3_win32_mbcs_to_utf8(zTemp); sqlite3_free(zTemp); } #endif @@ -46500,7 +42130,7 @@ static int winFullPathname( sqlite3_free(zOut); return SQLITE_OK; }else{ - return SQLITE_IOERR_NOMEM_BKPT; + return SQLITE_IOERR_NOMEM; } #endif } @@ -46575,85 +42205,65 @@ static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ #define winDlClose 0 #endif -/* State information for the randomness gatherer. */ -typedef struct EntropyGatherer EntropyGatherer; -struct EntropyGatherer { - unsigned char *a; /* Gather entropy into this buffer */ - int na; /* Size of a[] in bytes */ - int i; /* XOR next input into a[i] */ - int nXor; /* Number of XOR operations done */ -}; - -#if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) -/* Mix sz bytes of entropy into p. */ -static void xorMemory(EntropyGatherer *p, unsigned char *x, int sz){ - int j, k; - for(j=0, k=p->i; ja[k++] ^= x[j]; - if( k>=p->na ) k = 0; - } - p->i = k; - p->nXor += sz; -} -#endif /* !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) */ /* ** Write up to nBuf bytes of randomness into zBuf. */ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ -#if defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) + int n = 0; UNUSED_PARAMETER(pVfs); +#if defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) + n = nBuf; memset(zBuf, 0, nBuf); - return nBuf; #else - EntropyGatherer e; - UNUSED_PARAMETER(pVfs); - memset(zBuf, 0, nBuf); -#if defined(_MSC_VER) && _MSC_VER>=1400 && !SQLITE_OS_WINCE - rand_s((unsigned int*)zBuf); /* rand_s() is not available with MinGW */ -#endif /* defined(_MSC_VER) && _MSC_VER>=1400 */ - e.a = (unsigned char*)zBuf; - e.na = nBuf; - e.nXor = 0; - e.i = 0; - { + if( sizeof(SYSTEMTIME)<=nBuf-n ){ SYSTEMTIME x; osGetSystemTime(&x); - xorMemory(&e, (unsigned char*)&x, sizeof(SYSTEMTIME)); + memcpy(&zBuf[n], &x, sizeof(x)); + n += sizeof(x); } - { + if( sizeof(DWORD)<=nBuf-n ){ DWORD pid = osGetCurrentProcessId(); - xorMemory(&e, (unsigned char*)&pid, sizeof(DWORD)); + memcpy(&zBuf[n], &pid, sizeof(pid)); + n += sizeof(pid); } #if SQLITE_OS_WINRT - { + if( sizeof(ULONGLONG)<=nBuf-n ){ ULONGLONG cnt = osGetTickCount64(); - xorMemory(&e, (unsigned char*)&cnt, sizeof(ULONGLONG)); + memcpy(&zBuf[n], &cnt, sizeof(cnt)); + n += sizeof(cnt); } #else - { + if( sizeof(DWORD)<=nBuf-n ){ DWORD cnt = osGetTickCount(); - xorMemory(&e, (unsigned char*)&cnt, sizeof(DWORD)); + memcpy(&zBuf[n], &cnt, sizeof(cnt)); + n += sizeof(cnt); } -#endif /* SQLITE_OS_WINRT */ - { +#endif + if( sizeof(LARGE_INTEGER)<=nBuf-n ){ LARGE_INTEGER i; osQueryPerformanceCounter(&i); - xorMemory(&e, (unsigned char*)&i, sizeof(LARGE_INTEGER)); + memcpy(&zBuf[n], &i, sizeof(i)); + n += sizeof(i); } #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID - { + if( sizeof(UUID)<=nBuf-n ){ UUID id; memset(&id, 0, sizeof(UUID)); osUuidCreate(&id); - xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); + memcpy(zBuf, &id, sizeof(UUID)); + n += sizeof(UUID); + } + if( sizeof(UUID)<=nBuf-n ){ + UUID id; memset(&id, 0, sizeof(UUID)); osUuidCreateSequential(&id); - xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); + memcpy(zBuf, &id, sizeof(UUID)); + n += sizeof(UUID); } -#endif /* !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID */ - return e.nXor>nBuf ? nBuf : e.nXor; -#endif /* defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) */ +#endif +#endif /* defined(SQLITE_TEST) || defined(SQLITE_ZERO_PRNG_SEED) */ + return n; } @@ -46769,114 +42379,62 @@ static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ ** sqlite3_errmsg(), possibly making IO errors easier to debug. */ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ - DWORD e = osGetLastError(); UNUSED_PARAMETER(pVfs); - if( nBuf>0 ) winGetLastErrorMsg(e, nBuf, zBuf); - return e; + return winGetLastErrorMsg(osGetLastError(), nBuf, zBuf); } /* ** Initialize and deinitialize the operating system interface. */ -SQLITE_API int sqlite3_os_init(void){ +SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void){ static sqlite3_vfs winVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32", /* zName */ - &winAppData, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ + 0, /* pNext */ + "win32", /* zName */ + 0, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ }; #if defined(SQLITE_WIN32_HAS_WIDE) static sqlite3_vfs winLongPathVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ - SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32-longpath", /* zName */ - &winAppData, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ - }; -#endif - static sqlite3_vfs winNolockVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ - SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32-none", /* zName */ - &winNolockAppData, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ - }; -#if defined(SQLITE_WIN32_HAS_WIDE) - static sqlite3_vfs winLongPathNolockVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32-longpath-none", /* zName */ - &winNolockAppData, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ + 0, /* pNext */ + "win32-longpath", /* zName */ + 0, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ }; #endif @@ -46900,16 +42458,10 @@ SQLITE_API int sqlite3_os_init(void){ sqlite3_vfs_register(&winLongPathVfs, 0); #endif - sqlite3_vfs_register(&winNolockVfs, 0); - -#if defined(SQLITE_WIN32_HAS_WIDE) - sqlite3_vfs_register(&winLongPathNolockVfs, 0); -#endif - return SQLITE_OK; } -SQLITE_API int sqlite3_os_end(void){ +SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void){ #if SQLITE_OS_WINRT if( sleepObj!=NULL ){ osCloseHandle(sleepObj); @@ -46959,15 +42511,13 @@ SQLITE_API int sqlite3_os_end(void){ ** start of a transaction, and is thus usually less than a few thousand, ** but can be as large as 2 billion for a really big database. */ -/* #include "sqliteInt.h" */ /* Size of the Bitvec structure in bytes. */ #define BITVEC_SZ 512 /* Round the union size down to the nearest pointer boundary, since that's how ** it will be aligned within the Bitvec struct. */ -#define BITVEC_USIZE \ - (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) +#define BITVEC_USIZE (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) /* Type of the array "element" for the bitmap representation. ** Should be a power of 2, and ideally, evenly divide into BITVEC_USIZE. @@ -47052,10 +42602,10 @@ SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32 iSize){ ** If p is NULL (if the bitmap has not been created) or if ** i is out of range, then return false. */ -SQLITE_PRIVATE int sqlite3BitvecTestNotNull(Bitvec *p, u32 i){ - assert( p!=0 ); +SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ + if( p==0 ) return 0; + if( i>p->iSize || i==0 ) return 0; i--; - if( i>=p->iSize ) return 0; while( p->iDivisor ){ u32 bin = i/p->iDivisor; i = i%p->iDivisor; @@ -47075,9 +42625,6 @@ SQLITE_PRIVATE int sqlite3BitvecTestNotNull(Bitvec *p, u32 i){ return 0; } } -SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ - return p!=0 && sqlite3BitvecTestNotNull(p,i); -} /* ** Set the i-th bit. Return 0 on success and an error code if @@ -47102,7 +42649,7 @@ SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){ i = i%p->iDivisor; if( p->u.apSub[bin]==0 ){ p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor ); - if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM_BKPT; + if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM; } p = p->u.apSub[bin]; } @@ -47137,7 +42684,7 @@ SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){ int rc; u32 *aiValues = sqlite3StackAllocRaw(0, sizeof(p->u.aHash)); if( aiValues==0 ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; }else{ memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); memset(p->u.apSub, 0, sizeof(p->u.apSub)); @@ -47350,39 +42897,15 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ ************************************************************************* ** This file implements that page cache. */ -/* #include "sqliteInt.h" */ /* -** A complete page cache is an instance of this structure. Every -** entry in the cache holds a single page of the database file. The -** btree layer only operates on the cached copy of the database pages. -** -** A page cache entry is "clean" if it exactly matches what is currently -** on disk. A page is "dirty" if it has been modified and needs to be -** persisted to disk. -** -** pDirty, pDirtyTail, pSynced: -** All dirty pages are linked into the doubly linked list using -** PgHdr.pDirtyNext and pDirtyPrev. The list is maintained in LRU order -** such that p was added to the list more recently than p->pDirtyNext. -** PCache.pDirty points to the first (newest) element in the list and -** pDirtyTail to the last (oldest). -** -** The PCache.pSynced variable is used to optimize searching for a dirty -** page to eject from the cache mid-transaction. It is better to eject -** a page that does not require a journal sync than one that does. -** Therefore, pSynced is maintained to that it *almost* always points -** to either the oldest page in the pDirty/pDirtyTail list that has a -** clear PGHDR_NEED_SYNC flag or to a page that is older than this one -** (so that the right page to eject can be found by following pDirtyPrev -** pointers). +** A complete page cache is an instance of this structure. */ struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRefSum; /* Sum of ref counts over all pages */ + int nRef; /* Number of referenced pages */ int szCache; /* Configured cache size */ - int szSpill; /* Size before spilling occurs */ int szPage; /* Size of every page in this cache */ int szExtra; /* Size of extra space for each page */ u8 bPurgeable; /* True if pages are on backing store */ @@ -47390,97 +42913,9 @@ struct PCache { int (*xStress)(void*,PgHdr*); /* Call to try make a page clean */ void *pStress; /* Argument to xStress */ sqlite3_pcache *pCache; /* Pluggable cache module */ + PgHdr *pPage1; /* Reference to page 1 */ }; -/********************************** Test and Debug Logic **********************/ -/* -** Debug tracing macros. Enable by by changing the "0" to "1" and -** recompiling. -** -** When sqlite3PcacheTrace is 1, single line trace messages are issued. -** When sqlite3PcacheTrace is 2, a dump of the pcache showing all cache entries -** is displayed for many operations, resulting in a lot of output. -*/ -#if defined(SQLITE_DEBUG) && 0 - int sqlite3PcacheTrace = 2; /* 0: off 1: simple 2: cache dumps */ - int sqlite3PcacheMxDump = 9999; /* Max cache entries for pcacheDump() */ -# define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;} - void pcacheDump(PCache *pCache){ - int N; - int i, j; - sqlite3_pcache_page *pLower; - PgHdr *pPg; - unsigned char *a; - - if( sqlite3PcacheTrace<2 ) return; - if( pCache->pCache==0 ) return; - N = sqlite3PcachePagecount(pCache); - if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump; - for(i=1; i<=N; i++){ - pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); - if( pLower==0 ) continue; - pPg = (PgHdr*)pLower->pExtra; - printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); - a = (unsigned char *)pLower->pBuf; - for(j=0; j<12; j++) printf("%02x", a[j]); - printf("\n"); - if( pPg->pPage==0 ){ - sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); - } - } - } - #else -# define pcacheTrace(X) -# define pcacheDump(X) -#endif - -/* -** Check invariants on a PgHdr entry. Return true if everything is OK. -** Return false if any invariant is violated. -** -** This routine is for use inside of assert() statements only. For -** example: -** -** assert( sqlite3PcachePageSanity(pPg) ); -*/ -#if SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr *pPg){ - PCache *pCache; - assert( pPg!=0 ); - assert( pPg->pgno>0 ); /* Page number is 1 or more */ - pCache = pPg->pCache; - assert( pCache!=0 ); /* Every page has an associated PCache */ - if( pPg->flags & PGHDR_CLEAN ){ - assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ - assert( pCache->pDirty!=pPg ); /* CLEAN pages not on dirty list */ - assert( pCache->pDirtyTail!=pPg ); - } - /* WRITEABLE pages must also be DIRTY */ - if( pPg->flags & PGHDR_WRITEABLE ){ - assert( pPg->flags & PGHDR_DIRTY ); /* WRITEABLE implies DIRTY */ - } - /* NEED_SYNC can be set independently of WRITEABLE. This can happen, - ** for example, when using the sqlite3PagerDontWrite() optimization: - ** (1) Page X is journalled, and gets WRITEABLE and NEED_SEEK. - ** (2) Page X moved to freelist, WRITEABLE is cleared - ** (3) Page X reused, WRITEABLE is set again - ** If NEED_SYNC had been cleared in step 2, then it would not be reset - ** in step 3, and page might be written into the database without first - ** syncing the rollback journal, which might cause corruption on a power - ** loss. - ** - ** Another example is when the database page size is smaller than the - ** disk sector size. When any page of a sector is journalled, all pages - ** in that sector are marked NEED_SYNC even if they are still CLEAN, just - ** in case they are later modified, since all pages in the same sector - ** must be journalled and synced before any of those pages can be safely - ** written. - */ - return 1; -} -#endif /* SQLITE_DEBUG */ - - /********************************** Linked List Management ********************/ /* Allowed values for second argument to pcacheManageDirtyList() */ @@ -47497,16 +42932,17 @@ SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr *pPg){ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ PCache *p = pPage->pCache; - pcacheTrace(("%p.DIRTYLIST.%s %d\n", p, - addRemove==1 ? "REMOVE" : addRemove==2 ? "ADD" : "FRONT", - pPage->pgno)); if( addRemove & PCACHE_DIRTYLIST_REMOVE ){ assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); assert( pPage->pDirtyPrev || pPage==p->pDirty ); /* Update the PCache1.pSynced variable if necessary. */ if( p->pSynced==pPage ){ - p->pSynced = pPage->pDirtyPrev; + PgHdr *pSynced = pPage->pDirtyPrev; + while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ + pSynced = pSynced->pDirtyPrev; + } + p->pSynced = pSynced; } if( pPage->pDirtyNext ){ @@ -47518,15 +42954,10 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ if( pPage->pDirtyPrev ){ pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; }else{ - /* If there are now no dirty pages in the cache, set eCreate to 2. - ** This is an optimization that allows sqlite3PcacheFetch() to skip - ** searching for a dirty page to eject from the cache when it might - ** otherwise have to. */ assert( pPage==p->pDirty ); p->pDirty = pPage->pDirtyNext; - assert( p->bPurgeable || p->eCreate==2 ); - if( p->pDirty==0 ){ /*OPTIMIZATION-IF-TRUE*/ - assert( p->bPurgeable==0 || p->eCreate==1 ); + if( p->pDirty==0 && p->bPurgeable ){ + assert( p->eCreate==1 ); p->eCreate = 2; } } @@ -47548,19 +42979,10 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ } } p->pDirty = pPage; - - /* If pSynced is NULL and this page has a clear NEED_SYNC flag, set - ** pSynced to point to it. Checking the NEED_SYNC flag is an - ** optimization, as if pSynced points to a page with the NEED_SYNC - ** flag set sqlite3PcacheFetchStress() searches through all newer - ** entries of the dirty-list for a page with NEED_SYNC clear anyway. */ - if( !p->pSynced - && 0==(pPage->flags&PGHDR_NEED_SYNC) /*OPTIMIZATION-IF-FALSE*/ - ){ + if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ p->pSynced = pPage; } } - pcacheDump(p); } /* @@ -47569,15 +42991,18 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ */ static void pcacheUnpin(PgHdr *p){ if( p->pCache->bPurgeable ){ - pcacheTrace(("%p.UNPIN %d\n", p->pCache, p->pgno)); + if( p->pgno==1 ){ + p->pCache->pPage1 = 0; + } sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0); - pcacheDump(p->pCache); } } /* -** Compute the number of pages of cache requested. p->szCache is the +** Compute the number of pages of cache requested. p->szCache is the ** cache size requested by the "PRAGMA cache_size" statement. +** +** */ static int numberOfCachePages(PCache *p){ if( p->szCache>=0 ){ @@ -47640,8 +43065,6 @@ SQLITE_PRIVATE int sqlite3PcacheOpen( p->xStress = xStress; p->pStress = pStress; p->szCache = 100; - p->szSpill = 1; - pcacheTrace(("%p.OPEN szPage %d bPurgeable %d\n",p,szPage,bPurgeable)); return sqlite3PcacheSetPageSize(p, szPage); } @@ -47650,21 +43073,21 @@ SQLITE_PRIVATE int sqlite3PcacheOpen( ** are no outstanding page references when this function is called. */ SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ - assert( pCache->nRefSum==0 && pCache->pDirty==0 ); + assert( pCache->nRef==0 && pCache->pDirty==0 ); if( pCache->szPage ){ sqlite3_pcache *pNew; pNew = sqlite3GlobalConfig.pcache2.xCreate( szPage, pCache->szExtra + ROUND8(sizeof(PgHdr)), pCache->bPurgeable ); - if( pNew==0 ) return SQLITE_NOMEM_BKPT; + if( pNew==0 ) return SQLITE_NOMEM; sqlite3GlobalConfig.pcache2.xCachesize(pNew, numberOfCachePages(pCache)); if( pCache->pCache ){ sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } pCache->pCache = pNew; + pCache->pPage1 = 0; pCache->szPage = szPage; - pcacheTrace(("%p.PAGESIZE %d\n",pCache,szPage)); } return SQLITE_OK; } @@ -47699,13 +43122,11 @@ SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch( int createFlag /* If true, create page if it does not exist already */ ){ int eCreate; - sqlite3_pcache_page *pRes; assert( pCache!=0 ); assert( pCache->pCache!=0 ); assert( createFlag==3 || createFlag==0 ); assert( pgno>0 ); - assert( pCache->eCreate==((pCache->bPurgeable && pCache->pDirty) ? 1 : 2) ); /* eCreate defines what to do if the page does not exist. ** 0 Do not allocate a new page. (createFlag==0) @@ -47718,15 +43139,12 @@ SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch( assert( eCreate==0 || eCreate==1 || eCreate==2 ); assert( createFlag==0 || pCache->eCreate==eCreate ); assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); - pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); - pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno, - createFlag?" create":"",pRes)); - return pRes; + return sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); } /* ** If the sqlite3PcacheFetch() routine is unable to allocate a new -** page because no clean pages are available for reuse and the cache +** page because new clean pages are available for reuse and the cache ** size limit has been reached, then this routine can be invoked to ** try harder to allocate a page. This routine might invoke the stress ** callback to spill dirty pages to the journal. It will then try to @@ -47743,43 +43161,36 @@ SQLITE_PRIVATE int sqlite3PcacheFetchStress( PgHdr *pPg; if( pCache->eCreate==2 ) return 0; - if( sqlite3PcachePagecount(pCache)>pCache->szSpill ){ - /* Find a dirty page to write-out and recycle. First try to find a - ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC - ** cleared), but if that is not possible settle for any other - ** unreferenced dirty page. - ** - ** If the LRU page in the dirty list that has a clear PGHDR_NEED_SYNC - ** flag is currently referenced, then the following may leave pSynced - ** set incorrectly (pointing to other than the LRU page with NEED_SYNC - ** cleared). This is Ok, as pSynced is just an optimization. */ - for(pPg=pCache->pSynced; - pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); - pPg=pPg->pDirtyPrev - ); - pCache->pSynced = pPg; - if( !pPg ){ - for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); - } - if( pPg ){ - int rc; + + /* Find a dirty page to write-out and recycle. First try to find a + ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC + ** cleared), but if that is not possible settle for any other + ** unreferenced dirty page. + */ + for(pPg=pCache->pSynced; + pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); + pPg=pPg->pDirtyPrev + ); + pCache->pSynced = pPg; + if( !pPg ){ + for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); + } + if( pPg ){ + int rc; #ifdef SQLITE_LOG_CACHE_SPILL - sqlite3_log(SQLITE_FULL, - "spill page %d making room for %d - cache used: %d/%d", - pPg->pgno, pgno, - sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), + sqlite3_log(SQLITE_FULL, + "spill page %d making room for %d - cache used: %d/%d", + pPg->pgno, pgno, + sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), numberOfCachePages(pCache)); #endif - pcacheTrace(("%p.SPILL %d\n",pCache,pPg->pgno)); - rc = pCache->xStress(pCache->pStress, pPg); - pcacheDump(pCache); - if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ - return rc; - } + rc = pCache->xStress(pCache->pStress, pPg); + if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ + return rc; } } *ppPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); - return *ppPage==0 ? SQLITE_NOMEM_BKPT : SQLITE_OK; + return *ppPage==0 ? SQLITE_NOMEM : SQLITE_OK; } /* @@ -47800,14 +43211,13 @@ static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( assert( pPage!=0 ); pPgHdr = (PgHdr*)pPage->pExtra; assert( pPgHdr->pPage==0 ); - memset(&pPgHdr->pDirty, 0, sizeof(PgHdr) - offsetof(PgHdr,pDirty)); + memset(pPgHdr, 0, sizeof(PgHdr)); pPgHdr->pPage = pPage; pPgHdr->pData = pPage->pBuf; pPgHdr->pExtra = (void *)&pPgHdr[1]; memset(pPgHdr->pExtra, 0, pCache->szExtra); pPgHdr->pCache = pCache; pPgHdr->pgno = pgno; - pPgHdr->flags = PGHDR_CLEAN; return sqlite3PcacheFetchFinish(pCache,pgno,pPage); } @@ -47824,15 +43234,19 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish( ){ PgHdr *pPgHdr; - assert( pPage!=0 ); + if( pPage==0 ) return 0; pPgHdr = (PgHdr *)pPage->pExtra; if( !pPgHdr->pPage ){ return pcacheFetchFinishWithInit(pCache, pgno, pPage); } - pCache->nRefSum++; + if( 0==pPgHdr->nRef ){ + pCache->nRef++; + } pPgHdr->nRef++; - assert( sqlite3PcachePageSanity(pPgHdr) ); + if( pgno==1 ){ + pCache->pPage1 = pPgHdr; + } return pPgHdr; } @@ -47842,15 +43256,13 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish( */ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ assert( p->nRef>0 ); - p->pCache->nRefSum--; - if( (--p->nRef)==0 ){ - if( p->flags&PGHDR_CLEAN ){ + p->nRef--; + if( p->nRef==0 ){ + p->pCache->nRef--; + if( (p->flags&PGHDR_DIRTY)==0 ){ pcacheUnpin(p); - }else if( p->pDirtyPrev!=0 ){ /*OPTIMIZATION-IF-FALSE*/ - /* Move the page to the head of the dirty list. If p->pDirtyPrev==0, - ** then page p is already at the head of the dirty list and the - ** following call would be a no-op. Hence the OPTIMIZATION-IF-FALSE - ** tag above. */ + }else if( p->pDirtyPrev!=0 ){ + /* Move the page to the head of the dirty list. */ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); } } @@ -47861,9 +43273,7 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ */ SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr *p){ assert(p->nRef>0); - assert( sqlite3PcachePageSanity(p) ); p->nRef++; - p->pCache->nRefSum++; } /* @@ -47873,11 +43283,13 @@ SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr *p){ */ SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ assert( p->nRef==1 ); - assert( sqlite3PcachePageSanity(p) ); if( p->flags&PGHDR_DIRTY ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); } - p->pCache->nRefSum--; + p->pCache->nRef--; + if( p->pgno==1 ){ + p->pCache->pPage1 = 0; + } sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 1); } @@ -47886,17 +43298,11 @@ SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ ** make it so. */ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ + p->flags &= ~PGHDR_DONT_WRITE; assert( p->nRef>0 ); - assert( sqlite3PcachePageSanity(p) ); - if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){ /*OPTIMIZATION-IF-FALSE*/ - p->flags &= ~PGHDR_DONT_WRITE; - if( p->flags & PGHDR_CLEAN ){ - p->flags ^= (PGHDR_DIRTY|PGHDR_CLEAN); - pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno)); - assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); - pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); - } - assert( sqlite3PcachePageSanity(p) ); + if( 0==(p->flags & PGHDR_DIRTY) ){ + p->flags |= PGHDR_DIRTY; + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); } } @@ -47905,14 +43311,9 @@ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ ** make it so. */ SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr *p){ - assert( sqlite3PcachePageSanity(p) ); - if( ALWAYS((p->flags & PGHDR_DIRTY)!=0) ){ - assert( (p->flags & PGHDR_CLEAN)==0 ); + if( (p->flags & PGHDR_DIRTY) ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); - p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); - p->flags |= PGHDR_CLEAN; - pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno)); - assert( sqlite3PcachePageSanity(p) ); + p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC); if( p->nRef==0 ){ pcacheUnpin(p); } @@ -47924,24 +43325,11 @@ SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr *p){ */ SQLITE_PRIVATE void sqlite3PcacheCleanAll(PCache *pCache){ PgHdr *p; - pcacheTrace(("%p.CLEAN-ALL\n",pCache)); while( (p = pCache->pDirty)!=0 ){ sqlite3PcacheMakeClean(p); } } -/* -** Clear the PGHDR_NEED_SYNC and PGHDR_WRITEABLE flag from all dirty pages. -*/ -SQLITE_PRIVATE void sqlite3PcacheClearWritable(PCache *pCache){ - PgHdr *p; - pcacheTrace(("%p.CLEAR-WRITEABLE\n",pCache)); - for(p=pCache->pDirty; p; p=p->pDirtyNext){ - p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE); - } - pCache->pSynced = pCache->pDirtyTail; -} - /* ** Clear the PGHDR_NEED_SYNC flag from all dirty pages. */ @@ -47960,8 +43348,6 @@ SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ PCache *pCache = p->pCache; assert( p->nRef>0 ); assert( newPgno>0 ); - assert( sqlite3PcachePageSanity(p) ); - pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ @@ -47982,7 +43368,6 @@ SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ if( pCache->pCache ){ PgHdr *p; PgHdr *pNext; - pcacheTrace(("%p.TRUNCATE %d\n",pCache,pgno)); for(p=pCache->pDirty; p; p=pNext){ pNext = p->pDirtyNext; /* This routine never gets call with a positive pgno except right @@ -47990,19 +43375,14 @@ SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ ** it must be that pgno==0. */ assert( p->pgno>0 ); - if( p->pgno>pgno ){ + if( ALWAYS(p->pgno>pgno) ){ assert( p->flags&PGHDR_DIRTY ); sqlite3PcacheMakeClean(p); } } - if( pgno==0 && pCache->nRefSum ){ - sqlite3_pcache_page *pPage1; - pPage1 = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache,1,0); - if( ALWAYS(pPage1) ){ /* Page 1 is always available in cache, because - ** pCache->nRefSum>0 */ - memset(pPage1->pBuf, 0, pCache->szPage); - pgno = 1; - } + if( pgno==0 && pCache->pPage1 ){ + memset(pCache->pPage1->pData, 0, pCache->szPage); + pgno = 1; } sqlite3GlobalConfig.pcache2.xTruncate(pCache->pCache, pgno+1); } @@ -48013,7 +43393,6 @@ SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ */ SQLITE_PRIVATE void sqlite3PcacheClose(PCache *pCache){ assert( pCache->pCache!=0 ); - pcacheTrace(("%p.CLOSE\n",pCache)); sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } @@ -48026,31 +43405,29 @@ SQLITE_PRIVATE void sqlite3PcacheClear(PCache *pCache){ /* ** Merge two lists of pages connected by pDirty and in pgno order. -** Do not bother fixing the pDirtyPrev pointers. +** Do not both fixing the pDirtyPrev pointers. */ static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ PgHdr result, *pTail; pTail = &result; - assert( pA!=0 && pB!=0 ); - for(;;){ + while( pA && pB ){ if( pA->pgnopgno ){ pTail->pDirty = pA; pTail = pA; pA = pA->pDirty; - if( pA==0 ){ - pTail->pDirty = pB; - break; - } }else{ pTail->pDirty = pB; pTail = pB; pB = pB->pDirty; - if( pB==0 ){ - pTail->pDirty = pA; - break; - } } } + if( pA ){ + pTail->pDirty = pA; + }else if( pB ){ + pTail->pDirty = pB; + }else{ + pTail->pDirty = 0; + } return result.pDirty; } @@ -48091,8 +43468,7 @@ static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ } p = a[0]; for(i=1; inRefSum; + return pCache->nRef; } /* @@ -48152,25 +43525,6 @@ SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ numberOfCachePages(pCache)); } -/* -** Set the suggested cache-spill value. Make no changes if if the -** argument is zero. Return the effective cache-spill size, which will -** be the larger of the szSpill and szCache. -*/ -SQLITE_PRIVATE int sqlite3PcacheSetSpillsize(PCache *p, int mxPage){ - int res; - assert( p->pCache!=0 ); - if( mxPage ){ - if( mxPage<0 ){ - mxPage = (int)((-1024*(i64)mxPage)/(p->szPage+p->szExtra)); - } - p->szSpill = mxPage; - } - res = numberOfCachePages(p); - if( resszSpill ) res = p->szSpill; - return res; -} - /* ** Free up as much memory as possible from the page cache. */ @@ -48185,17 +43539,6 @@ SQLITE_PRIVATE void sqlite3PcacheShrink(PCache *pCache){ */ SQLITE_PRIVATE int sqlite3HeaderSizePcache(void){ return ROUND8(sizeof(PgHdr)); } -/* -** Return the number of dirty pages currently in the cache, as a percentage -** of the configured cache size. -*/ -SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache *pCache){ - PgHdr *pDirty; - int nDirty = 0; - int nCache = numberOfCachePages(pCache); - for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++; - return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; -} #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* @@ -48230,96 +43573,14 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. ** If the default page cache implementation is overridden, then neither of ** these two features are available. -** -** A Page cache line looks like this: -** -** ------------------------------------------------------------- -** | database page content | PgHdr1 | MemPage | PgHdr | -** ------------------------------------------------------------- -** -** The database page content is up front (so that buffer overreads tend to -** flow harmlessly into the PgHdr1, MemPage, and PgHdr extensions). MemPage -** is the extension added by the btree.c module containing information such -** as the database page number and how that database page is used. PgHdr -** is added by the pcache.c layer and contains information used to keep track -** of which pages are "dirty". PgHdr1 is an extension added by this -** module (pcache1.c). The PgHdr1 header is a subclass of sqlite3_pcache_page. -** PgHdr1 contains information needed to look up a page by its page number. -** The superclass sqlite3_pcache_page.pBuf points to the start of the -** database page content and sqlite3_pcache_page.pExtra points to PgHdr. -** -** The size of the extension (MemPage+PgHdr+PgHdr1) can be determined at -** runtime using sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &size). The -** sizes of the extensions sum to 272 bytes on x64 for 3.8.10, but this -** size can vary according to architecture, compile-time options, and -** SQLite library version number. -** -** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained -** using a separate memory allocation from the database page content. This -** seeks to overcome the "clownshoe" problem (also called "internal -** fragmentation" in academic literature) of allocating a few bytes more -** than a power of two with the memory allocator rounding up to the next -** power of two, and leaving the rounded-up space unused. -** -** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates -** with this module. Information is passed back and forth as PgHdr1 pointers. -** -** The pcache.c and pager.c modules deal pointers to PgHdr objects. -** The btree.c module deals with pointers to MemPage objects. -** -** SOURCE OF PAGE CACHE MEMORY: -** -** Memory for a page might come from any of three sources: -** -** (1) The general-purpose memory allocator - sqlite3Malloc() -** (2) Global page-cache memory provided using sqlite3_config() with -** SQLITE_CONFIG_PAGECACHE. -** (3) PCache-local bulk allocation. -** -** The third case is a chunk of heap memory (defaulting to 100 pages worth) -** that is allocated when the page cache is created. The size of the local -** bulk allocation can be adjusted using -** -** sqlite3_config(SQLITE_CONFIG_PAGECACHE, (void*)0, 0, N). -** -** If N is positive, then N pages worth of memory are allocated using a single -** sqlite3Malloc() call and that memory is used for the first N pages allocated. -** Or if N is negative, then -1024*N bytes of memory are allocated and used -** for as many pages as can be accomodated. -** -** Only one of (2) or (3) can be used. Once the memory available to (2) or -** (3) is exhausted, subsequent allocations fail over to the general-purpose -** memory allocator (1). -** -** Earlier versions of SQLite used only methods (1) and (2). But experiments -** show that method (3) with N==100 provides about a 5% performance boost for -** common workloads. */ -/* #include "sqliteInt.h" */ + typedef struct PCache1 PCache1; typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; -/* -** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. -*/ -struct PgHdr1 { - sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ - unsigned int iKey; /* Key value (page number) */ - u8 isPinned; /* Page in use, not on the LRU list */ - u8 isBulkLocal; /* This page from bulk local storage */ - u8 isAnchor; /* This is the PGroup.lru element */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ -}; - /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set ** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of @@ -48348,7 +43609,7 @@ struct PGroup { unsigned int nMinPage; /* Sum of nMin for purgeable caches */ unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ unsigned int nCurrentPage; /* Number of purgeable pages allocated */ - PgHdr1 lru; /* The beginning and end of the LRU list */ + PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ }; /* Each page cache is an instance of the following object. Every @@ -48366,9 +43627,8 @@ struct PCache1 { ** The PGroup mutex must be held when accessing nMax. */ PGroup *pGroup; /* PGroup this cache belongs to */ - int szPage; /* Size of database content section */ - int szExtra; /* sizeof(MemPage)+sizeof(PgHdr) */ - int szAlloc; /* Total size of one pcache line */ + int szPage; /* Size of allocated pages in bytes */ + int szExtra; /* Size of extra space in bytes */ int bPurgeable; /* True if cache is purgeable */ unsigned int nMin; /* Minimum number of pages reserved */ unsigned int nMax; /* Configured "cache_size" value */ @@ -48382,13 +43642,27 @@ struct PCache1 { unsigned int nPage; /* Total number of pages in apHash */ unsigned int nHash; /* Number of slots in apHash[] */ PgHdr1 **apHash; /* Hash table for fast lookup by key */ - PgHdr1 *pFree; /* List of unused pcache-local pages */ - void *pBulk; /* Bulk memory used by pcache-local */ }; /* -** Free slots in the allocator used to divide up the global page cache -** buffer provided using the SQLITE_CONFIG_PAGECACHE mechanism. +** Each cache entry is represented by an instance of the following +** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of +** PgHdr1.pCache->szPage bytes is allocated directly before this structure +** in memory. +*/ +struct PgHdr1 { + sqlite3_pcache_page page; + unsigned int iKey; /* Key value (page number) */ + u8 isPinned; /* Page in use, not on the LRU list */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ +}; + +/* +** Free slots in the allocator used to divide up the buffer provided using +** the SQLITE_CONFIG_PAGECACHE mechanism. */ struct PgFreeslot { PgFreeslot *pNext; /* Next free slot */ @@ -48406,12 +43680,10 @@ static SQLITE_WSD struct PCacheGlobal { ** The nFreeSlot and pFree values do require mutex protection. */ int isInit; /* True if initialized */ - int separateCache; /* Use a new PGroup for each PCache */ - int nInitPage; /* Initial bulk allocation size */ int szSlot; /* Size of each free slot */ int nSlot; /* The number of pcache slots */ int nReserve; /* Try to keep nFreeSlot above this */ - void *pStart, *pEnd; /* Bounds of global page cache memory */ + void *pStart, *pEnd; /* Bounds of pagecache malloc range */ /* Above requires no mutex. Use mutex below for variable that follow. */ sqlite3_mutex *mutex; /* Mutex for accessing the following: */ PgFreeslot *pFree; /* Free page blocks */ @@ -48433,20 +43705,12 @@ static SQLITE_WSD struct PCacheGlobal { /* ** Macros to enter and leave the PCache LRU mutex. */ -#if !defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 -# define pcache1EnterMutex(X) assert((X)->mutex==0) -# define pcache1LeaveMutex(X) assert((X)->mutex==0) -# define PCACHE1_MIGHT_USE_GROUP_MUTEX 0 -#else -# define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) -# define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) -# define PCACHE1_MIGHT_USE_GROUP_MUTEX 1 -#endif +#define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) +#define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) /******************************************************************************/ /******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ - /* ** This function is called during initialization if a static buffer is ** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE @@ -48459,7 +43723,6 @@ static SQLITE_WSD struct PCacheGlobal { SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ if( pcache1.isInit ){ PgFreeslot *p; - if( pBuf==0 ) sz = n = 0; sz = ROUNDDOWN8(sz); pcache1.szSlot = sz; pcache1.nSlot = pcache1.nFreeSlot = n; @@ -48477,44 +43740,6 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ } } -/* -** Try to initialize the pCache->pFree and pCache->pBulk fields. Return -** true if pCache->pFree ends up containing one or more free pages. -*/ -static int pcache1InitBulk(PCache1 *pCache){ - i64 szBulk; - char *zBulk; - if( pcache1.nInitPage==0 ) return 0; - /* Do not bother with a bulk allocation if the cache size very small */ - if( pCache->nMax<3 ) return 0; - sqlite3BeginBenignMalloc(); - if( pcache1.nInitPage>0 ){ - szBulk = pCache->szAlloc * (i64)pcache1.nInitPage; - }else{ - szBulk = -1024 * (i64)pcache1.nInitPage; - } - if( szBulk > pCache->szAlloc*(i64)pCache->nMax ){ - szBulk = pCache->szAlloc*(i64)pCache->nMax; - } - zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); - sqlite3EndBenignMalloc(); - if( zBulk ){ - int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; - int i; - for(i=0; iszPage]; - pX->page.pBuf = zBulk; - pX->page.pExtra = &pX[1]; - pX->isBulkLocal = 1; - pX->isAnchor = 0; - pX->pNext = pCache->pFree; - pCache->pFree = pX; - zBulk += pCache->szAlloc; - } - } - return pCache->pFree!=0; -} - /* ** Malloc function used within this file to allocate space from the buffer ** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no @@ -48535,7 +43760,7 @@ static void *pcache1Alloc(int nByte){ pcache1.nFreeSlot--; pcache1.bUnderPressure = pcache1.nFreeSlot=0 ); - sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_USED, 1); } sqlite3_mutex_leave(pcache1.mutex); @@ -48549,7 +43774,7 @@ static void *pcache1Alloc(int nByte){ if( p ){ int sz = sqlite3MallocSize(p); sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); sqlite3_mutex_leave(pcache1.mutex); } @@ -48562,9 +43787,10 @@ static void *pcache1Alloc(int nByte){ /* ** Free an allocated buffer obtained from pcache1Alloc(). */ -static void pcache1Free(void *p){ - if( p==0 ) return; - if( SQLITE_WITHIN(p, pcache1.pStart, pcache1.pEnd) ){ +static int pcache1Free(void *p){ + int nFreed = 0; + if( p==0 ) return 0; + if( p>=pcache1.pStart && ppGroup->mutex) ); - if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ - p = pCache->pFree; - pCache->pFree = p->pNext; - p->pNext = 0; - }else{ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - /* The group mutex must be released before pcache1Alloc() is called. This - ** is because it might call sqlite3_release_memory(), which assumes that - ** this mutex is not held. */ - assert( pcache1.separateCache==0 ); - assert( pCache->pGroup==&pcache1.grp ); - pcache1LeaveMutex(pCache->pGroup); -#endif - if( benignMalloc ){ sqlite3BeginBenignMalloc(); } + pcache1LeaveMutex(pCache->pGroup); #ifdef SQLITE_PCACHE_SEPARATE_HEADER - pPg = pcache1Alloc(pCache->szPage); - p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); - if( !pPg || !p ){ - pcache1Free(pPg); - sqlite3_free(p); - pPg = 0; - } + pPg = pcache1Alloc(pCache->szPage); + p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); + if( !pPg || !p ){ + pcache1Free(pPg); + sqlite3_free(p); + pPg = 0; + } #else - pPg = pcache1Alloc(pCache->szAlloc); - p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; -#endif - if( benignMalloc ){ sqlite3EndBenignMalloc(); } -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - pcache1EnterMutex(pCache->pGroup); + pPg = pcache1Alloc(ROUND8(sizeof(PgHdr1)) + pCache->szPage + pCache->szExtra); + p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; #endif - if( pPg==0 ) return 0; + pcache1EnterMutex(pCache->pGroup); + + if( pPg ){ p->page.pBuf = pPg; p->page.pExtra = &p[1]; - p->isBulkLocal = 0; - p->isAnchor = 0; - } - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage++; + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage++; + } + return p; } - return p; + return 0; } /* ** Free a page object allocated by pcache1AllocPage(). +** +** The pointer is allowed to be NULL, which is prudent. But it turns out +** that the current implementation happens to never call this routine +** with a NULL pointer, so we mark the NULL test with ALWAYS(). */ static void pcache1FreePage(PgHdr1 *p){ - PCache1 *pCache; - assert( p!=0 ); - pCache = p->pCache; - assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); - if( p->isBulkLocal ){ - p->pNext = pCache->pFree; - pCache->pFree = p; - }else{ + if( ALWAYS(p) ){ + PCache1 *pCache = p->pCache; + assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); pcache1Free(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER sqlite3_free(p); #endif - } - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage--; + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage--; + } } } @@ -48772,35 +43982,41 @@ static void pcache1ResizeHash(PCache1 *p){ ** ** The PGroup mutex must be held when this function is called. */ -static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ +static void pcache1PinPage(PgHdr1 *pPage){ PCache1 *pCache; + PGroup *pGroup; assert( pPage!=0 ); assert( pPage->isPinned==0 ); pCache = pPage->pCache; - assert( pPage->pLruNext ); - assert( pPage->pLruPrev ); - assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - pPage->pLruPrev->pLruNext = pPage->pLruNext; - pPage->pLruNext->pLruPrev = pPage->pLruPrev; + pGroup = pCache->pGroup; + assert( pPage->pLruNext || pPage==pGroup->pLruTail ); + assert( pPage->pLruPrev || pPage==pGroup->pLruHead ); + assert( sqlite3_mutex_held(pGroup->mutex) ); + if( pPage->pLruPrev ){ + pPage->pLruPrev->pLruNext = pPage->pLruNext; + }else{ + pGroup->pLruHead = pPage->pLruNext; + } + if( pPage->pLruNext ){ + pPage->pLruNext->pLruPrev = pPage->pLruPrev; + }else{ + pGroup->pLruTail = pPage->pLruPrev; + } pPage->pLruNext = 0; pPage->pLruPrev = 0; pPage->isPinned = 1; - assert( pPage->isAnchor==0 ); - assert( pCache->pGroup->lru.isAnchor==1 ); pCache->nRecyclable--; - return pPage; } /* ** Remove the page supplied as an argument from the hash table ** (PCache1.apHash structure) that it is currently stored in. -** Also free the page if freePage is true. ** ** The PGroup mutex must be held when this function is called. */ -static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ +static void pcache1RemoveFromHash(PgHdr1 *pPage){ unsigned int h; PCache1 *pCache = pPage->pCache; PgHdr1 **pp; @@ -48811,28 +44027,21 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ *pp = (*pp)->pNext; pCache->nPage--; - if( freeFlag ) pcache1FreePage(pPage); } /* ** If there are currently more than nMaxPage pages allocated, try ** to recycle pages to reduce the number allocated to nMaxPage. */ -static void pcache1EnforceMaxPage(PCache1 *pCache){ - PGroup *pGroup = pCache->pGroup; - PgHdr1 *p; +static void pcache1EnforceMaxPage(PGroup *pGroup){ assert( sqlite3_mutex_held(pGroup->mutex) ); - while( pGroup->nCurrentPage>pGroup->nMaxPage - && (p=pGroup->lru.pLruPrev)->isAnchor==0 - ){ + while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){ + PgHdr1 *p = pGroup->pLruTail; assert( p->pCache->pGroup==pGroup ); assert( p->isPinned==0 ); pcache1PinPage(p); - pcache1RemoveFromHash(p, 1); - } - if( pCache->nPage==0 && pCache->pBulk ){ - sqlite3_free(pCache->pBulk); - pCache->pBulk = pCache->pFree = 0; + pcache1RemoveFromHash(p); + pcache1FreePage(p); } } @@ -48847,30 +44056,12 @@ static void pcache1TruncateUnsafe( PCache1 *pCache, /* The cache to truncate */ unsigned int iLimit /* Drop pages with this pgno or larger */ ){ - TESTONLY( int nPage = 0; ) /* To assert pCache->nPage is correct */ - unsigned int h, iStop; + TESTONLY( unsigned int nPage = 0; ) /* To assert pCache->nPage is correct */ + unsigned int h; assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - assert( pCache->iMaxKey >= iLimit ); - assert( pCache->nHash > 0 ); - if( pCache->iMaxKey - iLimit < pCache->nHash ){ - /* If we are just shaving the last few pages off the end of the - ** cache, then there is no point in scanning the entire hash table. - ** Only scan those hash slots that might contain pages that need to - ** be removed. */ - h = iLimit % pCache->nHash; - iStop = pCache->iMaxKey % pCache->nHash; - TESTONLY( nPage = -10; ) /* Disable the pCache->nPage validity check */ - }else{ - /* This is the general case where many pages are being removed. - ** It is necessary to scan the entire hash table */ - h = pCache->nHash/2; - iStop = h - 1; - } - for(;;){ - PgHdr1 **pp; + for(h=0; hnHash; h++){ + PgHdr1 **pp = &pCache->apHash[h]; PgHdr1 *pPage; - assert( hnHash ); - pp = &pCache->apHash[h]; while( (pPage = *pp)!=0 ){ if( pPage->iKey>=iLimit ){ pCache->nPage--; @@ -48879,13 +44070,11 @@ static void pcache1TruncateUnsafe( pcache1FreePage(pPage); }else{ pp = &pPage->pNext; - TESTONLY( if( nPage>=0 ) nPage++; ) + TESTONLY( nPage++; ) } } - if( h==iStop ) break; - h = (h+1) % pCache->nHash; } - assert( nPage<0 || pCache->nPage==(unsigned)nPage ); + assert( pCache->nPage==nPage ); } /******************************************************************************/ @@ -48898,44 +44087,9 @@ static int pcache1Init(void *NotUsed){ UNUSED_PARAMETER(NotUsed); assert( pcache1.isInit==0 ); memset(&pcache1, 0, sizeof(pcache1)); - - - /* - ** The pcache1.separateCache variable is true if each PCache has its own - ** private PGroup (mode-1). pcache1.separateCache is false if the single - ** PGroup in pcache1.grp is used for all page caches (mode-2). - ** - ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT - ** - ** * Use a unified cache in single-threaded applications that have - ** configured a start-time buffer for use as page-cache memory using - ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL - ** pBuf argument. - ** - ** * Otherwise use separate caches (mode-1) - */ -#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) - pcache1.separateCache = 0; -#elif SQLITE_THREADSAFE - pcache1.separateCache = sqlite3GlobalConfig.pPage==0 - || sqlite3GlobalConfig.bCoreMutex>0; -#else - pcache1.separateCache = sqlite3GlobalConfig.pPage==0; -#endif - -#if SQLITE_THREADSAFE if( sqlite3GlobalConfig.bCoreMutex ){ - pcache1.grp.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU); - pcache1.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PMEM); - } -#endif - if( pcache1.separateCache - && sqlite3GlobalConfig.nPage!=0 - && sqlite3GlobalConfig.pPage==0 - ){ - pcache1.nInitPage = sqlite3GlobalConfig.nPage; - }else{ - pcache1.nInitPage = 0; + pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); + pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); } pcache1.grp.mxPinned = 10; pcache1.isInit = 1; @@ -48966,26 +44120,39 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ PGroup *pGroup; /* The group the new page cache will belong to */ int sz; /* Bytes of memory required to allocate the new cache */ + /* + ** The separateCache variable is true if each PCache has its own private + ** PGroup. In other words, separateCache is true for mode (1) where no + ** mutexing is required. + ** + ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT + ** + ** * Always use a unified cache in single-threaded applications + ** + ** * Otherwise (if multi-threaded and ENABLE_MEMORY_MANAGEMENT is off) + ** use separate caches (mode-1) + */ +#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 + const int separateCache = 0; +#else + int separateCache = sqlite3GlobalConfig.bCoreMutex>0; +#endif + assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); assert( szExtra < 300 ); - sz = sizeof(PCache1) + sizeof(PGroup)*pcache1.separateCache; + sz = sizeof(PCache1) + sizeof(PGroup)*separateCache; pCache = (PCache1 *)sqlite3MallocZero(sz); if( pCache ){ - if( pcache1.separateCache ){ + if( separateCache ){ pGroup = (PGroup*)&pCache[1]; pGroup->mxPinned = 10; }else{ pGroup = &pcache1.grp; } - if( pGroup->lru.isAnchor==0 ){ - pGroup->lru.isAnchor = 1; - pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; - } pCache->pGroup = pGroup; pCache->szPage = szPage; pCache->szExtra = szExtra; - pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); pCache->bPurgeable = (bPurgeable ? 1 : 0); pcache1EnterMutex(pGroup); pcache1ResizeHash(pCache); @@ -49017,7 +44184,7 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pCache->nMax = nMax; pCache->n90pct = pCache->nMax*9/10; - pcache1EnforceMaxPage(pCache); + pcache1EnforceMaxPage(pGroup); pcache1LeaveMutex(pGroup); } } @@ -49035,7 +44202,7 @@ static void pcache1Shrink(sqlite3_pcache *p){ pcache1EnterMutex(pGroup); savedMaxPage = pGroup->nMaxPage; pGroup->nMaxPage = 0; - pcache1EnforceMaxPage(pCache); + pcache1EnforceMaxPage(pGroup); pGroup->nMaxPage = savedMaxPage; pcache1LeaveMutex(pGroup); } @@ -49088,17 +44255,26 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( assert( pCache->nHash>0 && pCache->apHash ); /* Step 4. Try to recycle a page. */ - if( pCache->bPurgeable - && !pGroup->lru.pLruPrev->isAnchor - && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache)) - ){ + if( pCache->bPurgeable && pGroup->pLruTail && ( + (pCache->nPage+1>=pCache->nMax) + || pGroup->nCurrentPage>=pGroup->nMaxPage + || pcache1UnderMemoryPressure(pCache) + )){ PCache1 *pOther; - pPage = pGroup->lru.pLruPrev; + pPage = pGroup->pLruTail; assert( pPage->isPinned==0 ); - pcache1RemoveFromHash(pPage, 0); + pcache1RemoveFromHash(pPage); pcache1PinPage(pPage); pOther = pPage->pCache; - if( pOther->szAlloc != pCache->szAlloc ){ + + /* We want to verify that szPage and szExtra are the same for pOther + ** and pCache. Assert that we can verify this by comparing sums. */ + assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 ); + assert( pCache->szExtra<512 ); + assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 ); + assert( pOther->szExtra<512 ); + + if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){ pcache1FreePage(pPage); pPage = 0; }else{ @@ -49110,7 +44286,9 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** attempt to allocate a new one. */ if( !pPage ){ - pPage = pcache1AllocPage(pCache, createFlag==1); + if( createFlag==1 ) sqlite3BeginBenignMalloc(); + pPage = pcache1AllocPage(pCache); + if( createFlag==1 ) sqlite3EndBenignMalloc(); } if( pPage ){ @@ -49184,13 +44362,8 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** proceed to step 5. ** ** 5. Otherwise, allocate and return a new page buffer. -** -** There are two versions of this routine. pcache1FetchWithMutex() is -** the general case. pcache1FetchNoMutex() is a faster implementation for -** the common case where pGroup->mutex is NULL. The pcache1Fetch() wrapper -** invokes the appropriate routine. */ -static PgHdr1 *pcache1FetchNoMutex( +static sqlite3_pcache_page *pcache1Fetch( sqlite3_pcache *p, unsigned int iKey, int createFlag @@ -49198,66 +44371,28 @@ static PgHdr1 *pcache1FetchNoMutex( PCache1 *pCache = (PCache1 *)p; PgHdr1 *pPage = 0; + assert( offsetof(PgHdr1,page)==0 ); + assert( pCache->bPurgeable || createFlag!=1 ); + assert( pCache->bPurgeable || pCache->nMin==0 ); + assert( pCache->bPurgeable==0 || pCache->nMin==10 ); + assert( pCache->nMin==0 || pCache->bPurgeable ); + assert( pCache->nHash>0 ); + pcache1EnterMutex(pCache->pGroup); + /* Step 1: Search the hash table for an existing entry. */ pPage = pCache->apHash[iKey % pCache->nHash]; while( pPage && pPage->iKey!=iKey ){ pPage = pPage->pNext; } - /* Step 2: If the page was found in the hash table, then return it. - ** If the page was not in the hash table and createFlag is 0, abort. - ** Otherwise (page not in hash and createFlag!=0) continue with - ** subsequent steps to try to create the page. */ + /* Step 2: Abort if no existing page is found and createFlag is 0 */ if( pPage ){ - if( !pPage->isPinned ){ - return pcache1PinPage(pPage); - }else{ - return pPage; - } + if( !pPage->isPinned ) pcache1PinPage(pPage); }else if( createFlag ){ /* Steps 3, 4, and 5 implemented by this subroutine */ - return pcache1FetchStage2(pCache, iKey, createFlag); - }else{ - return 0; + pPage = pcache1FetchStage2(pCache, iKey, createFlag); } -} -#if PCACHE1_MIGHT_USE_GROUP_MUTEX -static PgHdr1 *pcache1FetchWithMutex( - sqlite3_pcache *p, - unsigned int iKey, - int createFlag -){ - PCache1 *pCache = (PCache1 *)p; - PgHdr1 *pPage; - - pcache1EnterMutex(pCache->pGroup); - pPage = pcache1FetchNoMutex(p, iKey, createFlag); assert( pPage==0 || pCache->iMaxKey>=iKey ); pcache1LeaveMutex(pCache->pGroup); - return pPage; -} -#endif -static sqlite3_pcache_page *pcache1Fetch( - sqlite3_pcache *p, - unsigned int iKey, - int createFlag -){ -#if PCACHE1_MIGHT_USE_GROUP_MUTEX || defined(SQLITE_DEBUG) - PCache1 *pCache = (PCache1 *)p; -#endif - - assert( offsetof(PgHdr1,page)==0 ); - assert( pCache->bPurgeable || createFlag!=1 ); - assert( pCache->bPurgeable || pCache->nMin==0 ); - assert( pCache->bPurgeable==0 || pCache->nMin==10 ); - assert( pCache->nMin==0 || pCache->bPurgeable ); - assert( pCache->nHash>0 ); -#if PCACHE1_MIGHT_USE_GROUP_MUTEX - if( pCache->pGroup->mutex ){ - return (sqlite3_pcache_page*)pcache1FetchWithMutex(p, iKey, createFlag); - }else -#endif - { - return (sqlite3_pcache_page*)pcache1FetchNoMutex(p, iKey, createFlag); - } + return (sqlite3_pcache_page*)pPage; } @@ -49282,16 +44417,22 @@ static void pcache1Unpin( ** part of the PGroup LRU list. */ assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); + assert( pGroup->pLruHead!=pPage && pGroup->pLruTail!=pPage ); assert( pPage->isPinned==1 ); if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){ - pcache1RemoveFromHash(pPage, 1); + pcache1RemoveFromHash(pPage); + pcache1FreePage(pPage); }else{ /* Add the page to the PGroup LRU list. */ - PgHdr1 **ppFirst = &pGroup->lru.pLruNext; - pPage->pLruPrev = &pGroup->lru; - (pPage->pLruNext = *ppFirst)->pLruPrev = pPage; - *ppFirst = pPage; + if( pGroup->pLruHead ){ + pGroup->pLruHead->pLruPrev = pPage; + pPage->pLruNext = pGroup->pLruHead; + pGroup->pLruHead = pPage; + }else{ + pGroup->pLruTail = pPage; + pGroup->pLruHead = pPage; + } pCache->nRecyclable++; pPage->isPinned = 0; } @@ -49362,15 +44503,14 @@ static void pcache1Destroy(sqlite3_pcache *p){ PGroup *pGroup = pCache->pGroup; assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); pcache1EnterMutex(pGroup); - if( pCache->nPage ) pcache1TruncateUnsafe(pCache, 0); + pcache1TruncateUnsafe(pCache, 0); assert( pGroup->nMaxPage >= pCache->nMax ); pGroup->nMaxPage -= pCache->nMax; assert( pGroup->nMinPage >= pCache->nMin ); pGroup->nMinPage -= pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; - pcache1EnforceMaxPage(pCache); + pcache1EnforceMaxPage(pGroup); pcache1LeaveMutex(pGroup); - sqlite3_free(pCache->pBulk); sqlite3_free(pCache->apHash); sqlite3_free(pCache); } @@ -49426,20 +44566,18 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ int nFree = 0; assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); assert( sqlite3_mutex_notheld(pcache1.mutex) ); - if( sqlite3GlobalConfig.nPage==0 ){ + if( pcache1.pStart==0 ){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); - while( (nReq<0 || nFreeisAnchor==0 - ){ + while( (nReq<0 || nFreepage.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER nFree += sqlite3MemSize(p); #endif assert( p->isPinned==0 ); pcache1PinPage(p); - pcache1RemoveFromHash(p, 1); + pcache1RemoveFromHash(p); + pcache1FreePage(p); } pcache1LeaveMutex(&pcache1.grp); } @@ -49460,7 +44598,7 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ){ PgHdr1 *p; int nRecyclable = 0; - for(p=pcache1.grp.lru.pLruNext; p && !p->isAnchor; p=p->pLruNext){ + for(p=pcache1.grp.pLruHead; p; p=p->pLruNext){ assert( p->isPinned==0 ); nRecyclable++; } @@ -49532,11 +44670,9 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST ** primitives are constant time. The cost of DESTROY is O(N). ** -** TEST and SMALLEST may not be used by the same RowSet. This used to -** be possible, but the feature was not used, so it was removed in order -** to simplify the code. +** There is an added cost of O(N) when switching between TEST and +** SMALLEST primitives. */ -/* #include "sqliteInt.h" */ /* @@ -49655,11 +44791,9 @@ SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){ */ static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ assert( p!=0 ); - if( p->nFresh==0 ){ /*OPTIMIZATION-IF-FALSE*/ - /* We could allocate a fresh RowSetEntry each time one is needed, but it - ** is more efficient to pull a preallocated entry from the pool */ + if( p->nFresh==0 ){ struct RowSetChunk *pNew; - pNew = sqlite3DbMallocRawNN(p->db, sizeof(*pNew)); + pNew = sqlite3DbMallocRaw(p->db, sizeof(*pNew)); if( pNew==0 ){ return 0; } @@ -49691,9 +44825,7 @@ SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet *p, i64 rowid){ pEntry->pRight = 0; pLast = p->pLast; if( pLast ){ - if( rowid<=pLast->v ){ /*OPTIMIZATION-IF-FALSE*/ - /* Avoid unnecessary sorts by preserving the ROWSET_SORTED flags - ** where possible */ + if( (p->rsFlags & ROWSET_SORTED)!=0 && rowid<=pLast->v ){ p->rsFlags &= ~ROWSET_SORTED; } pLast->pRight = pEntry; @@ -49717,26 +44849,28 @@ static struct RowSetEntry *rowSetEntryMerge( struct RowSetEntry *pTail; pTail = &head; - assert( pA!=0 && pB!=0 ); - for(;;){ + while( pA && pB ){ assert( pA->pRight==0 || pA->v<=pA->pRight->v ); assert( pB->pRight==0 || pB->v<=pB->pRight->v ); - if( pA->v<=pB->v ){ - if( pA->vv ) pTail = pTail->pRight = pA; + if( pA->vv ){ + pTail->pRight = pA; pA = pA->pRight; - if( pA==0 ){ - pTail->pRight = pB; - break; - } - }else{ - pTail = pTail->pRight = pB; + pTail = pTail->pRight; + }else if( pB->vv ){ + pTail->pRight = pB; pB = pB->pRight; - if( pB==0 ){ - pTail->pRight = pA; - break; - } + pTail = pTail->pRight; + }else{ + pA = pA->pRight; } } + if( pA ){ + assert( pA->pRight==0 || pA->v<=pA->pRight->v ); + pTail->pRight = pA; + }else{ + assert( pB==0 || pB->pRight==0 || pB->v<=pB->pRight->v ); + pTail->pRight = pB; + } return head.pRight; } @@ -49759,10 +44893,9 @@ static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ aBucket[i] = pIn; pIn = pNext; } - pIn = aBucket[0]; - for(i=1; i1 ){ /*OPTIMIZATION-IF-TRUE*/ - /* This branch causes a *balanced* tree to be generated. A valid tree - ** is still generated without this branch, but the tree is wildly - ** unbalanced and inefficient. */ - pLeft = rowSetNDeepTree(ppList, iDepth-1); - p = *ppList; - if( p==0 ){ /*OPTIMIZATION-IF-FALSE*/ - /* It is safe to always return here, but the resulting tree - ** would be unbalanced */ - return pLeft; - } - p->pLeft = pLeft; - *ppList = p->pRight; - p->pRight = rowSetNDeepTree(ppList, iDepth-1); - }else{ + if( *ppList==0 ){ + return 0; + } + if( iDepth==1 ){ p = *ppList; *ppList = p->pRight; p->pLeft = p->pRight = 0; + return p; + } + pLeft = rowSetNDeepTree(ppList, iDepth-1); + p = *ppList; + if( p==0 ){ + return pLeft; } + p->pLeft = pLeft; + *ppList = p->pRight; + p->pRight = rowSetNDeepTree(ppList, iDepth-1); return p; } @@ -49863,37 +44990,59 @@ static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ return p; } +/* +** Take all the entries on p->pEntry and on the trees in p->pForest and +** sort them all together into one big ordered list on p->pEntry. +** +** This routine should only be called once in the life of a RowSet. +*/ +static void rowSetToList(RowSet *p){ + + /* This routine is called only once */ + assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); + + if( (p->rsFlags & ROWSET_SORTED)==0 ){ + p->pEntry = rowSetEntrySort(p->pEntry); + } + + /* While this module could theoretically support it, sqlite3RowSetNext() + ** is never called after sqlite3RowSetText() for the same RowSet. So + ** there is never a forest to deal with. Should this change, simply + ** remove the assert() and the #if 0. */ + assert( p->pForest==0 ); +#if 0 + while( p->pForest ){ + struct RowSetEntry *pTree = p->pForest->pLeft; + if( pTree ){ + struct RowSetEntry *pHead, *pTail; + rowSetTreeToList(pTree, &pHead, &pTail); + p->pEntry = rowSetEntryMerge(p->pEntry, pHead); + } + p->pForest = p->pForest->pRight; + } +#endif + p->rsFlags |= ROWSET_NEXT; /* Verify this routine is never called again */ +} + /* ** Extract the smallest element from the RowSet. ** Write the element into *pRowid. Return 1 on success. Return ** 0 if the RowSet is already empty. ** ** After this routine has been called, the sqlite3RowSetInsert() -** routine may not be called again. -** -** This routine may not be called after sqlite3RowSetTest() has -** been used. Older versions of RowSet allowed that, but as the -** capability was not used by the code generator, it was removed -** for code economy. +** routine may not be called again. */ SQLITE_PRIVATE int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ assert( p!=0 ); - assert( p->pForest==0 ); /* Cannot be used with sqlite3RowSetText() */ /* Merge the forest into a single sorted list on first call */ - if( (p->rsFlags & ROWSET_NEXT)==0 ){ /*OPTIMIZATION-IF-FALSE*/ - if( (p->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ - p->pEntry = rowSetEntrySort(p->pEntry); - } - p->rsFlags |= ROWSET_SORTED|ROWSET_NEXT; - } + if( (p->rsFlags & ROWSET_NEXT)==0 ) rowSetToList(p); /* Return the next entry on the list */ if( p->pEntry ){ *pRowid = p->pEntry->v; p->pEntry = p->pEntry->pRight; - if( p->pEntry==0 ){ /*OPTIMIZATION-IF-TRUE*/ - /* Free memory immediately, rather than waiting on sqlite3_finalize() */ + if( p->pEntry==0 ){ sqlite3RowSetClear(p); } return 1; @@ -49916,15 +45065,13 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 /* This routine is never called after sqlite3RowSetNext() */ assert( pRowSet!=0 && (pRowSet->rsFlags & ROWSET_NEXT)==0 ); - /* Sort entries into the forest on the first test of a new batch. - ** To save unnecessary work, only do this when the batch number changes. + /* Sort entries into the forest on the first test of a new batch */ - if( iBatch!=pRowSet->iBatch ){ /*OPTIMIZATION-IF-FALSE*/ + if( iBatch!=pRowSet->iBatch ){ p = pRowSet->pEntry; if( p ){ struct RowSetEntry **ppPrevTree = &pRowSet->pForest; - if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ - /* Only sort the current set of entiries if they need it */ + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ p = rowSetEntrySort(p); } for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ @@ -49995,7 +45142,6 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 ** another is writing. */ #ifndef SQLITE_OMIT_DISKIO -/* #include "sqliteInt.h" */ /************** Include wal.h in the middle of pager.c ***********************/ /************** Begin file wal.h *********************************************/ /* @@ -50014,10 +45160,9 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 ** the implementation of each function in log.c for further details. */ -#ifndef SQLITE_WAL_H -#define SQLITE_WAL_H +#ifndef _WAL_H_ +#define _WAL_H_ -/* #include "sqliteInt.h" */ /* Additional values that can be added to the sync_flags argument of ** sqlite3WalFrames(): @@ -50044,7 +45189,6 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 -# define sqlite3WalFile(x) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -50127,11 +45271,6 @@ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op); */ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); -#ifdef SQLITE_ENABLE_SNAPSHOT -SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot); -SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot); -#endif - #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). @@ -50139,11 +45278,8 @@ SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapsho SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal); #endif -/* Return the sqlite3_file object for the WAL file */ -SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal); - #endif /* ifndef SQLITE_OMIT_WAL */ -#endif /* SQLITE_WAL_H */ +#endif /* _WAL_H_ */ /************** End of wal.h *************************************************/ /************** Continuing where we left off in pager.c **********************/ @@ -50554,7 +45690,6 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ */ #define MAX_SECTOR_SIZE 0x10000 - /* ** An instance of the following structure is allocated for each active ** savepoint and statement transaction in the system. All such structures @@ -50583,9 +45718,9 @@ struct PagerSavepoint { /* ** Bits of the Pager.doNotSpill flag. See further description below. */ -#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */ -#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */ -#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ +#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */ +#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */ +#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ /* ** An open page cache is an instance of struct Pager. A description of @@ -50667,11 +45802,11 @@ struct PagerSavepoint { ** while it is being traversed by code in pager_playback(). The SPILLFLAG_OFF ** case is a user preference. ** -** If the SPILLFLAG_NOSYNC bit is set, writing to the database from -** pagerStress() is permitted, but syncing the journal file is not. -** This flag is set by sqlite3PagerWrite() when the file-system sector-size -** is larger than the database page-size in order to prevent a journal sync -** from happening in between the journalling of two pages on the same sector. +** If the SPILLFLAG_NOSYNC bit is set, writing to the database from pagerStress() +** is permitted, but syncing the journal file is not. This flag is set +** by sqlite3PagerWrite() when the file-system sector-size is larger than +** the database page-size in order to prevent a journal sync from happening +** in between the journalling of two pages on the same sector. ** ** subjInMemory ** @@ -50750,7 +45885,6 @@ struct Pager { u8 useJournal; /* Use a rollback journal on this file */ u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ - u8 extraSync; /* sync directory after journal delete */ u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ @@ -50775,7 +45909,7 @@ struct Pager { u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 subjInMemory; /* True to use in-memory sub-journals */ u8 bUseFetch; /* True to use xFetch() */ - u8 hasHeldSharedLock; /* True if a shared lock has ever been held */ + u8 hasBeenUsed; /* True if any content previously read from this pager*/ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ @@ -50936,17 +46070,16 @@ static const unsigned char aJournalMagic[] = { ** ** if( pPager->jfd->pMethods ){ ... */ -#define isOpen(pFd) ((pFd)->pMethods!=0) +#define isOpen(pFd) ((pFd)->pMethods) /* ** Return true if this pager uses a write-ahead log instead of the usual ** rollback journal. Otherwise false. */ #ifndef SQLITE_OMIT_WAL -SQLITE_PRIVATE int sqlite3PagerUseWal(Pager *pPager){ +static int pagerUseWal(Pager *pPager){ return (pPager->pWal!=0); } -# define pagerUseWal(x) sqlite3PagerUseWal(x) #else # define pagerUseWal(x) 0 # define pagerRollbackWal(x) 0 @@ -50999,7 +46132,6 @@ static int assert_pager_state(Pager *p){ ** state. */ if( MEMDB ){ - assert( !isOpen(p->fd) ); assert( p->noSync ); assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_MEMORY @@ -51086,7 +46218,7 @@ static int assert_pager_state(Pager *p){ ** back to OPEN state. */ assert( pPager->errCode!=SQLITE_OK ); - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 || pPager->tempFile ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); break; } @@ -51161,21 +46293,19 @@ static int subjRequiresPage(PgHdr *pPg){ int i; for(i=0; inSavepoint; i++){ p = &pPager->aSavepoint[i]; - if( p->nOrig>=pgno && 0==sqlite3BitvecTestNotNull(p->pInSavepoint, pgno) ){ + if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){ return 1; } } return 0; } -#ifdef SQLITE_DEBUG /* ** Return true if the page is already in the journal file. */ static int pageInJournal(Pager *pPager, PgHdr *pPg){ return sqlite3BitvecTest(pPager->pInJournal, pPg->pgno); } -#endif /* ** Read a 32-bit integer from the given file descriptor. Store the integer @@ -51298,8 +46428,6 @@ static int jrnlBufferSize(Pager *pPager){ return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); } -#else -# define jrnlBufferSize(x) 0 #endif /* @@ -51460,7 +46588,6 @@ static i64 journalHdrOffset(Pager *pPager){ static int zeroJournalHdr(Pager *pPager, int doTruncate){ int rc = SQLITE_OK; /* Return code */ assert( isOpen(pPager->jfd) ); - assert( !sqlite3JournalIsInMemory(pPager->jfd) ); if( pPager->journalOff ){ const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ @@ -51790,8 +46917,7 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ || (0 != (rc = sqlite3OsWrite(pPager->jfd, zMaster, nMaster, iHdrOff+4))) || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster, nMaster))) || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster+4, cksum))) - || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, - iHdrOff+4+nMaster+8))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, iHdrOff+4+nMaster+8))) ){ return rc; } @@ -51842,7 +46968,7 @@ static void releaseAllSavepoints(Pager *pPager){ for(ii=0; iinSavepoint; ii++){ sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); } - if( !pPager->exclusiveMode || sqlite3JournalIsInMemory(pPager->sjfd) ){ + if( !pPager->exclusiveMode || sqlite3IsMemJournal(pPager->sjfd) ){ sqlite3OsClose(pPager->sjfd); } sqlite3_free(pPager->aSavepoint); @@ -51948,17 +47074,13 @@ static void pager_unlock(Pager *pPager){ ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. */ - assert( pPager->errCode==SQLITE_OK || !MEMDB ); if( pPager->errCode ){ - if( pPager->tempFile==0 ){ - pager_reset(pPager); - pPager->changeCountDone = 0; - pPager->eState = PAGER_OPEN; - }else{ - pPager->eState = (isOpen(pPager->jfd) ? PAGER_OPEN : PAGER_READER); - } - if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + assert( !MEMDB ); + pager_reset(pPager); + pPager->changeCountDone = pPager->tempFile; + pPager->eState = PAGER_OPEN; pPager->errCode = SQLITE_OK; + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); } pPager->journalOff = 0; @@ -52002,29 +47124,6 @@ static int pager_error(Pager *pPager, int rc){ static int pager_truncate(Pager *pPager, Pgno nPage); -/* -** The write transaction open on pPager is being committed (bCommit==1) -** or rolled back (bCommit==0). -** -** Return TRUE if and only if all dirty pages should be flushed to disk. -** -** Rules: -** -** * For non-TEMP databases, always sync to disk. This is necessary -** for transactions to be durable. -** -** * Sync TEMP database only on a COMMIT (not a ROLLBACK) when the backing -** file has been created already (via a spill on pagerStress()) and -** when the number of dirty pages in memory exceeds 25% of the total -** cache size. -*/ -static int pagerFlushOnCommit(Pager *pPager, int bCommit){ - if( pPager->tempFile==0 ) return 1; - if( !bCommit ) return 0; - if( !isOpen(pPager->fd) ) return 0; - return (sqlite3PCachePercentDirty(pPager->pPCache)>=25); -} - /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -52107,8 +47206,8 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ assert( !pagerUseWal(pPager) ); /* Finalize the journal file. */ - if( sqlite3JournalIsInMemory(pPager->jfd) ){ - /* assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); */ + if( sqlite3IsMemJournal(pPager->jfd) ){ + assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); sqlite3OsClose(pPager->jfd); }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){ if( pPager->journalOff==0 ){ @@ -52128,23 +47227,22 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) ){ - rc = zeroJournalHdr(pPager, hasMaster||pPager->tempFile); + rc = zeroJournalHdr(pPager, hasMaster); pPager->journalOff = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal ** file should be closed and deleted. If this connection writes to - ** the database file, it will do so using an in-memory journal. + ** the database file, it will do so using an in-memory journal. */ - int bDelete = !pPager->tempFile; - assert( sqlite3JournalIsInMemory(pPager->jfd)==0 ); + int bDelete = (!pPager->tempFile && sqlite3JournalExists(pPager->jfd)); assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_WAL ); sqlite3OsClose(pPager->jfd); if( bDelete ){ - rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, pPager->extraSync); + rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } } } @@ -52163,14 +47261,8 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - if( rc==SQLITE_OK ){ - if( pagerFlushOnCommit(pPager, bCommit) ){ - sqlite3PcacheCleanAll(pPager->pPCache); - }else{ - sqlite3PcacheClearWritable(pPager->pPCache); - } - sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); - } + sqlite3PcacheCleanAll(pPager->pPCache); + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); if( pagerUseWal(pPager) ){ /* Drop the WAL write-lock, if any. Also, if the connection was in @@ -52283,20 +47375,6 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif -#ifdef SQLITE_HAS_CODEC -/* -** Make sure the number of reserved bits is the same in the destination -** pager as it is in the source. This comes up when a VACUUM changes the -** number of reserved bits to the "optimal" amount. -*/ -SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ - if( pDest->nReserve!=pSrc->nReserve ){ - pDest->nReserve = pSrc->nReserve; - pagerReportSize(pDest); - } -} -#endif - /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -52399,7 +47477,7 @@ static int pager_playback_one_page( } } - /* If this page has already been played back before during the current + /* If this page has already been played by before during the current ** rollback, then don't bother to play it back again. */ if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ @@ -52454,7 +47532,7 @@ static int pager_playback_one_page( pPg = sqlite3PagerLookup(pPager, pgno); } assert( pPg || !MEMDB ); - assert( pPager->eState!=PAGER_OPEN || pPg==0 || pPager->tempFile ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), (isMainJrnl?"main-journal":"sub-journal") @@ -52476,9 +47554,9 @@ static int pager_playback_one_page( pPager->dbFileSize = pgno; } if( pPager->pBackup ){ - CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); + CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM); sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); - CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT, aData); + CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData); } }else if( !isMainJrnl && pPg==0 ){ /* If this is a rollback of a savepoint and data was not written to @@ -52500,10 +47578,11 @@ static int pager_playback_one_page( assert( isSavepnt ); assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)==0 ); pPager->doNotSpill |= SPILLFLAG_ROLLBACK; - rc = sqlite3PagerGet(pPager, pgno, &pPg, 1); + rc = sqlite3PagerAcquire(pPager, pgno, &pPg, 1); assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)!=0 ); pPager->doNotSpill &= ~SPILLFLAG_ROLLBACK; if( rc!=SQLITE_OK ) return rc; + pPg->flags &= ~PGHDR_NEED_READ; sqlite3PcacheMakeDirty(pPg); } if( pPg ){ @@ -52517,10 +47596,29 @@ static int pager_playback_one_page( pData = pPg->pData; memcpy(pData, (u8*)aData, pPager->pageSize); pPager->xReiniter(pPg); - /* It used to be that sqlite3PcacheMakeClean(pPg) was called here. But - ** that call was dangerous and had no detectable benefit since the cache - ** is normally cleaned by sqlite3PcacheCleanAll() after rollback and so - ** has been removed. */ + if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){ + /* If the contents of this page were just restored from the main + ** journal file, then its content must be as they were when the + ** transaction was first opened. In this case we can mark the page + ** as clean, since there will be no need to write it out to the + ** database. + ** + ** There is one exception to this rule. If the page is being rolled + ** back as part of a savepoint (or statement) rollback from an + ** unsynced portion of the main journal file, then it is not safe + ** to mark the page as clean. This is because marking the page as + ** clean will clear the PGHDR_NEED_SYNC flag. Since the page is + ** already in the journal file (recorded in Pager.pInJournal) and + ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to + ** again within this transaction, it will be marked as dirty but + ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially + ** be written out into the database file before its journal file + ** segment is synced. If a crash occurs during or following this, + ** database corruption may ensue. + */ + assert( !pagerUseWal(pPager) ); + sqlite3PcacheMakeClean(pPg); + } pager_set_pagehash(pPg); /* If this was page 1, then restore the value of Pager.dbFileVers. @@ -52530,7 +47628,7 @@ static int pager_playback_one_page( } /* Decode the page just read from disk */ - CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM_BKPT); + CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM); sqlite3PcacheRelease(pPg); } return rc; @@ -52596,7 +47694,7 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ pMaster = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); pJournal = (sqlite3_file *)(((u8 *)pMaster) + pVfs->szOsFile); if( !pMaster ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; }else{ const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); rc = sqlite3OsOpen(pVfs, zMaster, pMaster, flags, 0); @@ -52613,7 +47711,7 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ nMasterPtr = pVfs->mxPathname+1; zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 1); if( !zMasterJournal ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; goto delmaster_out; } zMasterPtr = &zMasterJournal[nMasterJournal+1]; @@ -52861,7 +47959,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** TODO: Technically the following is an error because it assumes that ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, - ** mxPathname is 512, which is the same as the minimum allowable value + ** mxPathname is 512, which is the same as the minimum allowable value ** for pageSize. */ zMaster = pPager->pTmpSpace; @@ -53083,7 +48181,7 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); } } - CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM_BKPT); + CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM); PAGER_INCR(sqlite3_pager_readdb_count); PAGER_INCR(pPager->nRead); @@ -53311,20 +48409,22 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ */ assert( pPager->eState==PAGER_OPEN ); assert( pPager->eLock>=SHARED_LOCK ); - assert( isOpen(pPager->fd) ); - assert( pPager->tempFile==0 ); nPage = sqlite3WalDbsize(pPager->pWal); - /* If the number of pages in the database is not available from the - ** WAL sub-system, determine the page counte based on the size of - ** the database file. If the size of the database file is not an - ** integer multiple of the page-size, round up the result. + /* If the database size was not available from the WAL sub-system, + ** determine it based on the size of the database file. If the size + ** of the database file is not an integer multiple of the page-size, + ** round down to the nearest page. Except, any file larger than 0 + ** bytes in size is considered to contain at least one page. */ - if( nPage==0 && ALWAYS(isOpen(pPager->fd)) ){ + if( nPage==0 ){ i64 n = 0; /* Size of db file in bytes */ - int rc = sqlite3OsFileSize(pPager->fd, &n); - if( rc!=SQLITE_OK ){ - return rc; + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( isOpen(pPager->fd) ){ + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; + } } nPage = (Pgno)((n+pPager->pageSize-1) / pPager->pageSize); } @@ -53442,7 +48542,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ if( pSavepoint ){ pDone = sqlite3BitvecCreate(pSavepoint->nOrig); if( !pDone ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } } @@ -53538,21 +48638,12 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ } /* -** Change the maximum number of in-memory pages that are allowed -** before attempting to recycle clean and unused pages. +** Change the maximum number of in-memory pages that are allowed. */ SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } -/* -** Change the maximum number of in-memory pages that are allowed -** before attempting to spill pages to journal. -*/ -SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager *pPager, int mxPage){ - return sqlite3PcacheSetSpillsize(pPager->pPCache, mxPage); -} - /* ** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. */ @@ -53589,7 +48680,7 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** The "level" in pgFlags & PAGER_SYNCHRONOUS_MASK sets the robustness ** of the database to damage due to OS crashes or power failures by ** changing the number of syncs()s when writing the journals. -** There are four levels: +** There are three levels: ** ** OFF sqlite3OsSync() is never called. This is the default ** for temporary and transient files. @@ -53609,10 +48700,6 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** assurance that the journal will not be corrupted to the ** point of causing damage to the database during rollback. ** -** EXTRA This is like FULL except that is also syncs the directory -** that contains the rollback journal after the rollback -** journal is unlinked. -** ** The above is for a rollback-journal mode. For WAL mode, OFF continues ** to mean that no syncs ever occur. NORMAL means that the WAL is synced ** prior to the start of checkpoint and that the database file is synced @@ -53620,8 +48707,7 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** was written back into the database. But no sync operations occur for ** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL ** file is synced following each commit operation, in addition to the -** syncs associated with NORMAL. There is no difference between FULL -** and EXTRA for WAL mode. +** syncs associated with NORMAL. ** ** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The ** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync @@ -53640,15 +48726,9 @@ SQLITE_PRIVATE void sqlite3PagerSetFlags( unsigned pgFlags /* Various flags */ ){ unsigned level = pgFlags & PAGER_SYNCHRONOUS_MASK; - if( pPager->tempFile ){ - pPager->noSync = 1; - pPager->fullSync = 0; - pPager->extraSync = 0; - }else{ - pPager->noSync = level==PAGER_SYNCHRONOUS_OFF ?1:0; - pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; - pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; - } + assert( level>=1 && level<=3 ); + pPager->noSync = (level==1 || pPager->tempFile) ?1:0; + pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; if( pPager->noSync ){ pPager->syncFlags = 0; pPager->ckptSyncFlags = 0; @@ -53810,7 +48890,7 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR } if( rc==SQLITE_OK ){ pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ) rc = SQLITE_NOMEM_BKPT; + if( !pNew ) rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ @@ -54086,7 +49166,7 @@ static int pagerAcquireMapPage( *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } p->pExtra = (void *)&p[1]; p->flags = PGHDR_MMAP; @@ -54400,9 +49480,8 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->eState==PAGER_WRITER_DBMOD ); assert( pPager->eLock==EXCLUSIVE_LOCK ); - assert( isOpen(pPager->fd) || pList->pDirty==0 ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -54445,7 +49524,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pList->pgno==1 ) pager_write_changecounter(pList); /* Encode the database */ - CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM_BKPT, pData); + CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); @@ -54490,20 +49569,19 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ static int openSubJournal(Pager *pPager){ int rc = SQLITE_OK; if( !isOpen(pPager->sjfd) ){ - const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE - | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE - | SQLITE_OPEN_DELETEONCLOSE; - int nStmtSpill = sqlite3Config.nStmtSpill; if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ - nStmtSpill = -1; + sqlite3MemJournalOpen(pPager->sjfd); + }else{ + rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL); } - rc = sqlite3JournalOpen(pPager->pVfs, 0, pPager->sjfd, flags, nStmtSpill); } return rc; } /* ** Append a record of the current state of page pPg to the sub-journal. +** It is the callers responsibility to use subjRequiresPage() to check +** that it is really required before calling this function. ** ** If successful, set the bit corresponding to pPg->pgno in the bitvecs ** for all open savepoints before returning. @@ -54535,7 +49613,7 @@ static int subjournalPage(PgHdr *pPg){ i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); char *pData2; - CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); + CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); rc = write32bits(pPager->sjfd, offset, pPg->pgno); if( rc==SQLITE_OK ){ @@ -54550,13 +49628,6 @@ static int subjournalPage(PgHdr *pPg){ } return rc; } -static int subjournalPageIfRequired(PgHdr *pPg){ - if( subjRequiresPage(pPg) ){ - return subjournalPage(pPg); - }else{ - return SQLITE_OK; - } -} /* ** This function is called by the pcache layer when it has reached some @@ -54614,7 +49685,9 @@ static int pagerStress(void *p, PgHdr *pPg){ pPg->pDirty = 0; if( pagerUseWal(pPager) ){ /* Write a single frame for this page to the log. */ - rc = subjournalPageIfRequired(pPg); + if( subjRequiresPage(pPg) ){ + rc = subjournalPage(pPg); + } if( rc==SQLITE_OK ){ rc = pagerWalFrames(pPager, pPg, 0, 0); } @@ -54627,6 +49700,39 @@ static int pagerStress(void *p, PgHdr *pPg){ rc = syncJournal(pPager, 1); } + /* If the page number of this page is larger than the current size of + ** the database image, it may need to be written to the sub-journal. + ** This is because the call to pager_write_pagelist() below will not + ** actually write data to the file in this case. + ** + ** Consider the following sequence of events: + ** + ** BEGIN; + ** + ** + ** SAVEPOINT sp; + ** + ** pagerStress(page X) + ** ROLLBACK TO sp; + ** + ** If (X>Y), then when pagerStress is called page X will not be written + ** out to the database file, but will be dropped from the cache. Then, + ** following the "ROLLBACK TO sp" statement, reading page X will read + ** data from the database file. This will be the copy of page X as it + ** was when the transaction started, not as it was when "SAVEPOINT sp" + ** was executed. + ** + ** The solution is to write the current data for page X into the + ** sub-journal file now (if it is not already there), so that it will + ** be restored to its current value when the "ROLLBACK TO sp" is + ** executed. + */ + if( NEVER( + rc==SQLITE_OK && pPg->pgno>pPager->dbSize && subjRequiresPage(pPg) + ) ){ + rc = subjournalPage(pPg); + } + /* Write the contents of the page out to the database file. */ if( rc==SQLITE_OK ){ assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); @@ -54643,25 +49749,6 @@ static int pagerStress(void *p, PgHdr *pPg){ return pager_error(pPager, rc); } -/* -** Flush all unreferenced dirty pages to disk. -*/ -SQLITE_PRIVATE int sqlite3PagerFlush(Pager *pPager){ - int rc = pPager->errCode; - if( !MEMDB ){ - PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); - assert( assert_pager_state(pPager) ); - while( rc==SQLITE_OK && pList ){ - PgHdr *pNext = pList->pDirty; - if( pList->nRef==0 ){ - rc = pagerStress((void*)pPager, pList); - } - pList = pNext; - } - } - - return rc; -} /* ** Allocate and initialize a new Pager object and put a pointer to it @@ -54718,8 +49805,18 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int nUri = 0; /* Number of bytes of URI args at *zUri */ /* Figure out how much space is required for each journal file-handle - ** (there are two of them, the main journal and the sub-journal). */ - journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); + ** (there are two of them, the main journal and the sub-journal). This + ** is the maximum space required for an in-memory journal file handle + ** and a regular journal file-handle. Note that a "regular journal-handle" + ** may be a wrapper capable of caching the first portion of the journal + ** file in memory to implement the atomic-write optimization (see + ** source file journal.c). + */ + if( sqlite3JournalSize(pVfs)>sqlite3MemJournalSize() ){ + journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); + }else{ + journalFileSize = ROUND8(sqlite3MemJournalSize()); + } /* Set the output variable to NULL in case an error occurs. */ *ppPager = 0; @@ -54729,7 +49826,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( memDb = 1; if( zFilename && zFilename[0] ){ zPathname = sqlite3DbStrDup(0, zFilename); - if( zPathname==0 ) return SQLITE_NOMEM_BKPT; + if( zPathname==0 ) return SQLITE_NOMEM; nPathname = sqlite3Strlen30(zPathname); zFilename = 0; } @@ -54745,7 +49842,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( nPathname = pVfs->mxPathname+1; zPathname = sqlite3DbMallocRaw(0, nPathname*2); if( zPathname==0 ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); @@ -54798,7 +49895,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); if( !pPtr ){ sqlite3DbFree(0, zPathname); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } pPager = (Pager*)(pPtr); pPager->pPCache = (PCache*)(pPtr += ROUND8(sizeof(*pPager))); @@ -54891,7 +49988,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( act_like_temp_file: tempFile = 1; pPager->eState = PAGER_READER; /* Pretend we already have a lock */ - pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE mode */ + pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE locking mode */ pPager->noLock = 1; /* Do no locking */ readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } @@ -54910,7 +50007,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( assert( nExtra<1000 ); nExtra = ROUND8(nExtra); rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, - !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); } /* If an error occurred above, free the Pager structure and close the file. @@ -54947,13 +50044,11 @@ SQLITE_PRIVATE int sqlite3PagerOpen( pPager->noSync = pPager->tempFile; if( pPager->noSync ){ assert( pPager->fullSync==0 ); - assert( pPager->extraSync==0 ); assert( pPager->syncFlags==0 ); assert( pPager->walSyncFlags==0 ); assert( pPager->ckptSyncFlags==0 ); }else{ pPager->fullSync = 1; - pPager->extraSync = 0; pPager->syncFlags = SQLITE_SYNC_NORMAL; pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS; pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; @@ -55070,7 +50165,6 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK && !locked ){ Pgno nPage; /* Number of pages in database file */ - assert( pPager->tempFile==0 ); rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ /* If the database is zero pages in size, that means that either (1) the @@ -55132,7 +50226,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ /* ** This function is called to obtain a shared lock on the database file. -** It is illegal to call sqlite3PagerGet() until after this function +** It is illegal to call sqlite3PagerAcquire() until after this function ** has been successfully called. If a shared-lock is already held when ** this function is called, it is a no-op. ** @@ -55163,17 +50257,17 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ /* This routine is only called from b-tree and only when there are no ** outstanding pages. This implies that the pager state should either ** be OPEN or READER. READER is only possible if the pager is or was in - ** exclusive access mode. */ + ** exclusive access mode. + */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); - assert( pPager->errCode==SQLITE_OK ); + if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ int bHotJournal = 1; /* True if there exists a hot journal-file */ assert( !MEMDB ); - assert( pPager->tempFile==0 || pPager->eLock==EXCLUSIVE_LOCK ); rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ @@ -55259,7 +50353,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ assert( rc==SQLITE_OK ); rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ - rc = pager_playback(pPager, !pPager->tempFile); + rc = pager_playback(pPager, 1); pPager->eState = PAGER_OPEN; } }else if( !pPager->exclusiveMode ){ @@ -55293,14 +50387,14 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile && pPager->hasHeldSharedLock ){ + if( !pPager->tempFile && pPager->hasBeenUsed ){ /* The shared-lock has just been acquired then check to ** see if the database has been modified. If the database has changed, - ** flush the cache. The hasHeldSharedLock flag prevents this from + ** flush the cache. The pPager->hasBeenUsed flag prevents this from ** occurring on the very first access to a file, in order to save a ** single unnecessary sqlite3OsRead() call at the start-up. ** - ** Database changes are detected by looking at 15 bytes beginning + ** Database changes is detected by looking at 15 bytes beginning ** at offset 24 into the file. The first 4 of these 16 bytes are ** a 32-bit counter that is incremented with each change. The ** other bytes change randomly with each file change when @@ -55355,7 +50449,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerBeginReadTransaction(pPager); } - if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ rc = pagerPagecount(pPager, &pPager->dbSize); } @@ -55366,7 +50460,6 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ assert( pPager->eState==PAGER_OPEN ); }else{ pPager->eState = PAGER_READER; - pPager->hasHeldSharedLock = 1; } return rc; } @@ -55435,7 +50528,7 @@ static void pagerUnlockIfUnused(Pager *pPager){ ** Since Lookup() never goes to disk, it never has to deal with locks ** or journal files. */ -SQLITE_PRIVATE int sqlite3PagerGet( +SQLITE_PRIVATE int sqlite3PagerAcquire( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ @@ -55450,25 +50543,21 @@ SQLITE_PRIVATE int sqlite3PagerGet( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pgno>1 && USEFETCH(pPager) + const int bMmapOk = (pgno!=1 && USEFETCH(pPager) && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) #ifdef SQLITE_HAS_CODEC && pPager->xCodec==0 #endif ); - /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here - ** allows the compiler optimizer to reuse the results of the "pgno>1" - ** test in the previous statement, and avoid testing pgno==0 in the - ** common case where pgno is large. */ - if( pgno<=1 && pgno==0 ){ - return SQLITE_CORRUPT_BKPT; - } assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); assert( noContent==0 || bMmapOk==0 ); - assert( pPager->hasHeldSharedLock==1 ); + if( pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } + pPager->hasBeenUsed = 1; /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ @@ -55488,7 +50577,7 @@ SQLITE_PRIVATE int sqlite3PagerGet( ); if( rc==SQLITE_OK && pData ){ - if( pPager->eState>PAGER_READER || pPager->tempFile ){ + if( pPager->eState>PAGER_READER ){ pPg = sqlite3PagerLookup(pPager, pgno); } if( pPg==0 ){ @@ -55513,14 +50602,9 @@ SQLITE_PRIVATE int sqlite3PagerGet( if( pBase==0 ){ rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); if( rc!=SQLITE_OK ) goto pager_acquire_err; - if( pBase==0 ){ - pPg = *ppPage = 0; - rc = SQLITE_NOMEM_BKPT; - goto pager_acquire_err; - } } pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); - assert( pPg!=0 ); + if( pPg==0 ) rc = SQLITE_NOMEM; } } @@ -55531,11 +50615,10 @@ SQLITE_PRIVATE int sqlite3PagerGet( pPg = 0; goto pager_acquire_err; } - assert( pPg==(*ppPage) ); - assert( pPg->pgno==pgno ); - assert( pPg->pPager==pPager || pPg->pPager==0 ); + assert( (*ppPage)->pgno==pgno ); + assert( (*ppPage)->pPager==pPager || (*ppPage)->pPager==0 ); - if( pPg->pPager && !noContent ){ + if( (*ppPage)->pPager && !noContent ){ /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); @@ -55546,6 +50629,7 @@ SQLITE_PRIVATE int sqlite3PagerGet( /* The pager cache has created a new page. Its content needs to ** be initialized. */ + pPg = *ppPage; pPg->pPager = pPager; /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page @@ -55555,8 +50639,7 @@ SQLITE_PRIVATE int sqlite3PagerGet( goto pager_acquire_err; } - assert( !isOpen(pPager->fd) || !MEMDB ); - if( !isOpen(pPager->fd) || pPager->dbSizedbSizefd) ){ if( pgno>pPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; @@ -55624,8 +50707,7 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pgno!=0 ); assert( pPager->pPCache!=0 ); pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); - assert( pPage==0 || pPager->hasHeldSharedLock ); - if( pPage==0 ) return 0; + assert( pPage==0 || pPager->hasBeenUsed ); return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); } @@ -55690,7 +50772,7 @@ static int pager_open_journal(Pager *pPager){ if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); if( pPager->pInJournal==0 ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } /* Open the journal file if it is not already open. */ @@ -55698,24 +50780,24 @@ static int pager_open_journal(Pager *pPager){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ sqlite3MemJournalOpen(pPager->jfd); }else{ - int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; - int nSpill; + const int flags = /* VFS flags to open journal file */ + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| + (pPager->tempFile ? + (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): + (SQLITE_OPEN_MAIN_JOURNAL) + ); - if( pPager->tempFile ){ - flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); - nSpill = sqlite3Config.nStmtSpill; - }else{ - flags |= SQLITE_OPEN_MAIN_JOURNAL; - nSpill = jrnlBufferSize(pPager); - } - /* Verify that the database still has the same name as it did when ** it was originally opened. */ rc = databaseIsUnmoved(pPager); if( rc==SQLITE_OK ){ - rc = sqlite3JournalOpen ( - pVfs, pPager->zJournal, pPager->jfd, flags, nSpill +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + rc = sqlite3JournalOpen( + pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) ); +#else + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); +#endif } } assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); @@ -55782,7 +50864,7 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory if( rc!=SQLITE_OK ){ return rc; } - (void)sqlite3WalExclusiveMode(pPager->pWal, 1); + sqlite3WalExclusiveMode(pPager->pWal, 1); } /* Grab the write lock on the log file. If successful, upgrade to @@ -55829,59 +50911,6 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory return rc; } -/* -** Write page pPg onto the end of the rollback journal. -*/ -static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ - Pager *pPager = pPg->pPager; - int rc; - u32 cksum; - char *pData2; - i64 iOff = pPager->journalOff; - - /* We should never write to the journal file the page that - ** contains the database locks. The following assert verifies - ** that we do not. */ - assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); - - assert( pPager->journalHdr<=pPager->journalOff ); - CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); - cksum = pager_cksum(pPager, (u8*)pData2); - - /* Even if an IO or diskfull error occurs while journalling the - ** page in the block above, set the need-sync flag for the page. - ** Otherwise, when the transaction is rolled back, the logic in - ** playback_one_page() will think that the page needs to be restored - ** in the database file. And if an IO error occurs while doing so, - ** then corruption may follow. - */ - pPg->flags |= PGHDR_NEED_SYNC; - - rc = write32bits(pPager->jfd, iOff, pPg->pgno); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); - if( rc!=SQLITE_OK ) return rc; - rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); - if( rc!=SQLITE_OK ) return rc; - - IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, - pPager->journalOff, pPager->pageSize)); - PAGER_INCR(sqlite3_pager_writej_count); - PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", - PAGERID(pPager), pPg->pgno, - ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); - - pPager->journalOff += 8 + pPager->pageSize; - pPager->nRec++; - assert( pPager->pInJournal!=0 ); - rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); - testcase( rc==SQLITE_NOMEM ); - assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); - rc |= addToSavepointBitvecs(pPager, pPg->pgno); - assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); - return rc; -} - /* ** Mark a single data page as writeable. The page is written into the ** main journal or sub-journal as required. If the page is written into @@ -55892,6 +50921,7 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ static int pager_write(PgHdr *pPg){ Pager *pPager = pPg->pPager; int rc = SQLITE_OK; + int inJournal; /* This routine is not called unless a write-transaction has already ** been started. The journal file may or may not be open at this point. @@ -55904,6 +50934,7 @@ static int pager_write(PgHdr *pPg){ assert( assert_pager_state(pPager) ); assert( pPager->errCode==0 ); assert( pPager->readOnly==0 ); + CHECK_PAGE(pPg); /* The journal file needs to be opened. Higher level routines have already @@ -55922,48 +50953,91 @@ static int pager_write(PgHdr *pPg){ assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); assert( assert_pager_state(pPager) ); - /* Mark the page that is about to be modified as dirty. */ - sqlite3PcacheMakeDirty(pPg); - - /* If a rollback journal is in use, them make sure the page that is about - ** to change is in the rollback journal, or if the page is a new page off - ** then end of the file, make sure it is marked as PGHDR_NEED_SYNC. + /* Mark the page as dirty. If the page has already been written + ** to the journal then we can return right away. */ - assert( (pPager->pInJournal!=0) == isOpen(pPager->jfd) ); - if( pPager->pInJournal!=0 - && sqlite3BitvecTestNotNull(pPager->pInJournal, pPg->pgno)==0 - ){ - assert( pagerUseWal(pPager)==0 ); - if( pPg->pgno<=pPager->dbOrigSize ){ - rc = pagerAddPageToRollbackJournal(pPg); - if( rc!=SQLITE_OK ){ - return rc; - } - }else{ - if( pPager->eState!=PAGER_WRITER_DBMOD ){ + sqlite3PcacheMakeDirty(pPg); + inJournal = pageInJournal(pPager, pPg); + if( inJournal && (pPager->nSavepoint==0 || !subjRequiresPage(pPg)) ){ + assert( !pagerUseWal(pPager) ); + }else{ + + /* The transaction journal now exists and we have a RESERVED or an + ** EXCLUSIVE lock on the main database file. Write the current page to + ** the transaction journal if it is not there already. + */ + if( !inJournal && !pagerUseWal(pPager) ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ + u32 cksum; + char *pData2; + i64 iOff = pPager->journalOff; + + /* We should never write to the journal file the page that + ** contains the database locks. The following assert verifies + ** that we do not. */ + assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); + + assert( pPager->journalHdr<=pPager->journalOff ); + CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + cksum = pager_cksum(pPager, (u8*)pData2); + + /* Even if an IO or diskfull error occurs while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ pPg->flags |= PGHDR_NEED_SYNC; + + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; + + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + pPager->journalOff, pPager->pageSize)); + PAGER_INCR(sqlite3_pager_writej_count); + PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); + + pPager->journalOff += 8 + pPager->pageSize; + pPager->nRec++; + assert( pPager->pInJournal!=0 ); + rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + rc |= addToSavepointBitvecs(pPager, pPg->pgno); + if( rc!=SQLITE_OK ){ + assert( rc==SQLITE_NOMEM ); + return rc; + } + }else{ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ + pPg->flags |= PGHDR_NEED_SYNC; + } + PAGERTRACE(("APPEND %d page %d needSync=%d\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); } - PAGERTRACE(("APPEND %d page %d needSync=%d\n", - PAGERID(pPager), pPg->pgno, - ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); } - } - - /* The PGHDR_DIRTY bit is set above when the page was added to the dirty-list - ** and before writing the page into the rollback journal. Wait until now, - ** after the page has been successfully journalled, before setting the - ** PGHDR_WRITEABLE bit that indicates that the page can be safely modified. - */ - pPg->flags |= PGHDR_WRITEABLE; - /* If the statement journal is open and the page is not in it, - ** then write the page into the statement journal. - */ - if( pPager->nSavepoint>0 ){ - rc = subjournalPageIfRequired(pPg); + /* If the statement journal is open and the page is not in it, + ** then write the current page to the statement journal. Note that + ** the statement journal format differs from the standard journal format + ** in that it omits the checksums and the header. + */ + if( pPager->nSavepoint>0 && subjRequiresPage(pPg) ){ + rc = subjournalPage(pPg); + } } - /* Update the database size and return. */ + /* Update the database size and return. + */ if( pPager->dbSizepgno ){ pPager->dbSize = pPg->pgno; } @@ -55978,17 +51052,17 @@ static int pager_write(PgHdr *pPg){ ** a write. ** ** Usually, the sector size is less than or equal to the page size, in which -** case pages can be individually written. This routine only runs in the -** exceptional case where the page size is smaller than the sector size. +** case pages can be individually written. This routine only runs in the exceptional +** case where the page size is smaller than the sector size. */ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ - int rc = SQLITE_OK; /* Return code */ - Pgno nPageCount; /* Total number of pages in database file */ - Pgno pg1; /* First page of the sector pPg is located on. */ - int nPage = 0; /* Number of pages starting at pg1 to journal */ - int ii; /* Loop counter */ - int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ - Pager *pPager = pPg->pPager; /* The pager that owns pPg */ + int rc = SQLITE_OK; /* Return code */ + Pgno nPageCount; /* Total number of pages in database file */ + Pgno pg1; /* First page of the sector pPg is located on. */ + int nPage = 0; /* Number of pages starting at pg1 to journal */ + int ii; /* Loop counter */ + int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ + Pager *pPager = pPg->pPager; /* The pager that owns pPg */ Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow @@ -56022,7 +51096,7 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ PgHdr *pPage; if( pg==pPg->pgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ if( pg!=PAGER_MJ_PGNO(pPager) ){ - rc = sqlite3PagerGet(pPager, pg, &pPage, 0); + rc = sqlite3PagerGet(pPager, pg, &pPage); if( rc==SQLITE_OK ){ rc = pager_write(pPage); if( pPage->flags&PGHDR_NEED_SYNC ){ @@ -56076,17 +51150,11 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ ** as appropriate. Otherwise, SQLITE_OK. */ SQLITE_PRIVATE int sqlite3PagerWrite(PgHdr *pPg){ - Pager *pPager = pPg->pPager; assert( (pPg->flags & PGHDR_MMAP)==0 ); - assert( pPager->eState>=PAGER_WRITER_LOCKED ); - assert( assert_pager_state(pPager) ); - if( pPager->errCode ){ - return pPager->errCode; - }else if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ - if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); - return SQLITE_OK; - }else if( pPager->sectorSize > (u32)pPager->pageSize ){ - assert( pPager->tempFile==0 ); + assert( pPg->pPager->eState>=PAGER_WRITER_LOCKED ); + assert( pPg->pPager->eState!=PAGER_ERROR ); + assert( assert_pager_state(pPg->pPager) ); + if( pPg->pPager->sectorSize > (u32)pPg->pPager->pageSize ){ return pagerWriteLargeSector(pPg); }else{ return pager_write(pPg); @@ -56100,7 +51168,7 @@ SQLITE_PRIVATE int sqlite3PagerWrite(PgHdr *pPg){ */ #ifndef NDEBUG SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){ - return pPg->flags & PGHDR_WRITEABLE; + return pPg->flags&PGHDR_DIRTY; } #endif @@ -56117,21 +51185,13 @@ SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){ ** ** Tests show that this optimization can quadruple the speed of large ** DELETE operations. -** -** This optimization cannot be used with a temp-file, as the page may -** have been dirty at the start of the transaction. In that case, if -** memory pressure forces page pPg out of the cache, the data does need -** to be written out to disk so that it may be read back in if the -** current transaction is rolled back. */ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ Pager *pPager = pPg->pPager; - if( !pPager->tempFile && (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ + if( (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) pPg->flags |= PGHDR_DONT_WRITE; - pPg->flags &= ~PGHDR_WRITEABLE; - testcase( pPg->flags & PGHDR_NEED_SYNC ); pager_set_pagehash(pPg); } } @@ -56190,7 +51250,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ assert( !pPager->tempFile && isOpen(pPager->fd) ); /* Open page 1 of the file for writing. */ - rc = sqlite3PagerGet(pPager, 1, &pPgHdr, 0); + rc = sqlite3PagerGet(pPager, 1, &pPgHdr); assert( pPgHdr==0 || rc==SQLITE_OK ); /* If page one was fetched successfully, and this function is not @@ -56210,7 +51270,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ if( DIRECT_MODE ){ const void *zBuf; assert( pPager->dbFileSize>0 ); - CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM_BKPT, zBuf); + CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); pPager->aStat[PAGER_STAT_WRITE]++; @@ -56268,17 +51328,14 @@ SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zMaster){ ** returned. */ SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ - int rc = pPager->errCode; + int rc = SQLITE_OK; + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); assert( assert_pager_state(pPager) ); - if( rc==SQLITE_OK ){ - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - || pPager->eState==PAGER_WRITER_LOCKED - ); - assert( assert_pager_state(pPager) ); - if( 0==pagerUseWal(pPager) ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - } + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } return rc; } @@ -56326,21 +51383,17 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( /* If a prior error occurred, report that error again. */ if( NEVER(pPager->errCode) ) return pPager->errCode; - /* Provide the ability to easily simulate an I/O error during testing */ - if( sqlite3FaultSim(400) ) return SQLITE_IOERR; - PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); /* If no database changes have been made, return early. */ if( pPager->eStatetempFile ); - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( 0==pagerFlushOnCommit(pPager, 1) ){ + if( MEMDB ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any - ** backup in progress needs to be restarted. */ + ** backup in progress needs to be restarted. + */ sqlite3BackupRestart(pPager->pBackup); }else{ if( pagerUseWal(pPager) ){ @@ -56349,7 +51402,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( if( pList==0 ){ /* Must have at least one page for the WAL commit flag. ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */ - rc = sqlite3PagerGet(pPager, 1, &pPageOne, 0); + rc = sqlite3PagerGet(pPager, 1, &pPageOne); pList = pPageOne; pList->pDirty = 0; } @@ -56605,14 +51658,12 @@ SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager *pPager){ return pPager->readOnly; } -#ifdef SQLITE_DEBUG /* -** Return the sum of the reference counts for all pages held by pPager. +** Return the number of references to the pager. */ SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ return sqlite3PcacheRefCount(pPager->pPCache); } -#endif /* ** Return the approximate number of bytes of memory currently @@ -56679,10 +51730,10 @@ SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, i } /* -** Return true if this is an in-memory or temp-file backed pager. +** Return true if this is an in-memory pager. */ SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ - return pPager->tempFile; + return MEMDB; } /* @@ -56695,63 +51746,55 @@ SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ ** occurs while opening the sub-journal file, then an IO error code is ** returned. Otherwise, SQLITE_OK. */ -static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ +SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ int rc = SQLITE_OK; /* Return code */ int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ - int ii; /* Iterator variable */ - PagerSavepoint *aNew; /* New Pager.aSavepoint array */ assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); - assert( nSavepoint>nCurrent && pPager->useJournal ); - /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM - ** if the allocation fails. Otherwise, zero the new portion in case a - ** malloc failure occurs while populating it in the for(...) loop below. - */ - aNew = (PagerSavepoint *)sqlite3Realloc( - pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint - ); - if( !aNew ){ - return SQLITE_NOMEM_BKPT; - } - memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); - pPager->aSavepoint = aNew; + if( nSavepoint>nCurrent && pPager->useJournal ){ + int ii; /* Iterator variable */ + PagerSavepoint *aNew; /* New Pager.aSavepoint array */ - /* Populate the PagerSavepoint structures just allocated. */ - for(ii=nCurrent; iidbSize; - if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ - aNew[ii].iOffset = pPager->journalOff; - }else{ - aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); - } - aNew[ii].iSubRec = pPager->nSubRec; - aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); - if( !aNew[ii].pInSavepoint ){ - return SQLITE_NOMEM_BKPT; + /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM + ** if the allocation fails. Otherwise, zero the new portion in case a + ** malloc failure occurs while populating it in the for(...) loop below. + */ + aNew = (PagerSavepoint *)sqlite3Realloc( + pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint + ); + if( !aNew ){ + return SQLITE_NOMEM; } - if( pagerUseWal(pPager) ){ - sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); + memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); + pPager->aSavepoint = aNew; + + /* Populate the PagerSavepoint structures just allocated. */ + for(ii=nCurrent; iidbSize; + if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ + aNew[ii].iOffset = pPager->journalOff; + }else{ + aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); + } + aNew[ii].iSubRec = pPager->nSubRec; + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); + if( !aNew[ii].pInSavepoint ){ + return SQLITE_NOMEM; + } + if( pagerUseWal(pPager) ){ + sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); + } + pPager->nSavepoint = ii+1; } - pPager->nSavepoint = ii+1; + assert( pPager->nSavepoint==nSavepoint ); + assertTruncateConstraint(pPager); } - assert( pPager->nSavepoint==nSavepoint ); - assertTruncateConstraint(pPager); - return rc; -} -SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ - assert( pPager->eState>=PAGER_WRITER_LOCKED ); - assert( assert_pager_state(pPager) ); - if( nSavepoint>pPager->nSavepoint && pPager->useJournal ){ - return pagerOpenSavepoint(pPager, nSavepoint); - }else{ - return SQLITE_OK; - } + return rc; } - /* ** This function is called to rollback or release (commit) a savepoint. ** The savepoint to release or rollback need not be the most recently @@ -56783,11 +51826,7 @@ SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ ** savepoint. If no errors occur, SQLITE_OK is returned. */ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ - int rc = pPager->errCode; - -#ifdef SQLITE_ENABLE_ZIPVFS - if( op==SAVEPOINT_RELEASE ) rc = SQLITE_OK; -#endif + int rc = pPager->errCode; /* Return code */ assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); @@ -56811,7 +51850,7 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ if( op==SAVEPOINT_RELEASE ){ if( nNew==0 && isOpen(pPager->sjfd) ){ /* Only truncate if it is an in-memory sub-journal. */ - if( sqlite3JournalIsInMemory(pPager->sjfd) ){ + if( sqlite3IsMemJournal(pPager->sjfd) ){ rc = sqlite3OsTruncate(pPager->sjfd, 0); assert( rc==SQLITE_OK ); } @@ -56828,20 +51867,6 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); } - -#ifdef SQLITE_ENABLE_ZIPVFS - /* If the cache has been modified but the savepoint cannot be rolled - ** back journal_mode=off, put the pager in the error state. This way, - ** if the VFS used by this pager includes ZipVFS, the entire transaction - ** can be rolled back at the ZipVFS level. */ - else if( - pPager->journalMode==PAGER_JOURNALMODE_OFF - && pPager->eState>=PAGER_WRITER_CACHEMOD - ){ - pPager->errCode = SQLITE_ABORT; - pPager->eState = PAGER_ERROR; - } -#endif } return rc; @@ -56864,7 +51889,7 @@ SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager *pPager, int nullIfMemDb){ /* ** Return the VFS structure for the pager. */ -SQLITE_PRIVATE sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ +SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ return pPager->pVfs; } @@ -56878,22 +51903,18 @@ SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ } /* -** Return the file handle for the journal file (if it exists). -** This will be either the rollback journal or the WAL file. +** Return the full pathname of the journal file. */ -SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ -#if SQLITE_OMIT_WAL - return pPager->jfd; -#else - return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; -#endif +SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ + return pPager->zJournal; } /* -** Return the full pathname of the journal file. +** Return true if fsync() calls are disabled for this pager. Return FALSE +** if fsync()s are executed normally. */ -SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ - return pPager->zJournal; +SQLITE_PRIVATE int sqlite3PagerNosync(Pager *pPager){ + return pPager->noSync; } #ifdef SQLITE_HAS_CODEC @@ -56980,8 +52001,7 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i /* In order to be able to rollback, an in-memory database must journal ** the page we are moving from. */ - assert( pPager->tempFile || !MEMDB ); - if( pPager->tempFile ){ + if( MEMDB ){ rc = sqlite3PagerWrite(pPg); if( rc ) return rc; } @@ -57004,8 +52024,9 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** one or more savepoint bitvecs. This is the reason this function ** may return SQLITE_NOMEM. */ - if( (pPg->flags & PGHDR_DIRTY)!=0 - && SQLITE_OK!=(rc = subjournalPageIfRequired(pPg)) + if( pPg->flags&PGHDR_DIRTY + && subjRequiresPage(pPg) + && SQLITE_OK!=(rc = subjournalPage(pPg)) ){ return rc; } @@ -57038,7 +52059,7 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); - if( pPager->tempFile ){ + if( MEMDB ){ /* Do not discard pages from an in-memory database since we might ** need to rollback later. Just move the page out of the way. */ sqlite3PcacheMove(pPgOld, pPager->dbSize+1); @@ -57055,7 +52076,8 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** to exist, in case the transaction needs to roll back. Use pPgOld ** as the original page since it has already been allocated. */ - if( pPager->tempFile && pPgOld ){ + if( MEMDB ){ + assert( pPgOld ); sqlite3PcacheMove(pPgOld, origPgno); sqlite3PagerUnrefNotNull(pPgOld); } @@ -57076,7 +52098,7 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** the journal file twice, but that is not a problem. */ PgHdr *pPgHdr; - rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr, 0); + rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); if( rc!=SQLITE_OK ){ if( needSyncPgno<=pPager->dbOrigSize ){ assert( pPager->pTmpSpace!=0 ); @@ -57307,12 +52329,10 @@ SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ ** Unless this is an in-memory or temporary database, clear the pager cache. */ SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *pPager){ - assert( MEMDB==0 || pPager->tempFile ); - if( pPager->tempFile==0 ) pager_reset(pPager); + if( !MEMDB && pPager->tempFile==0 ) pager_reset(pPager); } #endif - #ifndef SQLITE_OMIT_WAL /* ** This function is called when the user invokes "PRAGMA wal_checkpoint", @@ -57344,7 +52364,6 @@ SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager){ */ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ const sqlite3_io_methods *pMethods = pPager->fd->pMethods; - if( pPager->noLock ) return 0; return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); } @@ -57488,40 +52507,11 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; pagerFixMaplimit(pPager); - if( rc && !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); } } return rc; } -#ifdef SQLITE_ENABLE_SNAPSHOT -/* -** If this is a WAL database, obtain a snapshot handle for the snapshot -** currently open. Otherwise, return an error. -*/ -SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot){ - int rc = SQLITE_ERROR; - if( pPager->pWal ){ - rc = sqlite3WalSnapshotGet(pPager->pWal, ppSnapshot); - } - return rc; -} - -/* -** If this is a WAL database, store a pointer to pSnapshot. Next time a -** read transaction is opened, attempt to read from the snapshot it -** identifies. If this is not a WAL database, return an error. -*/ -SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot){ - int rc = SQLITE_OK; - if( pPager->pWal ){ - sqlite3WalSnapshotOpen(pPager->pWal, pSnapshot); - }else{ - rc = SQLITE_ERROR; - } - return rc; -} -#endif /* SQLITE_ENABLE_SNAPSHOT */ #endif /* !SQLITE_OMIT_WAL */ #ifdef SQLITE_ENABLE_ZIPVFS @@ -57538,15 +52528,16 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ } #endif + #endif /* SQLITE_OMIT_DISKIO */ /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC -SQLITE_API void sqlite3pager_get_codec(Pager *pPager, void **ctx) { +SQLITE_PRIVATE void sqlite3pager_get_codec(Pager *pPager, void **ctx) { *ctx = pPager->pCodec; } -SQLITE_API int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno) { +SQLITE_PRIVATE int sqlite3pager_is_mj_pgno(Pager *pPager, Pgno pgno) { return (PAGER_MJ_PGNO(pPager) == pgno) ? 1 : 0; } @@ -57554,7 +52545,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3Pager_get_fd(Pager *pPager) { return (isOpen(pPager->fd)) ? pPager->fd : NULL; } -SQLITE_API void sqlite3pager_sqlite3PagerSetCodec( +SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetCodec( Pager *pPager, void *(*xCodec)(void*,void*,Pgno,int), void (*xCodecSizeChng)(void*,int,int), @@ -57564,7 +52555,7 @@ SQLITE_API void sqlite3pager_sqlite3PagerSetCodec( sqlite3PagerSetCodec(pPager, xCodec, xCodecSizeChng, xCodecFree, pCodec); } -SQLITE_API void sqlite3pager_sqlite3PagerSetError( Pager *pPager, int error) { +SQLITE_PRIVATE void sqlite3pager_sqlite3PagerSetError( Pager *pPager, int error) { pPager->errCode = error; } @@ -57818,7 +52809,6 @@ SQLITE_API void sqlite3pager_sqlite3PagerSetError( Pager *pPager, int error) { */ #ifndef SQLITE_OMIT_WAL -/* #include "wal.h" */ /* ** Trace output macros @@ -57848,8 +52838,7 @@ SQLITE_PRIVATE int sqlite3WalTrace = 0; /* ** Indices of various locking bytes. WAL_NREADER is the number -** of available reader locks and should be at least 3. The default -** is SQLITE_SHM_NLOCK==8 and WAL_NREADER==5. +** of available reader locks and should be at least 3. */ #define WAL_WRITE_LOCK 0 #define WAL_ALL_BUT_WRITE 1 @@ -57869,10 +52858,7 @@ typedef struct WalCkptInfo WalCkptInfo; ** The following object holds a copy of the wal-index header content. ** ** The actual header in the wal-index consists of two copies of this -** object followed by one instance of the WalCkptInfo object. -** For all versions of SQLite through 3.10.0 and probably beyond, -** the locking bytes (WalCkptInfo.aLock) start at offset 120 and -** the total header size is 136 bytes. +** object. ** ** The szPage value can be any power of 2 between 512 and 32768, inclusive. ** Or it can be 1 to represent a 65536-byte page. The latter case was @@ -57905,16 +52891,6 @@ struct WalIndexHdr { ** However, a WAL_WRITE_LOCK thread can move the value of nBackfill from ** mxFrame back to zero when the WAL is reset. ** -** nBackfillAttempted is the largest value of nBackfill that a checkpoint -** has attempted to achieve. Normally nBackfill==nBackfillAtempted, however -** the nBackfillAttempted is set before any backfilling is done and the -** nBackfill is only set after all backfilling completes. So if a checkpoint -** crashes, nBackfillAttempted might be larger than nBackfill. The -** WalIndexHdr.mxFrame must never be less than nBackfillAttempted. -** -** The aLock[] field is a set of bytes used for locking. These bytes should -** never be read or written. -** ** There is one entry in aReadMark[] for each reader lock. If a reader ** holds read-lock K, then the value in aReadMark[K] is no greater than ** the mxFrame for that reader. The value READMARK_NOT_USED (0xffffffff) @@ -57954,9 +52930,6 @@ struct WalIndexHdr { struct WalCkptInfo { u32 nBackfill; /* Number of WAL frames backfilled into DB */ u32 aReadMark[WAL_NREADER]; /* Reader marks */ - u8 aLock[SQLITE_SHM_NLOCK]; /* Reserved space for locks */ - u32 nBackfillAttempted; /* WAL frames perhaps written, or maybe not */ - u32 notUsed0; /* Available for future enhancements */ }; #define READMARK_NOT_USED 0xffffffff @@ -57966,8 +52939,9 @@ struct WalCkptInfo { ** only support mandatory file-locks, we do not read or write data ** from the region of the file on which locks are applied. */ -#define WALINDEX_LOCK_OFFSET (sizeof(WalIndexHdr)*2+offsetof(WalCkptInfo,aLock)) -#define WALINDEX_HDR_SIZE (sizeof(WalIndexHdr)*2+sizeof(WalCkptInfo)) +#define WALINDEX_LOCK_OFFSET (sizeof(WalIndexHdr)*2 + sizeof(WalCkptInfo)) +#define WALINDEX_LOCK_RESERVED 16 +#define WALINDEX_HDR_SIZE (WALINDEX_LOCK_OFFSET+WALINDEX_LOCK_RESERVED) /* Size of header before each frame in wal */ #define WAL_FRAME_HDRSIZE 24 @@ -58020,16 +52994,11 @@ struct Wal { u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ WalIndexHdr hdr; /* Wal-index header for current transaction */ - u32 minFrame; /* Ignore wal frames before this one */ - u32 iReCksum; /* On commit, recalculate checksums from here */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif -#ifdef SQLITE_ENABLE_SNAPSHOT - WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ -#endif }; /* @@ -58122,7 +53091,7 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte); if( !apNew ){ *ppPage = 0; - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } memset((void*)&apNew[pWal->nWiData], 0, sizeof(u32*)*(iPage+1-pWal->nWiData)); @@ -58134,7 +53103,7 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ if( pWal->apWiData[iPage]==0 ){ if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); - if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM; }else{ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] @@ -58245,9 +53214,9 @@ static void walIndexWriteHdr(Wal *pWal){ pWal->hdr.isInit = 1; pWal->hdr.iVersion = WALINDEX_MAX_VERSION; walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); - memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); + memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr)); walShmBarrier(pWal); - memcpy((void*)&aHdr[0], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); + memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr)); } /* @@ -58275,18 +53244,14 @@ static void walEncodeFrame( assert( WAL_FRAME_HDRSIZE==24 ); sqlite3Put4byte(&aFrame[0], iPage); sqlite3Put4byte(&aFrame[4], nTruncate); - if( pWal->iReCksum==0 ){ - memcpy(&aFrame[8], pWal->hdr.aSalt, 8); + memcpy(&aFrame[8], pWal->hdr.aSalt, 8); - nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); - walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); - walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); + nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); + walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); - sqlite3Put4byte(&aFrame[16], aCksum[0]); - sqlite3Put4byte(&aFrame[20], aCksum[1]); - }else{ - memset(&aFrame[8], 0, 16); - } + sqlite3Put4byte(&aFrame[16], aCksum[0]); + sqlite3Put4byte(&aFrame[20], aCksum[1]); } /* @@ -58389,9 +53354,10 @@ static void walUnlockShared(Wal *pWal, int lockIdx){ SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED); WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx))); } -static int walLockExclusive(Wal *pWal, int lockIdx, int n){ +static int walLockExclusive(Wal *pWal, int lockIdx, int n, int fBlock){ int rc; if( pWal->exclusiveMode ) return SQLITE_OK; + if( fBlock ) sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_WAL_BLOCK, 0); rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, @@ -58552,13 +53518,13 @@ static void walCleanupHash(Wal *pWal){ ** via the hash table even after the cleanup. */ if( iLimit ){ - int j; /* Loop counter */ + int i; /* Loop counter */ int iKey; /* Hash key */ - for(j=1; j<=iLimit; j++){ - for(iKey=walHash(aPgno[j]); aHash[iKey]; iKey=walNextHash(iKey)){ - if( aHash[iKey]==j ) break; + for(i=1; i<=iLimit; i++){ + for(iKey=walHash(aPgno[i]); aHash[iKey]; iKey=walNextHash(iKey)){ + if( aHash[iKey]==i ) break; } - assert( aHash[iKey]==j ); + assert( aHash[iKey]==i ); } } #endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ @@ -58677,7 +53643,7 @@ static int walIndexRecover(Wal *pWal){ assert( pWal->writeLock ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; nLock = SQLITE_SHM_NLOCK - iLock; - rc = walLockExclusive(pWal, iLock, nLock); + rc = walLockExclusive(pWal, iLock, nLock, 0); if( rc ){ return rc; } @@ -58749,7 +53715,7 @@ static int walIndexRecover(Wal *pWal){ szFrame = szPage + WAL_FRAME_HDRSIZE; aFrame = (u8 *)sqlite3_malloc64(szFrame); if( !aFrame ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; goto recovery_error; } aData = &aFrame[WAL_FRAME_HDRSIZE]; @@ -58798,7 +53764,6 @@ static int walIndexRecover(Wal *pWal){ */ pInfo = walCkptInfo(pWal); pInfo->nBackfill = 0; - pInfo->nBackfillAttempted = pWal->hdr.mxFrame; pInfo->aReadMark[0] = 0; for(i=1; iaReadMark[i] = READMARK_NOT_USED; if( pWal->hdr.mxFrame ) pInfo->aReadMark[1] = pWal->hdr.mxFrame; @@ -58870,11 +53835,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( /* In the amalgamation, the os_unix.c and os_win.c source files come before ** this source file. Verify that the #defines of the locking byte offsets ** in os_unix.c and os_win.c agree with the WALINDEX_LOCK_OFFSET value. - ** For that matter, if the lock offset ever changes from its initial design - ** value of 120, we need to know that so there is an assert() to check it. */ - assert( 120==WALINDEX_LOCK_OFFSET ); - assert( 136==WALINDEX_HDR_SIZE ); #ifdef WIN_SHM_BASE assert( WIN_SHM_BASE==WALINDEX_LOCK_OFFSET ); #endif @@ -58887,7 +53848,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( *ppWal = 0; pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile); if( !pRet ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } pRet->pVfs = pVfs; @@ -59065,7 +54026,7 @@ static void walMergesort( int nMerge = 0; /* Number of elements in list aMerge */ ht_slot *aMerge = 0; /* List to be merged */ int iList; /* Index into input list */ - u32 iSub = 0; /* Index into aSub array */ + int iSub = 0; /* Index into aSub array */ struct Sublist aSub[13]; /* Array of sub-lists */ memset(aSub, 0, sizeof(aSub)); @@ -59076,9 +54037,7 @@ static void walMergesort( nMerge = 1; aMerge = &aList[iList]; for(iSub=0; iList & (1<aList && p->nList<=(1<aList==&aList[iList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); @@ -59089,9 +54048,7 @@ static void walMergesort( for(iSub++; iSubnList<=(1<aList==&aList[nList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); @@ -59151,7 +54108,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ + iLast*sizeof(ht_slot); p = (WalIterator *)sqlite3_malloc64(nByte); if( !p ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } memset(p, 0, nByte); p->nSegment = nSegment; @@ -59163,7 +54120,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) ); if( !aTmp ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; } for(i=0; rc==SQLITE_OK && ihdr.aSalt[1], &salt1, 4); walIndexWriteHdr(pWal); pInfo->nBackfill = 0; - pInfo->nBackfillAttempted = 0; pInfo->aReadMark[1] = 0; for(i=2; iaReadMark[i] = READMARK_NOT_USED; assert( pInfo->aReadMark[0]==0 ); @@ -59370,8 +54326,6 @@ static int walCheckpoint( i64 nSize; /* Current size of database file */ u32 nBackfill = pInfo->nBackfill; - pInfo->nBackfillAttempted = mxSafeFrame; - /* Sync the WAL to disk */ if( sync_flags ){ rc = sqlite3OsSync(pWal->pWalFd, sync_flags); @@ -59664,7 +54618,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; } - }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ + }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1, 1)) ){ pWal->writeLock = 1; if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ badHdr = walIndexTryHdr(pWal, pChanged); @@ -59755,7 +54709,6 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ int mxI; /* Index of largest aReadMark[] value */ int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ - u32 mxFrame; /* Wal frame to lock to */ assert( pWal->readLock<0 ); /* Not currently locked */ @@ -59819,12 +54772,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } pInfo = walCkptInfo(pWal); - if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame -#ifdef SQLITE_ENABLE_SNAPSHOT - && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 - || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) -#endif - ){ + if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame ){ /* The WAL has been completely backfilled (or it is empty). ** and can be safely ignored. */ @@ -59862,88 +54810,70 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ */ mxReadMark = 0; mxI = 0; - mxFrame = pWal->hdr.mxFrame; -#ifdef SQLITE_ENABLE_SNAPSHOT - if( pWal->pSnapshot && pWal->pSnapshot->mxFramepSnapshot->mxFrame; - } -#endif for(i=1; iaReadMark[i]; - if( mxReadMark<=thisMark && thisMark<=mxFrame ){ + if( mxReadMark<=thisMark && thisMark<=pWal->hdr.mxFrame ){ assert( thisMark!=READMARK_NOT_USED ); mxReadMark = thisMark; mxI = i; } } - if( (pWal->readOnly & WAL_SHM_RDONLY)==0 - && (mxReadMarkaReadMark[i] = mxFrame; - mxI = i; - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - break; - }else if( rc!=SQLITE_BUSY ){ - return rc; + /* There was once an "if" here. The extra "{" is to preserve indentation. */ + { + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMarkhdr.mxFrame || mxI==0) + ){ + for(i=1; iaReadMark[i] = pWal->hdr.mxFrame; + mxI = i; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + break; + }else if( rc!=SQLITE_BUSY ){ + return rc; + } } } - } - if( mxI==0 ){ - assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); - return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; - } + if( mxI==0 ){ + assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; + } - rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); - if( rc ){ - return rc==SQLITE_BUSY ? WAL_RETRY : rc; - } - /* Now that the read-lock has been obtained, check that neither the - ** value in the aReadMark[] array or the contents of the wal-index - ** header have changed. - ** - ** It is necessary to check that the wal-index header did not change - ** between the time it was read and when the shared-lock was obtained - ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility - ** that the log file may have been wrapped by a writer, or that frames - ** that occur later in the log than pWal->hdr.mxFrame may have been - ** copied into the database by a checkpointer. If either of these things - ** happened, then reading the database with the current value of - ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry - ** instead. - ** - ** Before checking that the live wal-index header has not changed - ** since it was read, set Wal.minFrame to the first frame in the wal - ** file that has not yet been checkpointed. This client will not need - ** to read any frames earlier than minFrame from the wal file - they - ** can be safely read directly from the database file. - ** - ** Because a ShmBarrier() call is made between taking the copy of - ** nBackfill and checking that the wal-header in shared-memory still - ** matches the one cached in pWal->hdr, it is guaranteed that the - ** checkpointer that set nBackfill was not working with a wal-index - ** header newer than that cached in pWal->hdr. If it were, that could - ** cause a problem. The checkpointer could omit to checkpoint - ** a version of page X that lies before pWal->minFrame (call that version - ** A) on the basis that there is a newer version (version B) of the same - ** page later in the wal file. But if version B happens to like past - ** frame pWal->hdr.mxFrame - then the client would incorrectly assume - ** that it can read version A from the database file. However, since - ** we can guarantee that the checkpointer that set nBackfill could not - ** see any pages past pWal->hdr.mxFrame, this problem does not come up. - */ - pWal->minFrame = pInfo->nBackfill+1; - walShmBarrier(pWal); - if( pInfo->aReadMark[mxI]!=mxReadMark - || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) - ){ - walUnlockShared(pWal, WAL_READ_LOCK(mxI)); - return WAL_RETRY; - }else{ - assert( mxReadMark<=pWal->hdr.mxFrame ); - pWal->readLock = (i16)mxI; + rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); + if( rc ){ + return rc==SQLITE_BUSY ? WAL_RETRY : rc; + } + /* Now that the read-lock has been obtained, check that neither the + ** value in the aReadMark[] array or the contents of the wal-index + ** header have changed. + ** + ** It is necessary to check that the wal-index header did not change + ** between the time it was read and when the shared-lock was obtained + ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility + ** that the log file may have been wrapped by a writer, or that frames + ** that occur later in the log than pWal->hdr.mxFrame may have been + ** copied into the database by a checkpointer. If either of these things + ** happened, then reading the database with the current value of + ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry + ** instead. + ** + ** This does not guarantee that the copy of the wal-index header is up to + ** date before proceeding. That would not be possible without somehow + ** blocking writers. It only guarantees that a dangerous checkpoint or + ** log-wrap (either of which would require an exclusive lock on + ** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid. + */ + walShmBarrier(pWal); + if( pInfo->aReadMark[mxI]!=mxReadMark + || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) + ){ + walUnlockShared(pWal, WAL_READ_LOCK(mxI)); + return WAL_RETRY; + }else{ + assert( mxReadMark<=pWal->hdr.mxFrame ); + pWal->readLock = (i16)mxI; + } } return rc; } @@ -59966,14 +54896,6 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ int rc; /* Return code */ int cnt = 0; /* Number of TryBeginRead attempts */ -#ifdef SQLITE_ENABLE_SNAPSHOT - int bChanged = 0; - WalIndexHdr *pSnapshot = pWal->pSnapshot; - if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ - bChanged = 1; - } -#endif - do{ rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); }while( rc==WAL_RETRY ); @@ -59981,66 +54903,6 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ testcase( (rc&0xff)==SQLITE_IOERR ); testcase( rc==SQLITE_PROTOCOL ); testcase( rc==SQLITE_OK ); - -#ifdef SQLITE_ENABLE_SNAPSHOT - if( rc==SQLITE_OK ){ - if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ - /* At this point the client has a lock on an aReadMark[] slot holding - ** a value equal to or smaller than pSnapshot->mxFrame, but pWal->hdr - ** is populated with the wal-index header corresponding to the head - ** of the wal file. Verify that pSnapshot is still valid before - ** continuing. Reasons why pSnapshot might no longer be valid: - ** - ** (1) The WAL file has been reset since the snapshot was taken. - ** In this case, the salt will have changed. - ** - ** (2) A checkpoint as been attempted that wrote frames past - ** pSnapshot->mxFrame into the database file. Note that the - ** checkpoint need not have completed for this to cause problems. - */ - volatile WalCkptInfo *pInfo = walCkptInfo(pWal); - - assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 ); - assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame ); - - /* It is possible that there is a checkpointer thread running - ** concurrent with this code. If this is the case, it may be that the - ** checkpointer has already determined that it will checkpoint - ** snapshot X, where X is later in the wal file than pSnapshot, but - ** has not yet set the pInfo->nBackfillAttempted variable to indicate - ** its intent. To avoid the race condition this leads to, ensure that - ** there is no checkpointer process by taking a shared CKPT lock - ** before checking pInfo->nBackfillAttempted. */ - rc = walLockShared(pWal, WAL_CKPT_LOCK); - - if( rc==SQLITE_OK ){ - /* Check that the wal file has not been wrapped. Assuming that it has - ** not, also check that no checkpointer has attempted to checkpoint any - ** frames beyond pSnapshot->mxFrame. If either of these conditions are - ** true, return SQLITE_BUSY_SNAPSHOT. Otherwise, overwrite pWal->hdr - ** with *pSnapshot and set *pChanged as appropriate for opening the - ** snapshot. */ - if( !memcmp(pSnapshot->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) - && pSnapshot->mxFrame>=pInfo->nBackfillAttempted - ){ - assert( pWal->readLock>0 ); - memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr)); - *pChanged = bChanged; - }else{ - rc = SQLITE_BUSY_SNAPSHOT; - } - - /* Release the shared CKPT lock obtained above. */ - walUnlockShared(pWal, WAL_CKPT_LOCK); - } - - - if( rc!=SQLITE_OK ){ - sqlite3WalEndReadTransaction(pWal); - } - } - } -#endif return rc; } @@ -60072,7 +54934,6 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ - int iMinHash; /* This routine is only be called from within a read transaction. */ assert( pWal->readLock>=0 || pWal->lockError ); @@ -60113,8 +54974,7 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( ** This condition filters out entries that were added to the hash ** table after the current read-transaction had started. */ - iMinHash = walFramePage(pWal->minFrame); - for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){ + for(iHash=walFramePage(iLast); iHash>=0 && iRead==0; iHash--){ volatile ht_slot *aHash; /* Pointer to hash table */ volatile u32 *aPgno; /* Pointer to array of page numbers */ u32 iZero; /* Frame number corresponding to aPgno[0] */ @@ -60129,7 +54989,7 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( nCollide = HASHTABLE_NSLOT; for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ + if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } @@ -60146,8 +55006,7 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( { u32 iRead2 = 0; u32 iTest; - assert( pWal->minFrame>0 ); - for(iTest=iLast; iTest>=pWal->minFrame; iTest--){ + for(iTest=iLast; iTest>0; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; break; @@ -60213,7 +55072,6 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ /* Cannot start a write transaction without first holding a read ** transaction. */ assert( pWal->readLock>=0 ); - assert( pWal->writeLock==0 && pWal->iReCksum==0 ); if( pWal->readOnly ){ return SQLITE_READONLY; @@ -60222,7 +55080,7 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ /* Only one writer allowed at a time. Get the write lock. Return ** SQLITE_BUSY if unable. */ - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1, 0); if( rc ){ return rc; } @@ -60249,7 +55107,6 @@ SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal){ if( pWal->writeLock ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; - pWal->iReCksum = 0; pWal->truncateOnCommit = 0; } return SQLITE_OK; @@ -60368,7 +55225,7 @@ static int walRestartLog(Wal *pWal){ if( pInfo->nBackfill>0 ){ u32 salt1; sqlite3_randomness(4, &salt1); - rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1, 0); if( rc==SQLITE_OK ){ /* If all readers are using WAL_READ_LOCK(0) (in other words if no ** readers are currently using the WAL), then the transactions @@ -60456,7 +55313,7 @@ static int walWriteOneFrame( void *pData; /* Data actually written */ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ #if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM_BKPT; + if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM; #else pData = pPage->pData; #endif @@ -60468,59 +55325,6 @@ static int walWriteOneFrame( return rc; } -/* -** This function is called as part of committing a transaction within which -** one or more frames have been overwritten. It updates the checksums for -** all frames written to the wal file by the current transaction starting -** with the earliest to have been overwritten. -** -** SQLITE_OK is returned if successful, or an SQLite error code otherwise. -*/ -static int walRewriteChecksums(Wal *pWal, u32 iLast){ - const int szPage = pWal->szPage;/* Database page size */ - int rc = SQLITE_OK; /* Return code */ - u8 *aBuf; /* Buffer to load data from wal file into */ - u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-headers in */ - u32 iRead; /* Next frame to read from wal file */ - i64 iCksumOff; - - aBuf = sqlite3_malloc(szPage + WAL_FRAME_HDRSIZE); - if( aBuf==0 ) return SQLITE_NOMEM_BKPT; - - /* Find the checksum values to use as input for the recalculating the - ** first checksum. If the first frame is frame 1 (implying that the current - ** transaction restarted the wal file), these values must be read from the - ** wal-file header. Otherwise, read them from the frame header of the - ** previous frame. */ - assert( pWal->iReCksum>0 ); - if( pWal->iReCksum==1 ){ - iCksumOff = 24; - }else{ - iCksumOff = walFrameOffset(pWal->iReCksum-1, szPage) + 16; - } - rc = sqlite3OsRead(pWal->pWalFd, aBuf, sizeof(u32)*2, iCksumOff); - pWal->hdr.aFrameCksum[0] = sqlite3Get4byte(aBuf); - pWal->hdr.aFrameCksum[1] = sqlite3Get4byte(&aBuf[sizeof(u32)]); - - iRead = pWal->iReCksum; - pWal->iReCksum = 0; - for(; rc==SQLITE_OK && iRead<=iLast; iRead++){ - i64 iOff = walFrameOffset(iRead, szPage); - rc = sqlite3OsRead(pWal->pWalFd, aBuf, szPage+WAL_FRAME_HDRSIZE, iOff); - if( rc==SQLITE_OK ){ - u32 iPgno, nDbSize; - iPgno = sqlite3Get4byte(aBuf); - nDbSize = sqlite3Get4byte(&aBuf[4]); - - walEncodeFrame(pWal, iPgno, nDbSize, &aBuf[WAL_FRAME_HDRSIZE], aFrame); - rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOff); - } - } - - sqlite3_free(aBuf); - return rc; -} - /* ** Write a set of frames to the log. The caller must hold the write-lock ** on the log file (obtained using sqlite3WalBeginWriteTransaction()). @@ -60541,8 +55345,6 @@ SQLITE_PRIVATE int sqlite3WalFrames( int szFrame; /* The size of a single frame */ i64 iOffset; /* Next byte to write in WAL file */ WalWriter w; /* The writer */ - u32 iFirst = 0; /* First frame that may be overwritten */ - WalIndexHdr *pLive; /* Pointer to shared header */ assert( pList ); assert( pWal->writeLock ); @@ -60558,11 +55360,6 @@ SQLITE_PRIVATE int sqlite3WalFrames( } #endif - pLive = (WalIndexHdr*)walIndexHdr(pWal); - if( memcmp(&pWal->hdr, (void *)pLive, sizeof(WalIndexHdr))!=0 ){ - iFirst = pLive->mxFrame+1; - } - /* See if it is possible to write these frames into the start of the ** log file, instead of appending to it at pWal->hdr.mxFrame. */ @@ -60627,33 +55424,6 @@ SQLITE_PRIVATE int sqlite3WalFrames( /* Write all frames into the log file exactly once */ for(p=pList; p; p=p->pDirty){ int nDbSize; /* 0 normally. Positive == commit flag */ - - /* Check if this page has already been written into the wal file by - ** the current transaction. If so, overwrite the existing frame and - ** set Wal.writeLock to WAL_WRITELOCK_RECKSUM - indicating that - ** checksums must be recomputed when the transaction is committed. */ - if( iFirst && (p->pDirty || isCommit==0) ){ - u32 iWrite = 0; - VVA_ONLY(rc =) sqlite3WalFindFrame(pWal, p->pgno, &iWrite); - assert( rc==SQLITE_OK || iWrite==0 ); - if( iWrite>=iFirst ){ - i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; - void *pData; - if( pWal->iReCksum==0 || iWriteiReCksum ){ - pWal->iReCksum = iWrite; - } -#if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; -#else - pData = p->pData; -#endif - rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOff); - if( rc ) return rc; - p->flags &= ~PGHDR_WAL_APPEND; - continue; - } - } - iFrame++; assert( iOffset==walFrameOffset(iFrame, szPage) ); nDbSize = (isCommit && p->pDirty==0) ? nTruncate : 0; @@ -60661,13 +55431,6 @@ SQLITE_PRIVATE int sqlite3WalFrames( if( rc ) return rc; pLast = p; iOffset += szFrame; - p->flags |= PGHDR_WAL_APPEND; - } - - /* Recalculate checksums within the wal file if required. */ - if( isCommit && pWal->iReCksum ){ - rc = walRewriteChecksums(pWal, iFrame); - if( rc ) return rc; } /* If this is the end of a transaction, then we might need to pad @@ -60685,21 +55448,16 @@ SQLITE_PRIVATE int sqlite3WalFrames( ** past the sector boundary is written after the sync. */ if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){ - int bSync = 1; if( pWal->padToSectorBoundary ){ int sectorSize = sqlite3SectorSize(pWal->pWalFd); w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; - bSync = (w.iSyncPoint==iOffset); - testcase( bSync ); while( iOffsethdr.mxFrame; for(p=pList; p && rc==SQLITE_OK; p=p->pDirty){ - if( (p->flags & PGHDR_WAL_APPEND)==0 ) continue; iFrame++; rc = walIndexAppend(pWal, iFrame, p->pgno); } @@ -60793,7 +55550,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive ** "checkpoint" lock on the database file. */ - rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1, 0); if( rc ){ /* EVIDENCE-OF: R-10421-19736 If any other process is running a ** checkpoint operation at the same time, the lock cannot be obtained and @@ -60837,7 +55594,6 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ - if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; }else{ @@ -60953,52 +55709,6 @@ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } -#ifdef SQLITE_ENABLE_SNAPSHOT -/* Create a snapshot object. The content of a snapshot is opaque to -** every other subsystem, so the WAL module can put whatever it needs -** in the object. -*/ -SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot){ - int rc = SQLITE_OK; - WalIndexHdr *pRet; - - assert( pWal->readLock>=0 && pWal->writeLock==0 ); - - pRet = (WalIndexHdr*)sqlite3_malloc(sizeof(WalIndexHdr)); - if( pRet==0 ){ - rc = SQLITE_NOMEM_BKPT; - }else{ - memcpy(pRet, &pWal->hdr, sizeof(WalIndexHdr)); - *ppSnapshot = (sqlite3_snapshot*)pRet; - } - - return rc; -} - -/* Try to open on pSnapshot when the next read-transaction starts -*/ -SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot){ - pWal->pSnapshot = (WalIndexHdr*)pSnapshot; -} - -/* -** Return a +ve value if snapshot p1 is newer than p2. A -ve value if -** p1 is older than p2 and zero if p1 and p2 are the same snapshot. -*/ -SQLITE_API int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){ - WalIndexHdr *pHdr1 = (WalIndexHdr*)p1; - WalIndexHdr *pHdr2 = (WalIndexHdr*)p2; - - /* aSalt[0] is a copy of the value stored in the wal file header. It - ** is incremented each time the wal file is restarted. */ - if( pHdr1->aSalt[0]aSalt[0] ) return -1; - if( pHdr1->aSalt[0]>pHdr2->aSalt[0] ) return +1; - if( pHdr1->mxFramemxFrame ) return -1; - if( pHdr1->mxFrame>pHdr2->mxFrame ) return +1; - return 0; -} -#endif /* SQLITE_ENABLE_SNAPSHOT */ - #ifdef SQLITE_ENABLE_ZIPVFS /* ** If the argument is not NULL, it points to a Wal object that holds a @@ -61011,12 +55721,6 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal){ } #endif -/* Return the sqlite3_file object for the WAL file -*/ -SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ - return pWal->pWalFd; -} - #endif /* #ifndef SQLITE_OMIT_WAL */ /************** End of wal.c *************************************************/ @@ -61038,7 +55742,6 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ ** big and we want to break it down some. This packaged seemed like ** a good breakout. */ -/* #include "btreeInt.h" */ #ifndef SQLITE_OMIT_SHARED_CACHE #if SQLITE_THREADSAFE @@ -61192,6 +55895,21 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree *p){ #endif +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Enter and leave a mutex on a Btree given a cursor owned by that +** Btree. These entry points are used by incremental I/O and can be +** omitted if that module is not used. +*/ +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor *pCur){ + sqlite3BtreeEnter(pCur->pBtree); +} +SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor *pCur){ + sqlite3BtreeLeave(pCur->pBtree); +} +#endif /* SQLITE_OMIT_INCRBLOB */ + + /* ** Enter the mutex on every Btree associated with a database ** connection. This is needed (for example) prior to parsing @@ -61225,6 +55943,14 @@ SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3 *db){ } } +/* +** Return true if a particular Btree requires a lock. Return FALSE if +** no lock is ever required since it is not sharable. +*/ +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree *p){ + return p->sharable; +} + #ifndef NDEBUG /* ** Return true if the current thread holds the database connection @@ -61298,25 +56024,6 @@ SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ } } #endif /* if SQLITE_THREADSAFE */ - -#ifndef SQLITE_OMIT_INCRBLOB -/* -** Enter a mutex on a Btree given a cursor owned by that Btree. -** -** These entry points are used by incremental I/O only. Enter() is required -** any time OMIT_SHARED_CACHE is not defined, regardless of whether or not -** the build is threadsafe. Leave() is only required by threadsafe builds. -*/ -SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor *pCur){ - sqlite3BtreeEnter(pCur->pBtree); -} -# if SQLITE_THREADSAFE -SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor *pCur){ - sqlite3BtreeLeave(pCur->pBtree); -} -# endif -#endif /* ifndef SQLITE_OMIT_INCRBLOB */ - #endif /* ifndef SQLITE_OMIT_SHARED_CACHE */ /************** End of btmutex.c *********************************************/ @@ -61336,7 +56043,6 @@ SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor *pCur){ ** See the header comment on "btreeInt.h" for additional information. ** Including a description of file format and an overview of operation. */ -/* #include "btreeInt.h" */ /* ** The header string that appears at the beginning of every @@ -61409,7 +56115,7 @@ static BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; ** The shared cache setting effects only future calls to ** sqlite3_open(), sqlite3_open16(), or sqlite3_open_v2(). */ -SQLITE_API int sqlite3_enable_shared_cache(int enable){ +SQLITE_API int SQLITE_STDCALL sqlite3_enable_shared_cache(int enable){ sqlite3GlobalConfig.sharedCacheEnabled = enable; return SQLITE_OK; } @@ -61673,7 +56379,7 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ if( !pLock ){ pLock = (BtLock *)sqlite3MallocZero(sizeof(BtLock)); if( !pLock ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } pLock->iTable = iTable; pLock->pBtree = p; @@ -61773,19 +56479,6 @@ static void releasePage(MemPage *pPage); /* Forward reference */ static int cursorHoldsMutex(BtCursor *p){ return sqlite3_mutex_held(p->pBt->mutex); } - -/* Verify that the cursor and the BtShared agree about what is the current -** database connetion. This is important in shared-cache mode. If the database -** connection pointers get out-of-sync, it is possible for routines like -** btreeInitPage() to reference an stale connection pointer that references a -** a connection that has already closed. This routine is used inside assert() -** statements only and for the purpose of double-checking that the btree code -** does keep the database connection pointers up-to-date. -*/ -static int cursorOwnsBtShared(BtCursor *p){ - assert( cursorHoldsMutex(p) ); - return (p->pBtree->db==p->pBt->db); -} #endif /* @@ -61826,15 +56519,13 @@ static void invalidateIncrblobCursors( int isClearTable /* True if all rows are being deleted */ ){ BtCursor *p; - if( pBtree->hasIncrblobCur==0 ) return; + BtShared *pBt = pBtree->pBt; assert( sqlite3BtreeHoldsMutex(pBtree) ); - pBtree->hasIncrblobCur = 0; - for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - if( (p->curFlags & BTCF_Incrblob)!=0 ){ - pBtree->hasIncrblobCur = 1; - if( isClearTable || p->info.nKey==iRow ){ - p->eState = CURSOR_INVALID; - } + for(p=pBt->pCursor; p; p=p->pNext){ + if( (p->curFlags & BTCF_Incrblob)!=0 + && (isClearTable || p->info.nKey==iRow) + ){ + p->eState = CURSOR_INVALID; } } } @@ -61885,7 +56576,7 @@ static int btreeSetHasContent(BtShared *pBt, Pgno pgno){ assert( pgno<=pBt->nPage ); pBt->pHasContent = sqlite3BitvecCreate(pBt->nPage); if( !pBt->pHasContent ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; } } if( rc==SQLITE_OK && pgno<=sqlite3BitvecSize(pBt->pHasContent) ){ @@ -61927,47 +56618,6 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){ pCur->iPage = -1; } -/* -** The cursor passed as the only argument must point to a valid entry -** when this function is called (i.e. have eState==CURSOR_VALID). This -** function saves the current cursor key in variables pCur->nKey and -** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error -** code otherwise. -** -** If the cursor is open on an intkey table, then the integer key -** (the rowid) is stored in pCur->nKey and pCur->pKey is left set to -** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is -** set to point to a malloced buffer pCur->nKey bytes in size containing -** the key. -*/ -static int saveCursorKey(BtCursor *pCur){ - int rc = SQLITE_OK; - assert( CURSOR_VALID==pCur->eState ); - assert( 0==pCur->pKey ); - assert( cursorHoldsMutex(pCur) ); - - if( pCur->curIntKey ){ - /* Only the rowid is required for a table btree */ - pCur->nKey = sqlite3BtreeIntegerKey(pCur); - }else{ - /* For an index btree, save the complete key content */ - void *pKey; - pCur->nKey = sqlite3BtreePayloadSize(pCur); - pKey = sqlite3Malloc( pCur->nKey ); - if( pKey ){ - rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); - if( rc==SQLITE_OK ){ - pCur->pKey = pKey; - }else{ - sqlite3_free(pKey); - } - }else{ - rc = SQLITE_NOMEM_BKPT; - } - } - assert( !pCur->curIntKey || !pCur->pKey ); - return rc; -} /* ** Save the current cursor position in the variables BtCursor.nKey @@ -61988,14 +56638,36 @@ static int saveCursorPosition(BtCursor *pCur){ }else{ pCur->skipNext = 0; } + rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); + assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ + + /* If this is an intKey table, then the above call to BtreeKeySize() + ** stores the integer key in pCur->nKey. In this case this value is + ** all that is required. Otherwise, if pCur is not open on an intKey + ** table, then malloc space for and store the pCur->nKey bytes of key + ** data. + */ + if( 0==pCur->apPage[0]->intKey ){ + void *pKey = sqlite3Malloc( pCur->nKey ); + if( pKey ){ + rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); + if( rc==SQLITE_OK ){ + pCur->pKey = pKey; + }else{ + sqlite3_free(pKey); + } + }else{ + rc = SQLITE_NOMEM; + } + } + assert( !pCur->apPage[0]->intKey || !pCur->pKey ); - rc = saveCursorKey(pCur); if( rc==SQLITE_OK ){ btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } - pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl|BTCF_AtLast); + invalidateOverflowCache(pCur); return rc; } @@ -62010,15 +56682,6 @@ static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); ** routine is called just before cursor pExcept is used to modify the ** table, for example in BtreeDelete() or BtreeInsert(). ** -** If there are two or more cursors on the same btree, then all such -** cursors should have their BTCF_Multiple flag set. The btreeCursor() -** routine enforces that rule. This routine only needs to be called in -** the uncommon case when pExpect has the BTCF_Multiple flag set. -** -** If pExpect!=NULL and if no other cursors are found on the same root-page, -** then the BTCF_Multiple flag on pExpect is cleared, to avoid another -** pointless call to this routine. -** ** Implementation note: This routine merely checks to see if any cursors ** need to be saved. It calls out to saveCursorsOnList() in the (unusual) ** event that cursors are in need to being saved. @@ -62030,9 +56693,7 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ for(p=pBt->pCursor; p; p=p->pNext){ if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ) break; } - if( p ) return saveCursorsOnList(p, iRoot, pExcept); - if( pExcept ) pExcept->curFlags &= ~BTCF_Multiple; - return SQLITE_OK; + return p ? saveCursorsOnList(p, iRoot, pExcept) : SQLITE_OK; } /* This helper routine to saveAllCursors does the actual work of saving @@ -62086,7 +56747,7 @@ static int btreeMoveto( ){ int rc; /* Status code */ UnpackedRecord *pIdxKey; /* Unpacked index key */ - char aSpace[384]; /* Temp space for pIdxKey - to avoid a malloc */ + char aSpace[200]; /* Temp space for pIdxKey - to avoid a malloc */ char *pFree = 0; if( pKey ){ @@ -62094,7 +56755,7 @@ static int btreeMoveto( pIdxKey = sqlite3VdbeAllocUnpackedRecord( pCur->pKeyInfo, aSpace, sizeof(aSpace), &pFree ); - if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; + if( pIdxKey==0 ) return SQLITE_NOMEM; sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey); if( pIdxKey->nField==0 ){ sqlite3DbFree(pCur->pKeyInfo->db, pFree); @@ -62120,7 +56781,7 @@ static int btreeMoveto( static int btreeRestoreCursorPosition(BtCursor *pCur){ int rc; int skipNext; - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState>=CURSOR_REQUIRESEEK ); if( pCur->eState==CURSOR_FAULT ){ return pCur->skipNext; @@ -62192,26 +56853,6 @@ SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow) return SQLITE_OK; } -#ifdef SQLITE_ENABLE_CURSOR_HINTS -/* -** Provide hints to the cursor. The particular hint given (and the type -** and number of the varargs parameters) is determined by the eHintType -** parameter. See the definitions of the BTREE_HINT_* macros for details. -*/ -SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ - /* Used only by system that substitute their own storage engine */ -} -#endif - -/* -** Provide flag hints to the cursor. -*/ -SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor *pCur, unsigned x){ - assert( x==BTREE_SEEK_EQ || x==BTREE_BULKLOAD || x==0 ); - pCur->hints = x; -} - - #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Given a page number of a regular database page, return the page @@ -62265,7 +56906,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ return; } iPtrmap = PTRMAP_PAGENO(pBt, key); - rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage, 0); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); if( rc!=SQLITE_OK ){ *pRC = rc; return; @@ -62308,7 +56949,7 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ assert( sqlite3_mutex_held(pBt->mutex) ); iPtrmap = PTRMAP_PAGENO(pBt, key); - rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage, 0); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); if( rc!=0 ){ return rc; } @@ -62340,85 +56981,39 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ ** the page, 1 means the second cell, and so forth) return a pointer ** to the cell content. ** -** findCellPastPtr() does the same except it skips past the initial -** 4-byte child pointer found on interior pages, if there is one. -** ** This routine works only for pages that do not contain overflow cells. */ #define findCell(P,I) \ - ((P)->aData + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) -#define findCellPastPtr(P,I) \ - ((P)->aDataOfst + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) + ((P)->aData + ((P)->maskPage & get2byte(&(P)->aCellIdx[2*(I)]))) +#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) /* -** This is common tail processing for btreeParseCellPtr() and -** btreeParseCellPtrIndex() for the case when the cell does not fit entirely -** on a single B-tree page. Make necessary adjustments to the CellInfo -** structure. +** This a more complex version of findCell() that works for +** pages that do contain overflow cells. */ -static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( - MemPage *pPage, /* Page containing the cell */ - u8 *pCell, /* Pointer to the cell text. */ - CellInfo *pInfo /* Fill in this structure */ -){ - /* If the payload will not fit completely on the local page, we have - ** to decide how much to store locally and how much to spill onto - ** overflow pages. The strategy is to minimize the amount of unused - ** space on overflow pages while keeping the amount of local storage - ** in between minLocal and maxLocal. - ** - ** Warning: changing the way overflow payload is distributed in any - ** way will result in an incompatible file format. - */ - int minLocal; /* Minimum amount of payload held locally */ - int maxLocal; /* Maximum amount of payload held locally */ - int surplus; /* Overflow payload available for local storage */ - - minLocal = pPage->minLocal; - maxLocal = pPage->maxLocal; - surplus = minLocal + (pInfo->nPayload - minLocal)%(pPage->pBt->usableSize-4); - testcase( surplus==maxLocal ); - testcase( surplus==maxLocal+1 ); - if( surplus <= maxLocal ){ - pInfo->nLocal = (u16)surplus; - }else{ - pInfo->nLocal = (u16)minLocal; +static u8 *findOverflowCell(MemPage *pPage, int iCell){ + int i; + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + for(i=pPage->nOverflow-1; i>=0; i--){ + int k; + k = pPage->aiOvfl[i]; + if( k<=iCell ){ + if( k==iCell ){ + return pPage->apOvfl[i]; + } + iCell--; + } } - pInfo->nSize = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell) + 4; + return findCell(pPage, iCell); } /* -** The following routines are implementations of the MemPage.xParseCell() -** method. -** -** Parse a cell content block and fill in the CellInfo structure. -** -** btreeParseCellPtr() => table btree leaf nodes -** btreeParseCellNoPayload() => table btree internal nodes -** btreeParseCellPtrIndex() => index btree nodes -** -** There is also a wrapper function btreeParseCell() that works for -** all MemPage types and that references the cell by index rather than -** by pointer. +** Parse a cell content block and fill in the CellInfo structure. There +** are two versions of this function. btreeParseCell() takes a +** cell index as the second argument and btreeParseCellPtr() +** takes a pointer to the body of the cell as its second argument. */ -static void btreeParseCellPtrNoPayload( - MemPage *pPage, /* Page containing the cell */ - u8 *pCell, /* Pointer to the cell text. */ - CellInfo *pInfo /* Fill in this structure */ -){ - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( pPage->leaf==0 ); - assert( pPage->childPtrSize==4 ); -#ifndef SQLITE_DEBUG - UNUSED_PARAMETER(pPage); -#endif - pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); - pInfo->nPayload = 0; - pInfo->nLocal = 0; - pInfo->pPayload = 0; - return; -} static void btreeParseCellPtr( MemPage *pPage, /* Page containing the cell */ u8 *pCell, /* Pointer to the cell text. */ @@ -62426,89 +57021,26 @@ static void btreeParseCellPtr( ){ u8 *pIter; /* For scanning through pCell */ u32 nPayload; /* Number of bytes of cell payload */ - u64 iKey; /* Extracted Key value */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); - assert( pPage->intKeyLeaf ); - assert( pPage->childPtrSize==0 ); - pIter = pCell; - - /* The next block of code is equivalent to: - ** - ** pIter += getVarint32(pIter, nPayload); - ** - ** The code is inlined to avoid a function call. - */ - nPayload = *pIter; - if( nPayload>=0x80 ){ - u8 *pEnd = &pIter[8]; - nPayload &= 0x7f; - do{ - nPayload = (nPayload<<7) | (*++pIter & 0x7f); - }while( (*pIter)>=0x80 && pIternKey); - ** - ** The code is inlined to avoid a function call. - */ - iKey = *pIter; - if( iKey>=0x80 ){ - u8 *pEnd = &pIter[7]; - iKey &= 0x7f; - while(1){ - iKey = (iKey<<7) | (*++pIter & 0x7f); - if( (*pIter)<0x80 ) break; - if( pIter>=pEnd ){ - iKey = (iKey<<8) | *++pIter; - break; - } - } - } - pIter++; - - pInfo->nKey = *(i64*)&iKey; - pInfo->nPayload = nPayload; - pInfo->pPayload = pIter; - testcase( nPayload==pPage->maxLocal ); - testcase( nPayload==pPage->maxLocal+1 ); - if( nPayload<=pPage->maxLocal ){ - /* This is the (easy) common case where the entire payload fits - ** on the local page. No overflow is required. - */ - pInfo->nSize = nPayload + (u16)(pIter - pCell); - if( pInfo->nSize<4 ) pInfo->nSize = 4; - pInfo->nLocal = (u16)nPayload; + if( pPage->intKeyLeaf ){ + assert( pPage->childPtrSize==0 ); + pIter = pCell + getVarint32(pCell, nPayload); + pIter += getVarint(pIter, (u64*)&pInfo->nKey); + }else if( pPage->noPayload ){ + assert( pPage->childPtrSize==4 ); + pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); + pInfo->nPayload = 0; + pInfo->nLocal = 0; + pInfo->iOverflow = 0; + pInfo->pPayload = 0; + return; }else{ - btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); - } -} -static void btreeParseCellPtrIndex( - MemPage *pPage, /* Page containing the cell */ - u8 *pCell, /* Pointer to the cell text. */ - CellInfo *pInfo /* Fill in this structure */ -){ - u8 *pIter; /* For scanning through pCell */ - u32 nPayload; /* Number of bytes of cell payload */ - - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( pPage->leaf==0 || pPage->leaf==1 ); - assert( pPage->intKeyLeaf==0 ); - pIter = pCell + pPage->childPtrSize; - nPayload = *pIter; - if( nPayload>=0x80 ){ - u8 *pEnd = &pIter[8]; - nPayload &= 0x7f; - do{ - nPayload = (nPayload<<7) | (*++pIter & 0x7f); - }while( *(pIter)>=0x80 && pIterchildPtrSize; + pIter += getVarint32(pIter, nPayload); + pInfo->nKey = nPayload; } - pIter++; - pInfo->nKey = nPayload; pInfo->nPayload = nPayload; pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); @@ -62520,8 +57052,33 @@ static void btreeParseCellPtrIndex( pInfo->nSize = nPayload + (u16)(pIter - pCell); if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; + pInfo->iOverflow = 0; }else{ - btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); + /* If the payload will not fit completely on the local page, we have + ** to decide how much to store locally and how much to spill onto + ** overflow pages. The strategy is to minimize the amount of unused + ** space on overflow pages while keeping the amount of local storage + ** in between minLocal and maxLocal. + ** + ** Warning: changing the way overflow payload is distributed in any + ** way will result in an incompatible file format. + */ + int minLocal; /* Minimum amount of payload held locally */ + int maxLocal; /* Maximum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ + + minLocal = pPage->minLocal; + maxLocal = pPage->maxLocal; + surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize - 4); + testcase( surplus==maxLocal ); + testcase( surplus==maxLocal+1 ); + if( surplus <= maxLocal ){ + pInfo->nLocal = (u16)surplus; + }else{ + pInfo->nLocal = (u16)minLocal; + } + pInfo->iOverflow = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell); + pInfo->nSize = pInfo->iOverflow + 4; } } static void btreeParseCell( @@ -62529,20 +57086,14 @@ static void btreeParseCell( int iCell, /* The cell index. First cell is 0 */ CellInfo *pInfo /* Fill in this structure */ ){ - pPage->xParseCell(pPage, findCell(pPage, iCell), pInfo); + btreeParseCellPtr(pPage, findCell(pPage, iCell), pInfo); } /* -** The following routines are implementations of the MemPage.xCellSize -** method. -** ** Compute the total number of bytes that a Cell needs in the cell ** data area of the btree-page. The return number includes the cell ** data header and the local payload, but not any overflow page or ** the space used by the cell pointer. -** -** cellSizePtrNoPayload() => table internal nodes -** cellSizePtr() => all index nodes & table leaf nodes */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ @@ -62555,12 +57106,18 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of ** this function verifies that this invariant is not violated. */ CellInfo debuginfo; - pPage->xParseCell(pPage, pCell, &debuginfo); + btreeParseCellPtr(pPage, pCell, &debuginfo); #endif + if( pPage->noPayload ){ + pEnd = &pIter[9]; + while( (*pIter++)&0x80 && pIterchildPtrSize==4 ); + return (u16)(pIter - pCell); + } nSize = *pIter; if( nSize>=0x80 ){ - pEnd = &pIter[8]; + pEnd = &pIter[9]; nSize &= 0x7f; do{ nSize = (nSize<<7) | (*++pIter & 0x7f); @@ -62592,34 +57149,12 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ assert( nSize==debuginfo.nSize || CORRUPT_DB ); return (u16)nSize; } -static u16 cellSizePtrNoPayload(MemPage *pPage, u8 *pCell){ - u8 *pIter = pCell + 4; /* For looping over bytes of pCell */ - u8 *pEnd; /* End mark for a varint */ - -#ifdef SQLITE_DEBUG - /* The value returned by this function should always be the same as - ** the (CellInfo.nSize) value found by doing a full parse of the - ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of - ** this function verifies that this invariant is not violated. */ - CellInfo debuginfo; - pPage->xParseCell(pPage, pCell, &debuginfo); -#else - UNUSED_PARAMETER(pPage); -#endif - - assert( pPage->childPtrSize==4 ); - pEnd = pIter + 9; - while( (*pIter++)&0x80 && pIterxCellSize(pPage, findCell(pPage, iCell)); + return cellSizePtr(pPage, findCell(pPage, iCell)); } #endif @@ -62633,9 +57168,9 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ CellInfo info; if( *pRC ) return; assert( pCell!=0 ); - pPage->xParseCell(pPage, pCell, &info); - if( info.nLocalpBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); } } @@ -62690,18 +57225,26 @@ static int defragmentPage(MemPage *pPage){ pc = get2byte(pAddr); testcase( pc==iCellFirst ); testcase( pc==iCellLast ); +#if !defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) /* These conditions have already been verified in btreeInitPage() - ** if PRAGMA cell_size_check=ON. + ** if SQLITE_ENABLE_OVERSIZE_CELL_CHECK is defined */ if( pciCellLast ){ return SQLITE_CORRUPT_BKPT; } +#endif assert( pc>=iCellFirst && pc<=iCellLast ); - size = pPage->xCellSize(pPage, &src[pc]); + size = cellSizePtr(pPage, &src[pc]); cbrk -= size; +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + if( cbrkusableSize ){ return SQLITE_CORRUPT_BKPT; } +#endif assert( cbrk+size<=usableSize && cbrk>=iCellFirst ); testcase( cbrk+size==usableSize ); testcase( pc+size==usableSize ); @@ -62739,20 +57282,18 @@ static int defragmentPage(MemPage *pPage){ ** This function may detect corruption within pPg. If corruption is ** detected then *pRc is set to SQLITE_CORRUPT and NULL is returned. ** -** Slots on the free list that are between 1 and 3 bytes larger than nByte -** will be ignored if adding the extra space to the fragmentation count -** causes the fragmentation count to exceed 60. +** If a slot of at least nByte bytes is found but cannot be used because +** there are already at least 60 fragmented bytes on the page, return NULL. +** In this case, if pbDefrag parameter is not NULL, set *pbDefrag to true. */ -static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ const int hdr = pPg->hdrOffset; u8 * const aData = pPg->aData; - int iAddr = hdr + 1; - int pc = get2byte(&aData[iAddr]); - int x; + int iAddr; + int pc; int usableSize = pPg->pBt->usableSize; - assert( pc>0 ); - do{ + for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ int size; /* Size of the free slot */ /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of ** increasing offset. */ @@ -62764,21 +57305,24 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** freeblock form a big-endian integer which is the size of the freeblock ** in bytes, including the 4-byte header. */ size = get2byte(&aData[pc+2]); - if( (x = size - nByte)>=0 ){ + if( size>=nByte ){ + int x = size - nByte; testcase( x==4 ); testcase( x==3 ); - if( pc < pPg->cellOffset+2*pPg->nCell || size+pc > usableSize ){ - *pRc = SQLITE_CORRUPT_BKPT; - return 0; - }else if( x<4 ){ + if( x<4 ){ /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total ** number of bytes in fragments may not exceed 60. */ - if( aData[hdr+7]>57 ) return 0; - + if( aData[hdr+7]>=60 ){ + if( pbDefrag ) *pbDefrag = 1; + return 0; + } /* Remove the slot from the free-list. Update the number of ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); aData[hdr+7] += (u8)x; + }else if( size+pc > usableSize ){ + *pRc = SQLITE_CORRUPT_BKPT; + return 0; }else{ /* The slot remains on the free-list. Reduce its size to account ** for the portion used by the new allocation. */ @@ -62786,9 +57330,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ } return &aData[pc + x]; } - iAddr = pc; - pc = get2byte(&aData[pc]); - }while( pc ); + } return 0; } @@ -62829,15 +57371,8 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** then the cell content offset of an empty page wants to be 65536. ** However, that integer is too large to be stored in a 2-byte unsigned ** integer, so a value of 0 is used in its place. */ - top = get2byte(&data[hdr+5]); - assert( top<=(int)pPage->pBt->usableSize ); /* Prevent by getAndInitPage() */ - if( gap>top ){ - if( top==0 && pPage->pBt->usableSize==65536 ){ - top = 65536; - }else{ - return SQLITE_CORRUPT_BKPT; - } - } + top = get2byteNotZero(&data[hdr+5]); + if( gap>top ) return SQLITE_CORRUPT_BKPT; /* If there is enough space between gap and top for one more cell pointer ** array entry offset, and if the freelist is not empty, then search the @@ -62846,14 +57381,15 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2==top ); testcase( gap+1==top ); testcase( gap==top ); - if( (data[hdr+2] || data[hdr+1]) && gap+2<=top ){ - u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ + int bDefrag = 0; + u8 *pSpace = pageFindSlot(pPage, nByte, &rc, &bDefrag); + if( rc ) return rc; + if( bDefrag ) goto defragment_page; if( pSpace ){ assert( pSpace>=data && (pSpace - data)<65536 ); *pIdx = (int)(pSpace - data); return SQLITE_OK; - }else if( rc ){ - return rc; } } @@ -62862,6 +57398,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ + defragment_page: assert( pPage->nCell>0 || CORRUPT_DB ); rc = defragmentPage(pPage); if( rc ) return rc; @@ -62908,7 +57445,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - assert( CORRUPT_DB || iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); + assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ @@ -62928,11 +57465,8 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ if( data[iPtr+1]==0 && data[iPtr]==0 ){ iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ }else{ - while( (iFreeBlk = get2byte(&data[iPtr]))0 && iFreeBlkiLast ) return SQLITE_CORRUPT_BKPT; @@ -62940,7 +57474,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ /* At this point: ** iFreeBlk: First freeblock after iStart, or zero if none - ** iPtr: The address of a pointer to iFreeBlk + ** iPtr: The address of a pointer iFreeBlk ** ** Check to see if iFreeBlk should be coalesced onto the end of iStart. */ @@ -62948,7 +57482,6 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ nFrag = iFreeBlk - iEnd; if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_BKPT; iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); - if( iEnd > pPage->pBt->usableSize ) return SQLITE_CORRUPT_BKPT; iSize = iEnd - iStart; iFreeBlk = get2byte(&data[iFreeBlk]); } @@ -63006,36 +57539,29 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); flagByte &= ~PTF_LEAF; pPage->childPtrSize = 4-4*pPage->leaf; - pPage->xCellSize = cellSizePtr; pBt = pPage->pBt; if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ - /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an - ** interior table b-tree page. */ + /* EVIDENCE-OF: R-03640-13415 A value of 5 means the page is an interior + ** table b-tree page. */ assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); - /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a - ** leaf table b-tree page. */ + /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf + ** table b-tree page. */ assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); pPage->intKey = 1; - if( pPage->leaf ){ - pPage->intKeyLeaf = 1; - pPage->xParseCell = btreeParseCellPtr; - }else{ - pPage->intKeyLeaf = 0; - pPage->xCellSize = cellSizePtrNoPayload; - pPage->xParseCell = btreeParseCellPtrNoPayload; - } + pPage->intKeyLeaf = pPage->leaf; + pPage->noPayload = !pPage->leaf; pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ - /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an - ** interior index b-tree page. */ + /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior + ** index b-tree page. */ assert( (PTF_ZERODATA)==2 ); - /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a - ** leaf index b-tree page. */ + /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf + ** index b-tree page. */ assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; pPage->intKeyLeaf = 0; - pPage->xParseCell = btreeParseCellPtrIndex; + pPage->noPayload = 0; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ @@ -63059,7 +57585,6 @@ static int decodeFlags(MemPage *pPage, int flagByte){ static int btreeInitPage(MemPage *pPage){ assert( pPage->pBt!=0 ); - assert( pPage->pBt->db!=0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); @@ -63091,7 +57616,6 @@ static int btreeInitPage(MemPage *pPage){ pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize; pPage->aDataEnd = &data[usableSize]; pPage->aCellIdx = &data[cellOffset]; - pPage->aDataOfst = &data[pPage->childPtrSize]; /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates ** the start of the cell content area. A zero value for this integer is ** interpreted as 65536. */ @@ -63119,19 +57643,20 @@ static int btreeInitPage(MemPage *pPage){ */ iCellFirst = cellOffset + 2*pPage->nCell; iCellLast = usableSize - 4; - if( pBt->db->flags & SQLITE_CellSizeCk ){ +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + { int i; /* Index into the cell pointer array */ int sz; /* Size of a cell */ if( !pPage->leaf ) iCellLast--; for(i=0; inCell; i++){ - pc = get2byteAligned(&data[cellOffset+i*2]); + pc = get2byte(&data[cellOffset+i*2]); testcase( pc==iCellFirst ); testcase( pc==iCellLast ); if( pciCellLast ){ return SQLITE_CORRUPT_BKPT; } - sz = pPage->xCellSize(pPage, &data[pc]); + sz = cellSizePtr(pPage, &data[pc]); testcase( pc+sz==usableSize ); if( pc+sz>usableSize ){ return SQLITE_CORRUPT_BKPT; @@ -63139,6 +57664,7 @@ static int btreeInitPage(MemPage *pPage){ } if( !pPage->leaf ) iCellLast++; } +#endif /* Compute the total free space on the page ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the @@ -63211,7 +57737,6 @@ static void zeroPage(MemPage *pPage, int flags){ pPage->cellOffset = first; pPage->aDataEnd = &data[pBt->usableSize]; pPage->aCellIdx = &data[first]; - pPage->aDataOfst = &data[pPage->childPtrSize]; pPage->nOverflow = 0; assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); @@ -63226,23 +57751,20 @@ static void zeroPage(MemPage *pPage, int flags){ */ static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ MemPage *pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); - if( pgno!=pPage->pgno ){ - pPage->aData = sqlite3PagerGetData(pDbPage); - pPage->pDbPage = pDbPage; - pPage->pBt = pBt; - pPage->pgno = pgno; - pPage->hdrOffset = pgno==1 ? 100 : 0; - } - assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); + pPage->aData = sqlite3PagerGetData(pDbPage); + pPage->pDbPage = pDbPage; + pPage->pBt = pBt; + pPage->pgno = pgno; + pPage->hdrOffset = pPage->pgno==1 ? 100 : 0; return pPage; } /* ** Get a page from the pager. Initialize the MemPage.pBt and -** MemPage.aData elements if needed. See also: btreeGetUnusedPage(). +** MemPage.aData elements if needed. ** -** If the PAGER_GET_NOCONTENT flag is set, it means that we do not care -** about the content of the page at this time. So do not go to the disk +** If the noContent flag is set, it means that we do not care about +** the content of the page at this time. So do not go to the disk ** to fetch the content. Just fill in the content with zeros for now. ** If in the future we call sqlite3PagerWrite() on this page, that ** means we have started to be concerned about content and the disk @@ -63259,7 +57781,7 @@ static int btreeGetPage( assert( flags==0 || flags==PAGER_GET_NOCONTENT || flags==PAGER_GET_READONLY ); assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); if( rc ) return rc; *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; @@ -63294,63 +57816,35 @@ SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree *p){ } /* -** Get a page from the pager and initialize it. -** -** If pCur!=0 then the page is being fetched as part of a moveToChild() -** call. Do additional sanity checking on the page in this case. -** And if the fetch fails, this routine must decrement pCur->iPage. +** Get a page from the pager and initialize it. This routine is just a +** convenience wrapper around separate calls to btreeGetPage() and +** btreeInitPage(). ** -** The page is fetched as read-write unless pCur is not NULL and is -** a read-only cursor. -** -** If an error occurs, then *ppPage is undefined. It +** If an error occurs, then the value *ppPage is set to is undefined. It ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( BtShared *pBt, /* The database file */ Pgno pgno, /* Number of the page to get */ MemPage **ppPage, /* Write the page pointer here */ - BtCursor *pCur, /* Cursor to receive the page, or NULL */ - int bReadOnly /* True for a read-only page */ + int bReadonly /* PAGER_GET_READONLY or 0 */ ){ int rc; - DbPage *pDbPage; assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pCur==0 || ppPage==&pCur->apPage[pCur->iPage] ); - assert( pCur==0 || bReadOnly==pCur->curPagerFlags ); - assert( pCur==0 || pCur->iPage>0 ); + assert( bReadonly==PAGER_GET_READONLY || bReadonly==0 ); if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; - goto getAndInitPage_error; - } - rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); - if( rc ){ - goto getAndInitPage_error; - } - *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); - if( (*ppPage)->isInit==0 ){ - btreePageFromDbPage(pDbPage, pgno, pBt); - rc = btreeInitPage(*ppPage); - if( rc!=SQLITE_OK ){ - releasePage(*ppPage); - goto getAndInitPage_error; + }else{ + rc = btreeGetPage(pBt, pgno, ppPage, bReadonly); + if( rc==SQLITE_OK && (*ppPage)->isInit==0 ){ + rc = btreeInitPage(*ppPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + } } } - assert( (*ppPage)->pgno==pgno ); - assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) ); - - /* If obtaining a child page for a cursor, we must verify that the page is - ** compatible with the root page. */ - if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ - rc = SQLITE_CORRUPT_BKPT; - releasePage(*ppPage); - goto getAndInitPage_error; - } - return SQLITE_OK; -getAndInitPage_error: - if( pCur ) pCur->iPage--; testcase( pgno==0 ); assert( pgno!=0 || rc==SQLITE_CORRUPT ); return rc; @@ -63360,49 +57854,18 @@ static int getAndInitPage( ** Release a MemPage. This should be called once for each prior ** call to btreeGetPage. */ -static void releasePageNotNull(MemPage *pPage){ - assert( pPage->aData ); - assert( pPage->pBt ); - assert( pPage->pDbPage!=0 ); - assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); - assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - sqlite3PagerUnrefNotNull(pPage->pDbPage); -} static void releasePage(MemPage *pPage){ - if( pPage ) releasePageNotNull(pPage); -} - -/* -** Get an unused page. -** -** This works just like btreeGetPage() with the addition: -** -** * If the page is already in use for some other purpose, immediately -** release it and return an SQLITE_CURRUPT error. -** * Make sure the isInit flag is clear -*/ -static int btreeGetUnusedPage( - BtShared *pBt, /* The btree */ - Pgno pgno, /* Number of the page to fetch */ - MemPage **ppPage, /* Return the page in this parameter */ - int flags /* PAGER_GET_NOCONTENT or PAGER_GET_READONLY */ -){ - int rc = btreeGetPage(pBt, pgno, ppPage, flags); - if( rc==SQLITE_OK ){ - if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ - releasePage(*ppPage); - *ppPage = 0; - return SQLITE_CORRUPT_BKPT; - } - (*ppPage)->isInit = 0; - }else{ - *ppPage = 0; + if( pPage ){ + assert( pPage->aData ); + assert( pPage->pBt ); + assert( pPage->pDbPage!=0 ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnrefNotNull(pPage->pDbPage); } - return rc; } - /* ** During a rollback, when the pager reloads information into the cache ** so that the cache is restored to its original state at the start of @@ -63509,7 +57972,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( } p = sqlite3MallocZero(sizeof(Btree)); if( !p ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } p->inTrans = TRANS_NONE; p->db = db; @@ -63533,7 +57996,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( p->sharable = 1; if( !zFullPathname ){ sqlite3_free(p); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } if( isMemdb ){ memcpy(zFullPathname, zFilename, nFilename); @@ -63601,7 +58064,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( pBt = sqlite3MallocZero( sizeof(*pBt) ); if( pBt==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; goto btree_open_out; } rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, @@ -63663,14 +58126,15 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( #if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) /* Add the new BtShared object to the linked list sharable BtShareds. */ - pBt->nRef = 1; if( p->sharable ){ MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) + pBt->nRef = 1; MUTEX_LOGIC( mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER);) if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); if( pBt->mutex==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; + db->mallocFailed = 0; goto btree_open_out; } } @@ -63693,12 +58157,12 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( for(i=0; inDb; i++){ if( (pSib = db->aDb[i].pBt)!=0 && pSib->sharable ){ while( pSib->pPrev ){ pSib = pSib->pPrev; } - if( (uptr)p->pBt<(uptr)pSib->pBt ){ + if( p->pBtpBt ){ p->pNext = pSib; p->pPrev = 0; pSib->pPrev = p; }else{ - while( pSib->pNext && (uptr)pSib->pNext->pBt<(uptr)p->pBt ){ + while( pSib->pNext && pSib->pNext->pBtpBt ){ pSib = pSib->pNext; } p->pNext = pSib->pNext; @@ -63736,7 +58200,6 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( assert( sqlite3_mutex_held(mutexOpen) ); sqlite3_mutex_leave(mutexOpen); } - assert( rc!=SQLITE_OK || sqlite3BtreeConnectionCount(*ppBtree)>0 ); return rc; } @@ -63881,11 +58344,19 @@ SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ } /* -** Change the "soft" limit on the number of pages in the cache. -** Unused and unmodified pages will be recycled when the number of -** pages in the cache exceeds this soft limit. But the size of the -** cache is allowed to grow larger than this limit if it contains -** dirty pages or pages still in active use. +** Change the limit on the number of pages allowed in the cache. +** +** The maximum number of cache pages is set to the absolute +** value of mxPage. If mxPage is negative, the pager will +** operate asynchronously - it will not stop to do fsync()s +** to insure data is written to the disk surface before +** continuing. Transactions still work if synchronous is off, +** and the database cannot be corrupted if this program +** crashes. But if the operating system crashes or there is +** an abrupt power failure when synchronous is off, the database +** could be left in an inconsistent and unrecoverable state. +** Synchronous is on by default so database corruption is not +** normally a worry. */ SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ BtShared *pBt = p->pBt; @@ -63896,26 +58367,6 @@ SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } -/* -** Change the "spill" limit on the number of pages in the cache. -** If the number of pages exceeds this limit during a write transaction, -** the pager might attempt to "spill" pages to the journal early in -** order to free up memory. -** -** The value returned is the current spill size. If zero is passed -** as an argument, no changes are made to the spill size setting, so -** using mxPage of 0 is a way to query the current spill size. -*/ -SQLITE_PRIVATE int sqlite3BtreeSetSpillSize(Btree *p, int mxPage){ - BtShared *pBt = p->pBt; - int res; - assert( sqlite3_mutex_held(p->db->mutex) ); - sqlite3BtreeEnter(p); - res = sqlite3PagerSetSpillsize(pBt->pPager, mxPage); - sqlite3BtreeLeave(p); - return res; -} - #if SQLITE_MAX_MMAP_SIZE>0 /* ** Change the limit on the amount of the database file that may be @@ -63953,6 +58404,21 @@ SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags( } #endif +/* +** Return TRUE if the given btree is set to safety level 1. In other +** words, return TRUE if no sync() occurs on the disk files. +*/ +SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree *p){ + BtShared *pBt = p->pBt; + int rc; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + assert( pBt && pBt->pPager ); + rc = sqlite3PagerNosync(pBt->pPager); + sqlite3BtreeLeave(p); + return rc; +} + /* ** Change the default pages size and the number of reserved bytes per page. ** Or, if the page size has already been fixed, return SQLITE_READONLY @@ -64198,25 +58664,9 @@ static int lockBtree(BtShared *pBt){ rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); if( rc!=SQLITE_OK ){ goto page1_init_failed; - }else{ -#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS - sqlite3 *db; - Db *pDb; - if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ - while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } - if( pDb->bSyncSet==0 - && pDb->safety_level==SQLITE_DEFAULT_SYNCHRONOUS+1 - ){ - pDb->safety_level = SQLITE_DEFAULT_WAL_SYNCHRONOUS+1; - sqlite3PagerSetFlags(pBt->pPager, - pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); - } - } -#endif - if( isOpen==0 ){ - releasePage(pPage1); - return SQLITE_OK; - } + }else if( isOpen==0 ){ + releasePage(pPage1); + return SQLITE_OK; } rc = SQLITE_NOTADB; } @@ -64358,7 +58808,7 @@ static void unlockBtreeIfUnused(BtShared *pBt){ assert( pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); pBt->pPage1 = 0; - releasePageNotNull(pPage1); + releasePage(pPage1); } } @@ -64456,6 +58906,7 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ ** proceed. */ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ + sqlite3 *pBlock = 0; BtShared *pBt = p->pBt; int rc = SQLITE_OK; @@ -64478,30 +58929,27 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ } #ifndef SQLITE_OMIT_SHARED_CACHE - { - sqlite3 *pBlock = 0; - /* If another database handle has already opened a write transaction - ** on this shared-btree structure and a second write transaction is - ** requested, return SQLITE_LOCKED. - */ - if( (wrflag && pBt->inTransaction==TRANS_WRITE) - || (pBt->btsFlags & BTS_PENDING)!=0 - ){ - pBlock = pBt->pWriter->db; - }else if( wrflag>1 ){ - BtLock *pIter; - for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ - if( pIter->pBtree!=p ){ - pBlock = pIter->pBtree->db; - break; - } + /* If another database handle has already opened a write transaction + ** on this shared-btree structure and a second write transaction is + ** requested, return SQLITE_LOCKED. + */ + if( (wrflag && pBt->inTransaction==TRANS_WRITE) + || (pBt->btsFlags & BTS_PENDING)!=0 + ){ + pBlock = pBt->pWriter->db; + }else if( wrflag>1 ){ + BtLock *pIter; + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->pBtree!=p ){ + pBlock = pIter->pBtree->db; + break; } } - if( pBlock ){ - sqlite3ConnectionBlocked(p->db, pBlock); - rc = SQLITE_LOCKED_SHAREDCACHE; - goto trans_begun; - } + } + if( pBlock ){ + sqlite3ConnectionBlocked(p->db, pBlock); + rc = SQLITE_LOCKED_SHAREDCACHE; + goto trans_begun; } #endif @@ -64665,22 +59113,20 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ u8 isInitOrig = pPage->isInit; int i; int nCell; - int rc; - rc = btreeInitPage(pPage); - if( rc ) return rc; + btreeInitPage(pPage); nCell = pPage->nCell; for(i=0; ixParseCell(pPage, pCell, &info); - if( info.nLocalaData+pPage->maskPage - && iFrom==get4byte(pCell+info.nSize-4) + btreeParseCellPtr(pPage, pCell, &info); + if( info.iOverflow + && pCell+info.iOverflow+3<=pPage->aData+pPage->maskPage + && iFrom==get4byte(&pCell[info.iOverflow]) ){ - put4byte(pCell+info.nSize-4, iTo); + put4byte(&pCell[info.iOverflow], iTo); break; } }else{ @@ -64974,7 +59420,7 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *p){ static int autoVacuumCommit(BtShared *pBt){ int rc = SQLITE_OK; Pager *pPager = pBt->pPager; - VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager); ) + VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) ); assert( sqlite3_mutex_held(pBt->mutex) ); invalidateAllOverflowCache(pBt); @@ -65383,13 +59829,13 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ ** on the database already. If a write-cursor is requested, then ** the caller is assumed to have an open write transaction. ** -** If the BTREE_WRCSR bit of wrFlag is clear, then the cursor can only -** be used for reading. If the BTREE_WRCSR bit is set, then the cursor -** can be used for reading or for writing if other conditions for writing -** are also met. These are the conditions that must be met in order -** for writing to be allowed: +** If wrFlag==0, then the cursor can only be used for reading. +** If wrFlag==1, then the cursor can be used for reading or for +** writing if other conditions for writing are also met. These +** are the conditions that must be met in order for writing to +** be allowed: ** -** 1: The cursor must have been opened with wrFlag containing BTREE_WRCSR +** 1: The cursor must have been opened with wrFlag==1 ** ** 2: Other database connections that share the same pager cache ** but which are not in the READ_UNCOMMITTED state may not have @@ -65401,16 +59847,6 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ ** ** 4: There must be an active transaction. ** -** The BTREE_FORDELETE bit of wrFlag may optionally be set if BTREE_WRCSR -** is set. If FORDELETE is set, that is a hint to the implementation that -** this cursor will only be used to seek to and delete entries of an index -** as part of a larger DELETE statement. The FORDELETE hint is not used by -** this implementation. But in a hypothetical alternative storage engine -** in which index entries are automatically deleted when corresponding table -** rows are deleted, the FORDELETE flag is a hint that all SEEK and DELETE -** operations on this cursor can be no-ops and all READ operations can -** return a null row (2-bytes: 0x01 0x00). -** ** No checking is done to make sure that page iTable really is the ** root page of a b-tree. If it is not, then the cursor acquired ** will not work correctly. @@ -65426,30 +59862,28 @@ static int btreeCursor( BtCursor *pCur /* Space for new cursor */ ){ BtShared *pBt = p->pBt; /* Shared b-tree handle */ - BtCursor *pX; /* Looping over other all cursors */ assert( sqlite3BtreeHoldsMutex(p) ); - assert( wrFlag==0 - || wrFlag==BTREE_WRCSR - || wrFlag==(BTREE_WRCSR|BTREE_FORDELETE) - ); + assert( wrFlag==0 || wrFlag==1 ); /* The following assert statements verify that if this is a sharable ** b-tree database, the connection is holding the required table locks, ** and that no other connection has any open cursor that conflicts with ** this lock. */ - assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) ); + assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, wrFlag+1) ); assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); /* Assert that the caller has opened the required transaction. */ assert( p->inTrans>TRANS_NONE ); assert( wrFlag==0 || p->inTrans==TRANS_WRITE ); assert( pBt->pPage1 && pBt->pPage1->aData ); - assert( wrFlag==0 || (pBt->btsFlags & BTS_READ_ONLY)==0 ); + if( NEVER(wrFlag && (pBt->btsFlags & BTS_READ_ONLY)!=0) ){ + return SQLITE_READONLY; + } if( wrFlag ){ allocateTempSpace(pBt); - if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM_BKPT; + if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM; } if( iTable==1 && btreePagecount(pBt)==0 ){ assert( wrFlag==0 ); @@ -65463,17 +59897,12 @@ static int btreeCursor( pCur->pKeyInfo = pKeyInfo; pCur->pBtree = p; pCur->pBt = pBt; - pCur->curFlags = wrFlag ? BTCF_WriteFlag : 0; - pCur->curPagerFlags = wrFlag ? 0 : PAGER_GET_READONLY; - /* If there are two or more cursors on the same btree, then all such - ** cursors *must* have the BTCF_Multiple flag set. */ - for(pX=pBt->pCursor; pX; pX=pX->pNext){ - if( pX->pgnoRoot==(Pgno)iTable ){ - pX->curFlags |= BTCF_Multiple; - pCur->curFlags |= BTCF_Multiple; - } - } + assert( wrFlag==0 || wrFlag==BTCF_WriteFlag ); + pCur->curFlags = wrFlag; pCur->pNext = pBt->pCursor; + if( pCur->pNext ){ + pCur->pNext->pPrev = pCur; + } pBt->pCursor = pCur; pCur->eState = CURSOR_INVALID; return SQLITE_OK; @@ -65486,13 +59915,9 @@ SQLITE_PRIVATE int sqlite3BtreeCursor( BtCursor *pCur /* Write new cursor here */ ){ int rc; - if( iTable<1 ){ - rc = SQLITE_CORRUPT_BKPT; - }else{ - sqlite3BtreeEnter(p); - rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); - sqlite3BtreeLeave(p); - } + sqlite3BtreeEnter(p); + rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + sqlite3BtreeLeave(p); return rc; } @@ -65531,18 +59956,13 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ BtShared *pBt = pCur->pBt; sqlite3BtreeEnter(pBtree); sqlite3BtreeClearCursor(pCur); - assert( pBt->pCursor!=0 ); - if( pBt->pCursor==pCur ){ - pBt->pCursor = pCur->pNext; + if( pCur->pPrev ){ + pCur->pPrev->pNext = pCur->pNext; }else{ - BtCursor *pPrev = pBt->pCursor; - do{ - if( pPrev->pNext==pCur ){ - pPrev->pNext = pCur->pNext; - break; - } - pPrev = pPrev->pNext; - }while( ALWAYS(pPrev) ); + pBt->pCursor = pCur->pNext; + } + if( pCur->pNext ){ + pCur->pNext->pPrev = pCur->pPrev; } for(i=0; i<=pCur->iPage; i++){ releasePage(pCur->apPage[i]); @@ -65562,6 +59982,13 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ ** ** BtCursor.info is a cache of the information in the current cell. ** Using this cache reduces the number of calls to btreeParseCell(). +** +** 2007-06-25: There is a bug in some versions of MSVC that cause the +** compiler to crash when getCellInfo() is implemented as a macro. +** But there is a measureable speed advantage to using the macro on gcc +** (when less compiler optimizations like -Os or -O0 are used and the +** compiler is not doing aggressive inlining.) So we use a real function +** for MSVC and a macro for everything else. Ticket #2457. */ #ifndef NDEBUG static void assertCellInfo(BtCursor *pCur){ @@ -65574,15 +60001,28 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ #else #define assertCellInfo(x) #endif -static SQLITE_NOINLINE void getCellInfo(BtCursor *pCur){ - if( pCur->info.nSize==0 ){ - int iPage = pCur->iPage; - pCur->curFlags |= BTCF_ValidNKey; - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); - }else{ - assertCellInfo(pCur); +#ifdef _MSC_VER + /* Use a real function in MSVC to work around bugs in that compiler. */ + static void getCellInfo(BtCursor *pCur){ + if( pCur->info.nSize==0 ){ + int iPage = pCur->iPage; + btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); + pCur->curFlags |= BTCF_ValidNKey; + }else{ + assertCellInfo(pCur); + } } -} +#else /* if not _MSC_VER */ + /* Use a macro in all other compilers so that the function is inlined */ +#define getCellInfo(pCur) \ + if( pCur->info.nSize==0 ){ \ + int iPage = pCur->iPage; \ + btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ + pCur->curFlags |= BTCF_ValidNKey; \ + }else{ \ + assertCellInfo(pCur); \ + } +#endif /* _MSC_VER */ #ifndef NDEBUG /* The next routine used only within assert() statements */ /* @@ -65596,33 +60036,46 @@ SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor *pCur){ #endif /* NDEBUG */ /* -** Return the value of the integer key or "rowid" for a table btree. -** This routine is only valid for a cursor that is pointing into a -** ordinary table btree. If the cursor points to an index btree or -** is invalid, the result of this routine is undefined. +** Set *pSize to the size of the buffer needed to hold the value of +** the key for the current entry. If the cursor is not pointing +** to a valid entry, *pSize is set to 0. +** +** For a table with the INTKEY flag set, this routine returns the key +** itself, not the number of bytes in the key. +** +** The caller must position the cursor prior to invoking this routine. +** +** This routine cannot fail. It always returns SQLITE_OK. */ -SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor *pCur){ +SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - assert( pCur->curIntKey ); getCellInfo(pCur); - return pCur->info.nKey; + *pSize = pCur->info.nKey; + return SQLITE_OK; } /* -** Return the number of bytes of payload for the entry that pCur is -** currently pointing to. For table btrees, this will be the amount -** of data. For index btrees, this will be the size of the key. +** Set *pSize to the number of bytes of data in the entry the +** cursor currently points to. ** ** The caller must guarantee that the cursor is pointing to a non-NULL ** valid entry. In other words, the calling procedure must guarantee ** that the cursor has Cursor.eState==CURSOR_VALID. +** +** Failure is not possible. This function always returns SQLITE_OK. +** It might just as well be a procedure (returning void) but we continue +** to return an integer result code for historical reasons. */ -SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor *pCur){ +SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>=0 ); + assert( pCur->iPageapPage[pCur->iPage]->intKeyLeaf==1 ); getCellInfo(pCur); - return pCur->info.nPayload; + *pSize = pCur->info.nPayload; + return SQLITE_OK; } /* @@ -65793,13 +60246,8 @@ static int accessPayload( #endif assert( offset+amt <= pCur->info.nPayload ); - assert( aPayload > pPage->aData ); - if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){ - /* Trying to read or write past the end of the data is an error. The - ** conditional above is really: - ** &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] - ** but is recast into its current form to avoid integer overflow problems - */ + if( &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] ){ + /* Trying to read or write past the end of the data is an error */ return SQLITE_CORRUPT_BKPT; } @@ -65839,7 +60287,7 @@ static int accessPayload( pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; }else{ pCur->nOvflAlloc = nOvfl*2; pCur->aOverflow = aNew; @@ -65867,9 +60315,7 @@ static int accessPayload( /* If required, populate the overflow page-list cache. */ if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ - assert( pCur->aOverflow[iIdx]==0 - || pCur->aOverflow[iIdx]==nextPage - || CORRUPT_DB ); + assert(!pCur->aOverflow[iIdx] || pCur->aOverflow[iIdx]==nextPage); pCur->aOverflow[iIdx] = nextPage; } @@ -65924,7 +60370,7 @@ static int accessPayload( && (bEnd || a==ovflSize) /* (6) */ && pBt->inTransaction==TRANS_READ /* (4) */ && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ - && 0==sqlite3PagerUseWal(pBt->pPager) /* (5) */ + && pBt->pPage1->aData[19]==0x01 /* (5) */ && &pBuf[-4]>=pBufStart /* (7) */ ){ u8 aSave[4]; @@ -65939,7 +60385,7 @@ static int accessPayload( { DbPage *pDbPage; - rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage, + rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, ((eOp&0x01)==0 ? PAGER_GET_READONLY : 0) ); if( rc==SQLITE_OK ){ @@ -66000,7 +60446,7 @@ SQLITE_PRIVATE int sqlite3BtreeData(BtCursor *pCur, u32 offset, u32 amt, void *p } #endif - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); rc = restoreCursorPosition(pCur); if( rc==SQLITE_OK ){ assert( pCur->eState==CURSOR_VALID ); @@ -66038,7 +60484,7 @@ static const void *fetchPayload( assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]); assert( pCur->eState==CURSOR_VALID ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); assert( pCur->info.nSize>0 ); assert( pCur->info.pPayload>pCur->apPage[pCur->iPage]->aData || CORRUPT_DB ); @@ -66064,7 +60510,10 @@ static const void *fetchPayload( ** These routines is used to get quick access to key and data ** in the common case where no overflow pages are used. */ -SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ +SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor *pCur, u32 *pAmt){ + return fetchPayload(pCur, pAmt); +} +SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){ return fetchPayload(pCur, pAmt); } @@ -66079,21 +60528,31 @@ SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ + int rc; + int i = pCur->iPage; + MemPage *pNewPage; BtShared *pBt = pCur->pBt; - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageiPage>=0 ); if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } + rc = getAndInitPage(pBt, newPgno, &pNewPage, + (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); + if( rc ) return rc; + pCur->apPage[i+1] = pNewPage; + pCur->aiIdx[i+1] = 0; + pCur->iPage++; + pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); - pCur->iPage++; - pCur->aiIdx[pCur->iPage] = 0; - return getAndInitPage(pBt, newPgno, &pCur->apPage[pCur->iPage], - pCur, pCur->curPagerFlags); + if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){ + return SQLITE_CORRUPT_BKPT; + } + return SQLITE_OK; } #if SQLITE_DEBUG @@ -66127,7 +60586,7 @@ static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ ** the largest cell index. */ static void moveToParent(BtCursor *pCur){ - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage>0 ); assert( pCur->apPage[pCur->iPage] ); @@ -66137,9 +60596,11 @@ static void moveToParent(BtCursor *pCur){ pCur->apPage[pCur->iPage]->pgno ); testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell ); + + releasePage(pCur->apPage[pCur->iPage]); + pCur->iPage--; pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); - releasePageNotNull(pCur->apPage[pCur->iPage--]); } /* @@ -66167,7 +60628,7 @@ static int moveToRoot(BtCursor *pCur){ MemPage *pRoot; int rc = SQLITE_OK; - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); @@ -66180,23 +60641,18 @@ static int moveToRoot(BtCursor *pCur){ } if( pCur->iPage>=0 ){ - while( pCur->iPage ){ - assert( pCur->apPage[pCur->iPage]!=0 ); - releasePageNotNull(pCur->apPage[pCur->iPage--]); - } + while( pCur->iPage ) releasePage(pCur->apPage[pCur->iPage--]); }else if( pCur->pgnoRoot==0 ){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - assert( pCur->iPage==(-1) ); rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->apPage[0], - 0, pCur->curPagerFlags); + (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; } pCur->iPage = 0; - pCur->curIntKey = pCur->apPage[0]->intKey; } pRoot = pCur->apPage[0]; assert( pRoot->pgno==pCur->pgnoRoot ); @@ -66246,7 +60702,7 @@ static int moveToLeftmost(BtCursor *pCur){ int rc = SQLITE_OK; MemPage *pPage; - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ assert( pCur->aiIdx[pCur->iPage]nCell ); @@ -66271,7 +60727,7 @@ static int moveToRightmost(BtCursor *pCur){ int rc = SQLITE_OK; MemPage *pPage = 0; - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); while( !(pPage = pCur->apPage[pCur->iPage])->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); @@ -66292,7 +60748,7 @@ static int moveToRightmost(BtCursor *pCur){ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ int rc; - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); rc = moveToRoot(pCur); if( rc==SQLITE_OK ){ @@ -66315,7 +60771,7 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ int rc; - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); /* If the cursor already points to the last entry, this is a no-op. */ @@ -66380,8 +60836,6 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ ** *pRes>0 The cursor is left pointing at an entry that ** is larger than intKey/pIdxKey. ** -** For index tables, the pIdxKey->eqSeen field is set to 1 if there -** exists an entry in the table that exactly matches pIdxKey. */ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( BtCursor *pCur, /* The cursor to be moved */ @@ -66393,16 +60847,15 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( int rc; RecordCompare xRecordCompare; - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( pRes ); assert( (pIdxKey==0)==(pCur->pKeyInfo==0) ); - assert( pCur->eState!=CURSOR_VALID || (pIdxKey==0)==(pCur->curIntKey!=0) ); /* If the cursor is already positioned at the point we are trying ** to move to, then just return without doing any work */ - if( pIdxKey==0 - && pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 + if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 + && pCur->apPage[0]->intKey ){ if( pCur->info.nKey==intKey ){ *pRes = 0; @@ -66437,8 +60890,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); return SQLITE_OK; } - assert( pCur->apPage[0]->intKey==pCur->curIntKey ); - assert( pCur->curIntKey || pIdxKey ); + assert( pCur->apPage[0]->intKey || pIdxKey ); for(;;){ int lwr, upr, idx, c; Pgno chldPg; @@ -66461,7 +60913,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( if( xRecordCompare==0 ){ for(;;){ i64 nCellKey; - pCell = findCellPastPtr(pPage, idx); + pCell = findCell(pPage, idx) + pPage->childPtrSize; if( pPage->intKeyLeaf ){ while( 0x80 <= *(pCell++) ){ if( pCell>=pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; @@ -66493,8 +60945,8 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( } }else{ for(;;){ - int nCell; /* Size of the pCell cell in bytes */ - pCell = findCellPastPtr(pPage, idx); + int nCell; + pCell = findCell(pPage, idx) + pPage->childPtrSize; /* The maximum supported page-size is 65536 bytes. This means that ** the maximum number of record bytes stored on an index B-Tree @@ -66522,27 +60974,14 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( /* The record flows over onto one or more overflow pages. In ** this case the whole cell needs to be parsed, a buffer allocated ** and accessPayload() used to retrieve the record into the - ** buffer before VdbeRecordCompare() can be called. - ** - ** If the record is corrupt, the xRecordCompare routine may read - ** up to two varints past the end of the buffer. An extra 18 - ** bytes of padding is allocated at the end of the buffer in - ** case this happens. */ + ** buffer before VdbeRecordCompare() can be called. */ void *pCellKey; u8 * const pCellBody = pCell - pPage->childPtrSize; - pPage->xParseCell(pPage, pCellBody, &pCur->info); + btreeParseCellPtr(pPage, pCellBody, &pCur->info); nCell = (int)pCur->info.nKey; - testcase( nCell<0 ); /* True if key size is 2^32 or more */ - testcase( nCell==0 ); /* Invalid key size: 0x80 0x80 0x00 */ - testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ - testcase( nCell==2 ); /* Minimum legal index key size */ - if( nCell<2 ){ - rc = SQLITE_CORRUPT_BKPT; - goto moveto_finish; - } - pCellKey = sqlite3Malloc( nCell+18 ); + pCellKey = sqlite3Malloc( nCell ); if( pCellKey==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; goto moveto_finish; } pCur->aiIdx[pCur->iPage] = (u16)idx; @@ -66642,7 +61081,7 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){ int idx; MemPage *pPage; - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); assert( *pRes==0 ); if( pCur->eState!=CURSOR_VALID ){ @@ -66706,7 +61145,7 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){ } SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ MemPage *pPage; - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( pRes!=0 ); assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); @@ -66751,7 +61190,7 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){ int rc; MemPage *pPage; - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( pRes!=0 ); assert( *pRes==0 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); @@ -66807,7 +61246,7 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){ return rc; } SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( pRes!=0 ); assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); @@ -66833,7 +61272,8 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ ** sqlite3PagerUnref() on the new page when it is done. ** ** SQLITE_OK is returned on success. Any other return value indicates -** an error. *ppPage is set to NULL in the event of an error. +** an error. *ppPage and *pPgno are undefined in the event of an error. +** Do not invoke sqlite3PagerUnref() on *ppPage if an error is returned. ** ** If the "nearby" parameter is not 0, then an effort is made to ** locate a page close to the page number "nearby". This can be used in an @@ -66876,7 +61316,6 @@ static int allocateBtreePage( /* There are pages on the freelist. Reuse one of those pages. */ Pgno iTrunk; u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ - u32 nSearch = 0; /* Count of the number of search attempts */ /* If eMode==BTALLOC_EXACT and a query of the pointer-map ** shows that the page 'nearby' is somewhere on the free-list, then @@ -66925,10 +61364,10 @@ static int allocateBtreePage( iTrunk = get4byte(&pPage1->aData[32]); } testcase( iTrunk==mxPage ); - if( iTrunk>mxPage || nSearch++ > n ){ + if( iTrunk>mxPage ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetUnusedPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); } if( rc ){ pTrunk = 0; @@ -66993,7 +61432,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetUnusedPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -67073,12 +61512,11 @@ static int allocateBtreePage( } put4byte(&aData[4], k-1); noContent = !btreeGetHasContent(pBt, *pPgno)? PAGER_GET_NOCONTENT : 0; - rc = btreeGetUnusedPage(pBt, *pPgno, ppPage, noContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); - *ppPage = 0; } } searchList = 0; @@ -67122,7 +61560,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -67136,12 +61574,11 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetUnusedPage(pBt, *pPgno, ppPage, bNoContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); - *ppPage = 0; } TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); } @@ -67151,8 +61588,17 @@ static int allocateBtreePage( end_allocate_page: releasePage(pTrunk); releasePage(pPrevTrunk); - assert( rc!=SQLITE_OK || sqlite3PagerPageRefcount((*ppPage)->pDbPage)<=1 ); - assert( rc!=SQLITE_OK || (*ppPage)->isInit==0 ); + if( rc==SQLITE_OK ){ + if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ + releasePage(*ppPage); + *ppPage = 0; + return SQLITE_CORRUPT_BKPT; + } + (*ppPage)->isInit = 0; + }else{ + *ppPage = 0; + } + assert( rc!=SQLITE_OK || sqlite3PagerIswriteable((*ppPage)->pDbPage) ); return rc; } @@ -67177,10 +61623,9 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ int nFree; /* Initial number of pages on free-list */ assert( sqlite3_mutex_held(pBt->mutex) ); - assert( CORRUPT_DB || iPage>1 ); + assert( iPage>1 ); assert( !pMemPage || pMemPage->pgno==iPage ); - if( iPage<2 ) return SQLITE_CORRUPT_BKPT; if( pMemPage ){ pPage = pMemPage; sqlite3PagerRef(pPage->pDbPage); @@ -67320,21 +61765,19 @@ static int clearCell( u32 ovflPageSize; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - pPage->xParseCell(pPage, pCell, &info); + btreeParseCellPtr(pPage, pCell, &info); *pnSize = info.nSize; - if( info.nLocal==info.nPayload ){ + if( info.iOverflow==0 ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } - if( pCell+info.nSize-1 > pPage->aData+pPage->maskPage ){ + if( pCell+info.iOverflow+3 > pPage->aData+pPage->maskPage ){ return SQLITE_CORRUPT_BKPT; /* Cell extends past end of page */ } - ovflPgno = get4byte(pCell + info.nSize - 4); + ovflPgno = get4byte(&pCell[info.iOverflow]); assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize; - assert( nOvfl>0 || - (CORRUPT_DB && (info.nPayload + ovflPageSize)0 ); while( nOvfl-- ){ Pgno iNext = 0; MemPage *pOvfl = 0; @@ -67391,7 +61834,9 @@ static int clearCell( static int fillInCell( MemPage *pPage, /* The page that contains the cell */ unsigned char *pCell, /* Complete text of the cell */ - const BtreePayload *pX, /* Payload with which to construct the cell */ + const void *pKey, i64 nKey, /* The key */ + const void *pData,int nData, /* The data */ + int nZero, /* Extra zero bytes to append to pData */ int *pnSize /* Write cell size here */ ){ int nPayload; @@ -67415,21 +61860,28 @@ static int fillInCell( /* Fill in the header. */ nHeader = pPage->childPtrSize; - if( pPage->intKey ){ - nPayload = pX->nData + pX->nZero; - pSrc = pX->pData; - nSrc = pX->nData; - assert( pPage->intKeyLeaf ); /* fillInCell() only called for leaves */ + nPayload = nData + nZero; + if( pPage->intKeyLeaf ){ nHeader += putVarint32(&pCell[nHeader], nPayload); - nHeader += putVarint(&pCell[nHeader], *(u64*)&pX->nKey); }else{ - assert( pX->nKey<=0x7fffffff && pX->pKey!=0 ); - nSrc = nPayload = (int)pX->nKey; - pSrc = pX->pKey; - nHeader += putVarint32(&pCell[nHeader], nPayload); + assert( nData==0 ); + assert( nZero==0 ); } + nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); - /* Fill in the payload */ + /* Fill in the payload size */ + if( pPage->intKey ){ + pSrc = pData; + nSrc = nData; + nData = 0; + }else{ + if( NEVER(nKey>0x7fffffff || pKey==0) ){ + return SQLITE_CORRUPT_BKPT; + } + nPayload = (int)nKey; + pSrc = pKey; + nSrc = (int)nKey; + } if( nPayload<=pPage->maxLocal ){ n = nHeader + nPayload; testcase( n==3 ); @@ -67465,11 +61917,12 @@ static int fillInCell( #if SQLITE_DEBUG { CellInfo info; - pPage->xParseCell(pPage, pCell, &info); - assert( nHeader==(int)(info.pPayload - pCell) ); - assert( info.nKey==pX->nKey ); + btreeParseCellPtr(pPage, pCell, &info); + assert( nHeader=(int)(info.pPayload - pCell) ); + assert( info.nKey==nKey ); assert( *pnSize == info.nSize ); assert( spaceLeft == info.nLocal ); + assert( pPrior == &pCell[info.iOverflow] ); } #endif @@ -67552,6 +62005,10 @@ static int fillInCell( pSrc += n; nSrc -= n; spaceLeft -= n; + if( nSrc==0 ){ + nSrc = nData; + pSrc = pData; + } } releasePage(pToRelease); return SQLITE_OK; @@ -67575,7 +62032,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ if( *pRC ) return; assert( idx>=0 && idxnCell ); - assert( CORRUPT_DB || sz==cellSize(pPage, idx) ); + assert( sz==cellSize(pPage, idx) ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); data = pPage->aData; @@ -67618,8 +62075,6 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** in pTemp or the original pCell) and also record its index. ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. -** -** *pRC must be SQLITE_OK when this routine is called. */ static void insertCell( MemPage *pPage, /* Page into which we are copying */ @@ -67632,10 +62087,13 @@ static void insertCell( ){ int idx = 0; /* Where to write new cell content in data[] */ int j; /* Loop counter */ + int end; /* First byte past the last cell pointer in data[] */ + int ins; /* Index in data[] where new cell pointer is inserted */ + int cellOffset; /* Address of first cell pointer in data[] */ u8 *data; /* The content of the whole page */ - u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ - assert( *pRC==SQLITE_OK ); + if( *pRC ) return; + assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); assert( MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); @@ -67647,7 +62105,7 @@ static void insertCell( ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size ** might be less than 8 (leaf-size + pointer) on the interior node. Hence ** the term after the || in the following assert(). */ - assert( sz==pPage->xCellSize(pPage, pCell) || (sz==8 && iChild>0) ); + assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp, pCell, sz); @@ -67660,14 +62118,6 @@ static void insertCell( assert( j<(int)(sizeof(pPage->apOvfl)/sizeof(pPage->apOvfl[0])) ); pPage->apOvfl[j] = pCell; pPage->aiOvfl[j] = (u16)i; - - /* When multiple overflows occur, they are always sequential and in - ** sorted order. This invariants arise because multiple overflows can - ** only occur when inserting divider cells into the parent page during - ** balancing, and the dividers are adjacent and sorted. - */ - assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ - assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ }else{ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ @@ -67676,26 +62126,24 @@ static void insertCell( } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); data = pPage->aData; - assert( &data[pPage->cellOffset]==pPage->aCellIdx ); + cellOffset = pPage->cellOffset; + end = cellOffset + 2*pPage->nCell; + ins = cellOffset + 2*i; rc = allocateSpace(pPage, sz, &idx); if( rc ){ *pRC = rc; return; } - /* The allocateSpace() routine guarantees the following properties - ** if it returns successfully */ - assert( idx >= 0 ); - assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); + /* The allocateSpace() routine guarantees the following two properties + ** if it returns success */ + assert( idx >= end+2 ); assert( idx+sz <= (int)pPage->pBt->usableSize ); + pPage->nCell++; pPage->nFree -= (u16)(2 + sz); memcpy(&data[idx], pCell, sz); if( iChild ){ put4byte(&data[idx], iChild); } - pIns = pPage->aCellIdx + i*2; - memmove(pIns+2, pIns, 2*(pPage->nCell - i)); - put2byte(pIns, idx); - pPage->nCell++; - /* increment the cell count */ - if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; - assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell ); + memmove(&data[ins+2], &data[ins], end-ins); + put2byte(&data[ins], idx); + put2byte(&data[pPage->hdrOffset+3], pPage->nCell); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ /* The cell may contain a pointer to an overflow page. If so, write @@ -67707,52 +62155,6 @@ static void insertCell( } } -/* -** A CellArray object contains a cache of pointers and sizes for a -** consecutive sequence of cells that might be held on multiple pages. -*/ -typedef struct CellArray CellArray; -struct CellArray { - int nCell; /* Number of cells in apCell[] */ - MemPage *pRef; /* Reference page */ - u8 **apCell; /* All cells begin balanced */ - u16 *szCell; /* Local size of all cells in apCell[] */ -}; - -/* -** Make sure the cell sizes at idx, idx+1, ..., idx+N-1 have been -** computed. -*/ -static void populateCellCache(CellArray *p, int idx, int N){ - assert( idx>=0 && idx+N<=p->nCell ); - while( N>0 ){ - assert( p->apCell[idx]!=0 ); - if( p->szCell[idx]==0 ){ - p->szCell[idx] = p->pRef->xCellSize(p->pRef, p->apCell[idx]); - }else{ - assert( CORRUPT_DB || - p->szCell[idx]==p->pRef->xCellSize(p->pRef, p->apCell[idx]) ); - } - idx++; - N--; - } -} - -/* -** Return the size of the Nth element of the cell array -*/ -static SQLITE_NOINLINE u16 computeCellSize(CellArray *p, int N){ - assert( N>=0 && NnCell ); - assert( p->szCell[N]==0 ); - p->szCell[N] = p->pRef->xCellSize(p->pRef, p->apCell[N]); - return p->szCell[N]; -} -static u16 cachedCellSize(CellArray *p, int N){ - assert( N>=0 && NnCell ); - if( p->szCell[N] ) return p->szCell[N]; - return computeCellSize(p, N); -} - /* ** Array apCell[] contains pointers to nCell b-tree page cells. The ** szCell[] array contains the size in bytes of each cell. This function @@ -67766,7 +62168,7 @@ static u16 cachedCellSize(CellArray *p, int N){ ** The MemPage.nFree field is invalidated by this function. It is the ** responsibility of the caller to set it correctly. */ -static int rebuildPage( +static void rebuildPage( MemPage *pPg, /* Edit this page */ int nCell, /* Final number of cells on page */ u8 **apCell, /* Array of cells */ @@ -67787,16 +62189,14 @@ static int rebuildPage( pData = pEnd; for(i=0; iaData && pCellxCellSize(pPg, pCell) || CORRUPT_DB ); - testcase( szCell[i]!=pPg->xCellSize(pPg,pCell) ); + assert( szCell[i]==cellSizePtr(pPg, pCell) ); } /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ @@ -67807,7 +62207,6 @@ static int rebuildPage( put2byte(&aData[hdr+3], pPg->nCell); put2byte(&aData[hdr+5], pData - aData); aData[hdr+7] = 0x00; - return SQLITE_OK; } /* @@ -67840,31 +62239,25 @@ static int pageInsertArray( u8 *pBegin, /* End of cell-pointer array */ u8 **ppData, /* IN/OUT: Page content -area pointer */ u8 *pCellptr, /* Pointer to cell-pointer area */ - int iFirst, /* Index of first cell to add */ int nCell, /* Number of cells to add to pPg */ - CellArray *pCArray /* Array of cells */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ ){ int i; u8 *aData = pPg->aData; u8 *pData = *ppData; - int iEnd = iFirst + nCell; + const int bFreelist = aData[1] || aData[2]; assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ - for(i=iFirst; iapCell[i] will never overlap on a well-formed - ** database. But they might for a corrupt database. Hence use memmove() - ** since memcpy() sends SIGABORT with overlapping buffers on OpenBSD */ - assert( (pSlot+sz)<=pCArray->apCell[i] - || pSlot>=(pCArray->apCell[i]+sz) - || CORRUPT_DB ); - memmove(pSlot, pCArray->apCell[i], sz); + memcpy(pSlot, apCell[i], sz); put2byte(pCellptr, (pSlot - aData)); pCellptr += 2; } @@ -67883,27 +62276,22 @@ static int pageInsertArray( */ static int pageFreeArray( MemPage *pPg, /* Page to edit */ - int iFirst, /* First cell to delete */ int nCell, /* Cells to delete */ - CellArray *pCArray /* Array of cells */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ ){ u8 * const aData = pPg->aData; u8 * const pEnd = &aData[pPg->pBt->usableSize]; u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; int i; - int iEnd = iFirst + nCell; u8 *pFree = 0; int szFree = 0; - for(i=iFirst; iapCell[i]; - if( SQLITE_WITHIN(pCell, pStart, pEnd) ){ - int sz; - /* No need to use cachedCellSize() here. The sizes of all cells that - ** are to be freed have already been computing while deciding which - ** cells need freeing */ - sz = pCArray->szCell[i]; assert( sz>0 ); + for(i=0; i=pStart && pCellaData && (pFree - aData)<65536 ); @@ -67938,12 +62326,13 @@ static int pageFreeArray( ** The pPg->nFree field is invalid when this function returns. It is the ** responsibility of the caller to set it correctly. */ -static int editPage( +static void editPage( MemPage *pPg, /* Edit this page */ int iOld, /* Index of first cell currently on page */ int iNew, /* Index of new first cell on page */ int nNew, /* Final number of cells on page */ - CellArray *pCArray /* Array of cells and sizes */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ ){ u8 * const aData = pPg->aData; const int hdr = pPg->hdrOffset; @@ -67962,12 +62351,16 @@ static int editPage( /* Remove cells from the start and end of the page */ if( iOldaCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); nCell -= nShift; } if( iNewEnd < iOldEnd ){ - nCell -= pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); + nCell -= pageFreeArray( + pPg, iOldEnd-iNewEnd, &apCell[iNewEnd], &szCell[iNewEnd] + ); } pData = &aData[get2byteNotZero(&aData[hdr+5])]; @@ -67981,7 +62374,7 @@ static int editPage( memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); if( pageInsertArray( pPg, pBegin, &pData, pCellptr, - iNew, nAdd, pCArray + nAdd, &apCell[iNew], &szCell[iNew] ) ) goto editpage_fail; nCell += nAdd; } @@ -67995,7 +62388,7 @@ static int editPage( nCell++; if( pageInsertArray( pPg, pBegin, &pData, pCellptr, - iCell+iNew, 1, pCArray + 1, &apCell[iCell + iNew], &szCell[iCell + iNew] ) ) goto editpage_fail; } } @@ -68004,7 +62397,7 @@ static int editPage( pCellptr = &pPg->aCellIdx[nCell*2]; if( pageInsertArray( pPg, pBegin, &pData, pCellptr, - iNew+nCell, nNew-nCell, pCArray + nNew-nCell, &apCell[iNew+nCell], &szCell[iNew+nCell] ) ) goto editpage_fail; pPg->nCell = nNew; @@ -68015,21 +62408,19 @@ static int editPage( #ifdef SQLITE_DEBUG for(i=0; iapCell[i+iNew]; - int iOff = get2byteAligned(&pPg->aCellIdx[i*2]); - if( SQLITE_WITHIN(pCell, aData, &aData[pPg->pBt->usableSize]) ){ + u8 *pCell = apCell[i+iNew]; + int iOff = get2byte(&pPg->aCellIdx[i*2]); + if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ pCell = &pTmp[pCell - aData]; } - assert( 0==memcmp(pCell, &aData[iOff], - pCArray->pRef->xCellSize(pCArray->pRef, pCArray->apCell[i+iNew])) ); + assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); } #endif - return SQLITE_OK; + return; editpage_fail: /* Unable to edit this page. Rebuild it from scratch instead. */ - populateCellCache(pCArray, iNew, nNew); - return rebuildPage(pPg, nNew, &pCArray->apCell[iNew], &pCArray->szCell[iNew]); + rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]); } /* @@ -68095,14 +62486,13 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ u8 *pOut = &pSpace[4]; u8 *pCell = pPage->apOvfl[0]; - u16 szCell = pPage->xCellSize(pPage, pCell); + u16 szCell = cellSizePtr(pPage, pCell); u8 *pStop; assert( sqlite3PagerIswriteable(pNew->pDbPage) ); assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - rc = rebuildPage(pNew, 1, &pCell, &szCell); - if( NEVER(rc) ) return rc; + rebuildPage(pNew, 1, &pCell, &szCell); pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map @@ -68141,10 +62531,8 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ while( ((*(pOut++) = *(pCell++))&0x80) && pCellnCell, pSpace, (int)(pOut-pSpace), - 0, pPage->pgno, &rc); - } + insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno, &rc); /* Set the right-child pointer of pParent to point to the new page. */ put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); @@ -68177,9 +62565,9 @@ static int ptrmapCheckPages(MemPage **apPage, int nPage){ u8 *z; z = findCell(pPage, j); - pPage->xParseCell(pPage, z, &info); - if( info.nLocalpgno && e==PTRMAP_OVERFLOW1 ); } @@ -68297,6 +62685,9 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ ** If aOvflSpace is set to a null pointer, this function returns ** SQLITE_NOMEM. */ +#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM) +#pragma optimize("", off) +#endif static int balance_nonroot( MemPage *pParent, /* Parent page of siblings being balanced */ int iParentIdx, /* Index of "the page" in pParent */ @@ -68305,6 +62696,7 @@ static int balance_nonroot( int bBulk /* True if this call is part of a bulk load */ ){ BtShared *pBt; /* The whole database */ + int nCell = 0; /* Number of cells in apCell[] */ int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ int nNew = 0; /* Number of pages in apNew[] */ int nOld; /* Number of pages in apOld[] */ @@ -68315,6 +62707,7 @@ static int balance_nonroot( int leafData; /* True if pPage is a leaf of a LEAFDATA tree */ int usableSpace; /* Bytes in pPage beyond the header */ int pageFlags; /* Value of pPage->aData[0] */ + int subtotal; /* Subtotal of bytes in cells on one page */ int iSpace1 = 0; /* First unused byte of aSpace1[] */ int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ int szScratch; /* Size of scratch memory requested */ @@ -68322,20 +62715,19 @@ static int balance_nonroot( MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ - int cntNew[NB+2]; /* Index in b.paCell[] of cell after i-th page */ - int cntOld[NB+2]; /* Old index in b.apCell[] */ + int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ + int cntOld[NB+2]; /* Old index in aCell[] after i-th page */ int szNew[NB+2]; /* Combined size of cells placed on i-th page */ + u8 **apCell = 0; /* All cells begin balanced */ + u16 *szCell; /* Local size of all cells in apCell[] */ u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ u8 abDone[NB+2]; /* True after i'th new page is populated */ Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ - CellArray b; /* Parsed information on cells being balanced */ memset(abDone, 0, sizeof(abDone)); - b.nCell = 0; - b.apCell = 0; pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -68353,7 +62745,7 @@ static int balance_nonroot( assert( pParent->nOverflow==0 || pParent->aiOvfl[0]==iParentIdx ); if( !aOvflSpace ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } /* Find the sibling pages to balance. Also locate the cells in pParent @@ -68389,7 +62781,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i], 0, 0); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -68400,12 +62792,12 @@ static int balance_nonroot( if( i+nxDiv==pParent->aiOvfl[0] && pParent->nOverflow ){ apDiv[i] = pParent->apOvfl[0]; pgno = get4byte(apDiv[i]); - szNew[i] = pParent->xCellSize(pParent, apDiv[i]); + szNew[i] = cellSizePtr(pParent, apDiv[i]); pParent->nOverflow = 0; }else{ apDiv[i] = findCell(pParent, i+nxDiv-pParent->nOverflow); pgno = get4byte(apDiv[i]); - szNew[i] = pParent->xCellSize(pParent, apDiv[i]); + szNew[i] = cellSizePtr(pParent, apDiv[i]); /* Drop the cell from the parent page. apDiv[i] still points to ** the cell within the parent, even though it has been dropped. @@ -68444,200 +62836,130 @@ static int balance_nonroot( ** Allocate space for memory structures */ szScratch = - nMaxCells*sizeof(u8*) /* b.apCell */ - + nMaxCells*sizeof(u16) /* b.szCell */ + nMaxCells*sizeof(u8*) /* apCell */ + + nMaxCells*sizeof(u16) /* szCell */ + pBt->pageSize; /* aSpace1 */ /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer ** that is more than 6 times the database page size. */ assert( szScratch<=6*(int)pBt->pageSize ); - b.apCell = sqlite3ScratchMalloc( szScratch ); - if( b.apCell==0 ){ - rc = SQLITE_NOMEM_BKPT; + apCell = sqlite3ScratchMalloc( szScratch ); + if( apCell==0 ){ + rc = SQLITE_NOMEM; goto balance_cleanup; } - b.szCell = (u16*)&b.apCell[nMaxCells]; - aSpace1 = (u8*)&b.szCell[nMaxCells]; + szCell = (u16*)&apCell[nMaxCells]; + aSpace1 = (u8*)&szCell[nMaxCells]; assert( EIGHT_BYTE_ALIGNMENT(aSpace1) ); /* ** Load pointers to all cells on sibling pages and the divider cells - ** into the local b.apCell[] array. Make copies of the divider cells + ** into the local apCell[] array. Make copies of the divider cells ** into space obtained from aSpace1[]. The divider cells have already ** been removed from pParent. ** ** If the siblings are on leaf pages, then the child pointers of the ** divider cells are stripped from the cells before they are copied - ** into aSpace1[]. In this way, all cells in b.apCell[] are without + ** into aSpace1[]. In this way, all cells in apCell[] are without ** child pointers. If siblings are not leaves, then all cell in - ** b.apCell[] include child pointers. Either way, all cells in b.apCell[] + ** apCell[] include child pointers. Either way, all cells in apCell[] ** are alike. ** ** leafCorrection: 4 if pPage is a leaf. 0 if pPage is not a leaf. ** leafData: 1 if pPage holds key+data and pParent holds only keys. */ - b.pRef = apOld[0]; - leafCorrection = b.pRef->leaf*4; - leafData = b.pRef->intKeyLeaf; + leafCorrection = apOld[0]->leaf*4; + leafData = apOld[0]->intKeyLeaf; for(i=0; inCell; - u8 *aData = pOld->aData; - u16 maskPage = pOld->maskPage; - u8 *piCell = aData + pOld->cellOffset; - u8 *piEnd; - - /* Verify that all sibling pages are of the same "type" (table-leaf, - ** table-interior, index-leaf, or index-interior). - */ - if( pOld->aData[0]!=apOld[0]->aData[0] ){ - rc = SQLITE_CORRUPT_BKPT; - goto balance_cleanup; - } - /* Load b.apCell[] with pointers to all cells in pOld. If pOld - ** constains overflow cells, include them in the b.apCell[] array - ** in the correct spot. - ** - ** Note that when there are multiple overflow cells, it is always the - ** case that they are sequential and adjacent. This invariant arises - ** because multiple overflows can only occurs when inserting divider - ** cells into a parent on a prior balance, and divider cells are always - ** adjacent and are inserted in order. There is an assert() tagged - ** with "NOTE 1" in the overflow cell insertion loop to prove this - ** invariant. - ** - ** This must be done in advance. Once the balance starts, the cell - ** offset section of the btree page will be overwritten and we will no - ** long be able to find the cells if a pointer to each cell is not saved - ** first. - */ - memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); + limit = pOld->nCell+pOld->nOverflow; if( pOld->nOverflow>0 ){ - limit = pOld->aiOvfl[0]; for(j=0; jnOverflow; k++){ - assert( k==0 || pOld->aiOvfl[k-1]+1==pOld->aiOvfl[k] );/* NOTE 1 */ - b.apCell[b.nCell] = pOld->apOvfl[k]; - b.nCell++; + }else{ + u8 *aData = pOld->aData; + u16 maskPage = pOld->maskPage; + u16 cellOffset = pOld->cellOffset; + for(j=0; jcellOffset + 2*pOld->nCell; - while( piCellmaxLocal+23 ); assert( iSpace1 <= (int)pBt->pageSize ); memcpy(pTemp, apDiv[i], sz); - b.apCell[b.nCell] = pTemp+leafCorrection; + apCell[nCell] = pTemp+leafCorrection; assert( leafCorrection==0 || leafCorrection==4 ); - b.szCell[b.nCell] = b.szCell[b.nCell] - leafCorrection; + szCell[nCell] = szCell[nCell] - leafCorrection; if( !pOld->leaf ){ assert( leafCorrection==0 ); assert( pOld->hdrOffset==0 ); /* The right pointer of the child page pOld becomes the left ** pointer of the divider cell */ - memcpy(b.apCell[b.nCell], &pOld->aData[8], 4); + memcpy(apCell[nCell], &pOld->aData[8], 4); }else{ assert( leafCorrection==4 ); - while( b.szCell[b.nCell]<4 ){ + if( szCell[nCell]<4 ){ /* Do not allow any cells smaller than 4 bytes. If a smaller cell ** does exist, pad it with 0x00 bytes. */ - assert( b.szCell[b.nCell]==3 || CORRUPT_DB ); - assert( b.apCell[b.nCell]==&aSpace1[iSpace1-3] || CORRUPT_DB ); + assert( szCell[nCell]==3 ); + assert( apCell[nCell]==&aSpace1[iSpace1-3] ); aSpace1[iSpace1++] = 0x00; - b.szCell[b.nCell]++; + szCell[nCell] = 4; } } - b.nCell++; + nCell++; } } /* - ** Figure out the number of pages needed to hold all b.nCell cells. + ** Figure out the number of pages needed to hold all nCell cells. ** Store this number in "k". Also compute szNew[] which is the total ** size of all cells on the i-th page and cntNew[] which is the index - ** in b.apCell[] of the cell that divides page i from page i+1. - ** cntNew[k] should equal b.nCell. + ** in apCell[] of the cell that divides page i from page i+1. + ** cntNew[k] should equal nCell. ** ** Values computed by this block: ** ** k: The total number of sibling pages ** szNew[i]: Spaced used on the i-th sibling page. - ** cntNew[i]: Index in b.apCell[] and b.szCell[] for the first cell to + ** cntNew[i]: Index in apCell[] and szCell[] for the first cell to ** the right of the i-th sibling page. ** usableSpace: Number of bytes of space available on each sibling. ** */ usableSpace = pBt->usableSize - 12 + leafCorrection; - for(i=0; inFree; - if( szNew[i]<0 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } - for(j=0; jnOverflow; j++){ - szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); - } - cntNew[i] = cntOld[i]; - } - k = nOld; - for(i=0; iusableSpace ){ - if( i+1>=k ){ - k = i+2; - if( k>NB+2 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } - szNew[k-1] = 0; - cntNew[k-1] = b.nCell; - } - sz = 2 + cachedCellSize(&b, cntNew[i]-1); - szNew[i] -= sz; - if( !leafData ){ - if( cntNew[i]usableSpace ) break; - szNew[i] += sz; - cntNew[i]++; - if( !leafData ){ - if( cntNew[i]=b.nCell ){ - k = i+1; - }else if( cntNew[i] <= (i>0 ? cntNew[i-1] : 0) ){ - rc = SQLITE_CORRUPT_BKPT; - goto balance_cleanup; - } - } + for(subtotal=k=i=0; i usableSpace ){ + szNew[k] = subtotal - szCell[i] - 2; + cntNew[k] = i; + if( leafData ){ i--; } + subtotal = 0; + k++; + if( k>NB+1 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + } + } + szNew[k] = subtotal; + cntNew[k] = nCell; + k++; /* ** The packing computed by the previous block is biased toward the siblings @@ -68658,27 +62980,19 @@ static int balance_nonroot( r = cntNew[i-1] - 1; d = r + 1 - leafData; - (void)cachedCellSize(&b, d); - do{ - assert( d szLeft-(b.szCell[r]+(i==k-1?0:2)))){ - break; - } - szRight += b.szCell[d] + 2; - szLeft -= b.szCell[r] + 2; - cntNew[i-1] = r; - r--; - d--; - }while( r>=0 ); + assert( d1 ? cntNew[i-2] : 0) ){ - rc = SQLITE_CORRUPT_BKPT; - goto balance_cleanup; - } } /* Sanity check: For a non-corrupt database file one of the follwing @@ -68698,6 +63012,10 @@ static int balance_nonroot( /* ** Allocate k new pages. Reuse old pages where possible. */ + if( apOld[0]->pgno<=1 ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } pageFlags = apOld[0]->aData[0]; for(i=0; inCell + pOld->nOverflow + !leafData; @@ -68839,15 +63157,15 @@ static int balance_nonroot( ** overflow cell), we can skip updating the pointer map entries. */ if( iOld>=nNew || pNew->pgno!=aPgno[iOld] - || !SQLITE_WITHIN(pCell,aOld,&aOld[usableSize]) + || pCell=&aOld[usableSize] ){ if( !leafCorrection ){ ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); } - if( cachedCellSize(&b,i)>pNew->minLocal ){ + if( szCell[i]>pNew->minLocal ){ ptrmapPutOvflPtr(pNew, pCell, &rc); } - if( rc ) goto balance_cleanup; } } } @@ -68861,21 +63179,20 @@ static int balance_nonroot( j = cntNew[i]; assert( jleaf ){ memcpy(&pNew->aData[8], pCell, 4); }else if( leafData ){ /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in b.apCell[]. Instead, the divider + ** then there is no divider cell in apCell[]. Instead, the divider ** cell consists of the integer key for the right-most cell of ** the sibling-page assembled above only. */ CellInfo info; j--; - pNew->xParseCell(pNew, b.apCell[j], &info); + btreeParseCellPtr(pNew, apCell[j], &info); pCell = pTemp; sz = 4 + putVarint(&pCell[4], info.nKey); pTemp = 0; @@ -68888,13 +63205,13 @@ static int balance_nonroot( ** any cell). But it is important to pass the correct size to ** insertCell(), so reparse the cell now. ** - ** This can only happen for b-trees used to evaluate "IN (SELECT ...)" - ** and WITHOUT ROWID tables with exactly one column which is the - ** primary key. + ** Note that this can never happen in an SQLite data file, as all + ** cells are at least 4 bytes. It only happens in b-trees used + ** to evaluate "IN (SELECT ...)" and similar clauses. */ - if( b.szCell[j]==4 ){ + if( szCell[j]==4 ){ assert(leafCorrection==4); - sz = pParent->xCellSize(pParent, pCell); + sz = cellSizePtr(pParent, pCell); } } iOvflSpace += sz; @@ -68950,13 +63267,12 @@ static int balance_nonroot( iNew = iOld = 0; nNewCell = cntNew[0]; }else{ - iOld = iPgnFree = usableSpace-szNew[iPg]; assert( apNew[iPg]->nOverflow==0 ); @@ -68986,7 +63302,7 @@ static int balance_nonroot( ** by smaller than the child due to the database header, and so all the ** free space needs to be up front. */ - assert( nNew==1 || CORRUPT_DB ); + assert( nNew==1 ); rc = defragmentPage(apNew[0]); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == @@ -69007,7 +63323,7 @@ static int balance_nonroot( assert( pParent->isInit ); TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", - nOld, nNew, b.nCell)); + nOld, nNew, nCell)); /* Free any old pages that were not reused as new pages. */ @@ -69030,7 +63346,7 @@ static int balance_nonroot( ** Cleanup before returning. */ balance_cleanup: - sqlite3ScratchFree(b.apCell); + sqlite3ScratchFree(apCell); for(i=0; i= 1700 && defined(_M_ARM) +#pragma optimize("", on) +#endif /* @@ -69124,8 +63443,8 @@ static int balance(BtCursor *pCur){ u8 aBalanceQuickSpace[13]; u8 *pFree = 0; - VVA_ONLY( int balance_quick_called = 0 ); - VVA_ONLY( int balance_deeper_called = 0 ); + TESTONLY( int balance_quick_called = 0 ); + TESTONLY( int balance_deeper_called = 0 ); do { int iPage = pCur->iPage; @@ -69138,8 +63457,7 @@ static int balance(BtCursor *pCur){ ** and copy the current contents of the root-page to it. The ** next iteration of the do-loop will balance the child page. */ - assert( balance_deeper_called==0 ); - VVA_ONLY( balance_deeper_called++ ); + assert( (balance_deeper_called++)==0 ); rc = balance_deeper(pPage, &pCur->apPage[1]); if( rc==SQLITE_OK ){ pCur->iPage = 1; @@ -69178,8 +63496,7 @@ static int balance(BtCursor *pCur){ ** function. If this were not verified, a subtle bug involving reuse ** of the aBalanceQuickSpace[] might sneak in. */ - assert( balance_quick_called==0 ); - VVA_ONLY( balance_quick_called++ ); + assert( (balance_quick_called++)==0 ); rc = balance_quick(pParent, pPage, aBalanceQuickSpace); }else #endif @@ -69236,19 +63553,13 @@ static int balance(BtCursor *pCur){ /* -** Insert a new record into the BTree. The content of the new record -** is described by the pX object. The pCur cursor is used only to -** define what table the record should be inserted into, and is left -** pointing at a random location. +** Insert a new record into the BTree. The key is given by (pKey,nKey) +** and the data is given by (pData,nData). The cursor is used only to +** define what table the record should be inserted into. The cursor +** is left pointing at a random location. ** -** For a table btree (used for rowid tables), only the pX.nKey value of -** the key is used. The pX.pKey value must be NULL. The pX.nKey is the -** rowid or INTEGER PRIMARY KEY of the row. The pX.nData,pData,nZero fields -** hold the content of the row. -** -** For an index btree (used for indexes and WITHOUT ROWID tables), the -** key is an arbitrary byte sequence stored in pX.pKey,nKey. The -** pX.pData,nData,nZero fields must be zero. +** For an INTKEY table, only the nKey value of the key is used. pKey is +** ignored. For a ZERODATA table, the pData and nData are both ignored. ** ** If the seekResult parameter is non-zero, then a successful call to ** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already @@ -69265,7 +63576,9 @@ static int balance(BtCursor *pCur){ */ SQLITE_PRIVATE int sqlite3BtreeInsert( BtCursor *pCur, /* Insert data into the table of this cursor */ - const BtreePayload *pX, /* Content of the row to be inserted */ + const void *pKey, i64 nKey, /* The key of the new record */ + const void *pData, int nData, /* The data of the new record */ + int nZero, /* Number of extra 0 bytes to append to data */ int appendBias, /* True if this is likely an append */ int seekResult /* Result of prior MovetoUnpacked() call */ ){ @@ -69284,7 +63597,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( return pCur->skipNext; } - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( (pCur->curFlags & BTCF_WriteFlag)!=0 && pBt->inTransaction==TRANS_WRITE && (pBt->btsFlags & BTS_READ_ONLY)==0 ); @@ -69295,7 +63608,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** keys with no associated data. If the cursor was opened expecting an ** intkey table, the caller should be inserting integer keys with a ** blob of associated data. */ - assert( (pX->pKey==0)==(pCur->pKeyInfo==0) ); + assert( (pKey==0)==(pCur->pKeyInfo==0) ); /* Save the positions of any other cursors open on this table. ** @@ -69308,46 +63621,42 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** doing any work. To avoid thwarting these optimizations, it is important ** not to clear the cursor here. */ - if( pCur->curFlags & BTCF_Multiple ){ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); - if( rc ) return rc; - } + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; if( pCur->pKeyInfo==0 ){ - assert( pX->pKey==0 ); /* If this is an insert into a table b-tree, invalidate any incrblob ** cursors open on the row being replaced */ - invalidateIncrblobCursors(p, pX->nKey, 0); + invalidateIncrblobCursors(p, nKey, 0); /* If the cursor is currently on the last row and we are appending a - ** new row onto the end, set the "loc" to avoid an unnecessary - ** btreeMoveto() call */ - if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0 - && pCur->info.nKey==pX->nKey-1 ){ - loc = -1; - }else if( loc==0 ){ - rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, appendBias, &loc); - if( rc ) return rc; + ** new row onto the end, set the "loc" to avoid an unnecessary btreeMoveto() + ** call */ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 + && pCur->info.nKey==nKey-1 ){ + loc = -1; } - }else if( loc==0 ){ - rc = btreeMoveto(pCur, pX->pKey, pX->nKey, appendBias, &loc); + } + + if( !loc ){ + rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); if( rc ) return rc; } assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) ); pPage = pCur->apPage[pCur->iPage]; - assert( pPage->intKey || pX->nKey>=0 ); + assert( pPage->intKey || nKey>=0 ); assert( pPage->leaf || !pPage->intKey ); TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", - pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, + pCur->pgnoRoot, nKey, nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit ); newCell = pBt->pTmpSpace; assert( newCell!=0 ); - rc = fillInCell(pPage, newCell, pX, &szNew); + rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); if( rc ) goto end_insert; - assert( szNew==pPage->xCellSize(pPage, newCell) ); + assert( szNew==cellSizePtr(pPage, newCell) ); assert( szNew <= MX_CELL_SIZE(pBt) ); idx = pCur->aiIdx[pCur->iPage]; if( loc==0 ){ @@ -69371,7 +63680,6 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( assert( pPage->leaf ); } insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); - assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); /* If no error has occurred and pPage has an overflow cell, call balance() @@ -69395,8 +63703,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** row without seeking the cursor. This can be a big performance boost. */ pCur->info.nSize = 0; - if( pPage->nOverflow ){ - assert( rc==SQLITE_OK ); + if( rc==SQLITE_OK && pPage->nOverflow ){ pCur->curFlags &= ~(BTCF_ValidNKey); rc = balance(pCur); @@ -69414,23 +63721,10 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( } /* -** Delete the entry that the cursor is pointing to. -** -** If the BTREE_SAVEPOSITION bit of the flags parameter is zero, then -** the cursor is left pointing at an arbitrary location after the delete. -** But if that bit is set, then the cursor is left in a state such that -** the next call to BtreeNext() or BtreePrev() moves it to the same row -** as it would have been on if the call to BtreeDelete() had been omitted. -** -** The BTREE_AUXDELETE bit of flags indicates that is one of several deletes -** associated with a single table entry and its indexes. Only one of those -** deletes is considered the "primary" delete. The primary delete occurs -** on a cursor that is not a BTREE_FORDELETE cursor. All but one delete -** operation on non-FORDELETE cursors is tagged with the AUXDELETE flag. -** The BTREE_AUXDELETE bit is a hint that is not used by this implementation, -** but which might be used by alternative storage engines. +** Delete the entry that the cursor is pointing to. The cursor +** is left pointing at an arbitrary location. */ -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ Btree *p = pCur->pBtree; BtShared *pBt = p->pBt; int rc; /* Return code */ @@ -69439,46 +63733,25 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ int iCellIdx; /* Index of cell to delete */ int iCellDepth; /* Depth of node containing pCell */ u16 szCell; /* Size of the cell being deleted */ - int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ - u8 bPreserve = flags & BTREE_SAVEPOSITION; /* Keep cursor valid */ - assert( cursorOwnsBtShared(pCur) ); + assert( cursorHoldsMutex(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( pCur->curFlags & BTCF_WriteFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); - assert( pCur->eState==CURSOR_VALID ); - assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); + + if( NEVER(pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell) + || NEVER(pCur->eState!=CURSOR_VALID) + ){ + return SQLITE_ERROR; /* Something has gone awry. */ + } iCellDepth = pCur->iPage; iCellIdx = pCur->aiIdx[iCellDepth]; pPage = pCur->apPage[iCellDepth]; pCell = findCell(pPage, iCellIdx); - /* If the bPreserve flag is set to true, then the cursor position must - ** be preserved following this delete operation. If the current delete - ** will cause a b-tree rebalance, then this is done by saving the cursor - ** key and leaving the cursor in CURSOR_REQUIRESEEK state before - ** returning. - ** - ** Or, if the current delete will not cause a rebalance, then the cursor - ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately - ** before or after the deleted entry. In this case set bSkipnext to true. */ - if( bPreserve ){ - if( !pPage->leaf - || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) - ){ - /* A b-tree rebalance will be required after deleting this entry. - ** Save the cursor key. */ - rc = saveCursorKey(pCur); - if( rc ) return rc; - }else{ - bSkipnext = 1; - } - } - /* If the page containing the entry to delete is not a leaf page, move ** the cursor to the largest entry in the tree that is smaller than ** the entry being deleted. This cell will replace the cell being deleted @@ -69493,11 +63766,12 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ } /* Save the positions of any other cursors open on this table before - ** making any modifications. */ - if( pCur->curFlags & BTCF_Multiple ){ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); - if( rc ) return rc; - } + ** making any modifications. Make the page containing the entry to be + ** deleted writable. Then free any overflow pages associated with the + ** entry and finally remove the cell itself from within the page. + */ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; /* If this is a delete operation to remove a row from a table b-tree, ** invalidate any incrblob cursors open on the row being deleted. */ @@ -69505,9 +63779,6 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ invalidateIncrblobCursors(p, pCur->info.nKey, 0); } - /* Make the page containing the entry to be deleted writable. Then free any - ** overflow pages associated with the entry and finally remove the cell - ** itself from within the page. */ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; rc = clearCell(pPage, pCell, &szCell); @@ -69526,15 +63797,12 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ unsigned char *pTmp; pCell = findCell(pLeaf, pLeaf->nCell-1); - if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; - nCell = pLeaf->xCellSize(pLeaf, pCell); + nCell = cellSizePtr(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); - if( rc==SQLITE_OK ){ - insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); - } + insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); if( rc ) return rc; } @@ -69563,23 +63831,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ } if( rc==SQLITE_OK ){ - if( bSkipnext ){ - assert( bPreserve && (pCur->iPage==iCellDepth || CORRUPT_DB) ); - assert( pPage==pCur->apPage[pCur->iPage] || CORRUPT_DB ); - assert( (pPage->nCell>0 || CORRUPT_DB) && iCellIdx<=pPage->nCell ); - pCur->eState = CURSOR_SKIPNEXT; - if( iCellIdx>=pPage->nCell ){ - pCur->skipNext = -1; - pCur->aiIdx[iCellDepth] = pPage->nCell-1; - }else{ - pCur->skipNext = 1; - } - }else{ - rc = moveToRoot(pCur); - if( bPreserve ){ - pCur->eState = CURSOR_REQUIRESEEK; - } - } + moveToRoot(pCur); } return rc; } @@ -69637,8 +63889,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ pgnoRoot++; } - assert( pgnoRoot>=3 || CORRUPT_DB ); - testcase( pgnoRoot<3 ); + assert( pgnoRoot>=3 ); /* Allocate a page. The page that currently resides at pgnoRoot will ** be moved to the allocated page (unless the allocated page happens @@ -69767,7 +64018,7 @@ static int clearDatabasePage( if( pgno>btreePagecount(pBt) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage, 0, 0); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; if( pPage->bBusy ){ rc = SQLITE_CORRUPT_BKPT; @@ -69788,8 +64039,7 @@ static int clearDatabasePage( rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); if( rc ) goto cleardatabasepage_out; }else if( pnChange ){ - assert( pPage->intKey || CORRUPT_DB ); - testcase( !pPage->intKey ); + assert( pPage->intKey ); *pnChange += pPage->nCell; } if( freePageFlag ){ @@ -69886,14 +64136,6 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return SQLITE_LOCKED_SHAREDCACHE; } - /* - ** It is illegal to drop the sqlite_master table on page 1. But again, - ** this error is caught long before reaching this point. - */ - if( NEVER(iTable<2) ){ - return SQLITE_CORRUPT_BKPT; - } - rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); @@ -69904,67 +64146,76 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ *piMoved = 0; + if( iTable>1 ){ #ifdef SQLITE_OMIT_AUTOVACUUM - freePage(pPage, &rc); - releasePage(pPage); + freePage(pPage, &rc); + releasePage(pPage); #else - if( pBt->autoVacuum ){ - Pgno maxRootPgno; - sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); + if( pBt->autoVacuum ){ + Pgno maxRootPgno; + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); - if( iTable==maxRootPgno ){ - /* If the table being dropped is the table with the largest root-page - ** number in the database, put the root page on the free list. - */ - freePage(pPage, &rc); - releasePage(pPage); - if( rc!=SQLITE_OK ){ - return rc; + if( iTable==maxRootPgno ){ + /* If the table being dropped is the table with the largest root-page + ** number in the database, put the root page on the free list. + */ + freePage(pPage, &rc); + releasePage(pPage); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + /* The table being dropped does not have the largest root-page + ** number in the database. So move the page that does into the + ** gap left by the deleted root-page. + */ + MemPage *pMove; + releasePage(pPage); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + pMove = 0; + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + freePage(pMove, &rc); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + *piMoved = maxRootPgno; } - }else{ - /* The table being dropped does not have the largest root-page - ** number in the database. So move the page that does into the - ** gap left by the deleted root-page. + + /* Set the new 'max-root-page' value in the database header. This + ** is the old value less one, less one more if that happens to + ** be a root-page number, less one again if that is the + ** PENDING_BYTE_PAGE. */ - MemPage *pMove; - releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); - releasePage(pMove); - if( rc!=SQLITE_OK ){ - return rc; - } - pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); - freePage(pMove, &rc); - releasePage(pMove); - if( rc!=SQLITE_OK ){ - return rc; + maxRootPgno--; + while( maxRootPgno==PENDING_BYTE_PAGE(pBt) + || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ + maxRootPgno--; } - *piMoved = maxRootPgno; - } + assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); - /* Set the new 'max-root-page' value in the database header. This - ** is the old value less one, less one more if that happens to - ** be a root-page number, less one again if that is the - ** PENDING_BYTE_PAGE. - */ - maxRootPgno--; - while( maxRootPgno==PENDING_BYTE_PAGE(pBt) - || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ - maxRootPgno--; + rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); + }else{ + freePage(pPage, &rc); + releasePage(pPage); } - assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); - - rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); +#endif }else{ - freePage(pPage, &rc); + /* If sqlite3BtreeDropTable was called on page 1. + ** This really never should happen except in a corrupt + ** database. + */ + zeroPage(pPage, PTF_INTKEY|PTF_LEAF ); releasePage(pPage); } -#endif return rc; } SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ @@ -70143,6 +64394,7 @@ static void checkAppendMsg( ... ){ va_list ap; + char zBuf[200]; if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; @@ -70151,9 +64403,10 @@ static void checkAppendMsg( sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); } if( pCheck->zPfx ){ - sqlite3XPrintf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); + sqlite3_snprintf(sizeof(zBuf), zBuf, pCheck->zPfx, pCheck->v1, pCheck->v2); + sqlite3StrAccumAppendAll(&pCheck->errMsg, zBuf); } - sqlite3VXPrintf(&pCheck->errMsg, zFormat, ap); + sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); va_end(ap); if( pCheck->errMsg.accError==STRACCUM_NOMEM ){ pCheck->mallocFailed = 1; @@ -70257,7 +64510,7 @@ static void checkList( break; } if( checkRef(pCheck, iPage) ) break; - if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ + if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage) ){ checkAppendMsg(pCheck, "failed to get page %d", iPage); break; } @@ -70300,10 +64553,6 @@ static void checkList( #endif iPage = get4byte(pOvflData); sqlite3PagerUnref(pOvflPage); - - if( isFreeList && N<(iPage!=0) ){ - checkAppendMsg(pCheck, "free-page count in header is too small"); - } } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -70369,42 +64618,35 @@ static int btreeHeapPull(u32 *aHeap, u32 *pOut){ ** ** 1. Make sure that cells and freeblocks do not overlap ** but combine to completely cover the page. -** 2. Make sure integer cell keys are in order. -** 3. Check the integrity of overflow pages. -** 4. Recursively call checkTreePage on all children. -** 5. Verify that the depth of all children is the same. +** NO 2. Make sure cell keys are in order. +** NO 3. Make sure no key is less than or equal to zLowerBound. +** NO 4. Make sure no key is greater than or equal to zUpperBound. +** 5. Check the integrity of overflow pages. +** 6. Recursively call checkTreePage on all children. +** 7. Verify that the depth of all children is the same. +** 8. Make sure this page is at least 33% full or else it is +** the root of the tree. */ static int checkTreePage( IntegrityCk *pCheck, /* Context for the sanity check */ int iPage, /* Page number of the page to check */ - i64 *piMinKey, /* Write minimum integer primary key here */ - i64 maxKey /* Error if integer primary key greater than this */ + i64 *pnParentMinKey, + i64 *pnParentMaxKey ){ - MemPage *pPage = 0; /* The page being analyzed */ - int i; /* Loop counter */ - int rc; /* Result code from subroutine call */ - int depth = -1, d2; /* Depth of a subtree */ - int pgno; /* Page number */ - int nFrag; /* Number of fragmented bytes on the page */ - int hdr; /* Offset to the page header */ - int cellStart; /* Offset to the start of the cell pointer array */ - int nCell; /* Number of cells */ - int doCoverageCheck = 1; /* True if cell coverage checking should be done */ - int keyCanBeEqual = 1; /* True if IPK can be equal to maxKey - ** False if IPK must be strictly less than maxKey */ - u8 *data; /* Page content */ - u8 *pCell; /* Cell content */ - u8 *pCellIdx; /* Next element of the cell pointer array */ - BtShared *pBt; /* The BtShared object that owns pPage */ - u32 pc; /* Address of a cell */ - u32 usableSize; /* Usable size of the page */ - u32 contentOffset; /* Offset to the start of the cell content area */ - u32 *heap = 0; /* Min-heap used for checking cell coverage */ - u32 x, prev = 0; /* Next and previous entry on the min-heap */ + MemPage *pPage; + int i, rc, depth, d2, pgno, cnt; + int hdr, cellStart; + int nCell; + u8 *data; + BtShared *pBt; + int usableSize; + u32 *heap = 0; + u32 x, prev = 0; + i64 nMinKey = 0; + i64 nMaxKey = 0; const char *saved_zPfx = pCheck->zPfx; int saved_v1 = pCheck->v1; int saved_v2 = pCheck->v2; - u8 savedIsInit = 0; /* Check that the page exists */ @@ -70417,95 +64659,54 @@ static int checkTreePage( if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); + depth = -1; goto end_of_check; } /* Clear MemPage.isInit to make sure the corruption detection code in ** btreeInitPage() is executed. */ - savedIsInit = pPage->isInit; pPage->isInit = 0; if( (rc = btreeInitPage(pPage))!=0 ){ assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ checkAppendMsg(pCheck, "btreeInitPage() returns error code %d", rc); + releasePage(pPage); + depth = -1; goto end_of_check; } - data = pPage->aData; - hdr = pPage->hdrOffset; - /* Set up for cell analysis */ - pCheck->zPfx = "On tree page %d cell %d: "; - contentOffset = get2byteNotZero(&data[hdr+5]); - assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ - - /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the - ** number of cells on the page. */ - nCell = get2byte(&data[hdr+3]); - assert( pPage->nCell==nCell ); - - /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page - ** immediately follows the b-tree page header. */ - cellStart = hdr + 12 - 4*pPage->leaf; - assert( pPage->aCellIdx==&data[cellStart] ); - pCellIdx = &data[cellStart + 2*(nCell-1)]; - - if( !pPage->leaf ){ - /* Analyze the right-child page of internal pages */ - pgno = get4byte(&data[hdr+8]); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - pCheck->zPfx = "On page %d at right child: "; - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); - } -#endif - depth = checkTreePage(pCheck, pgno, &maxKey, maxKey); - keyCanBeEqual = 0; - }else{ - /* For leaf pages, the coverage check will occur in the same loop - ** as the other cell checks, so initialize the heap. */ - heap = pCheck->heap; - heap[0] = 0; - } - - /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte - ** integer offsets to the cell contents. */ - for(i=nCell-1; i>=0 && pCheck->mxErr; i--){ + /* Check out all the cells. + */ + depth = 0; + for(i=0; inCell && pCheck->mxErr; i++){ + u8 *pCell; + u32 sz; CellInfo info; - /* Check cell size */ + /* Check payload overflow pages + */ + pCheck->zPfx = "On tree page %d cell %d: "; + pCheck->v1 = iPage; pCheck->v2 = i; - assert( pCellIdx==&data[cellStart + i*2] ); - pc = get2byteAligned(pCellIdx); - pCellIdx -= 2; - if( pcusableSize-4 ){ - checkAppendMsg(pCheck, "Offset %d out of range %d..%d", - pc, contentOffset, usableSize-4); - doCoverageCheck = 0; - continue; - } - pCell = &data[pc]; - pPage->xParseCell(pPage, pCell, &info); - if( pc+info.nSize>usableSize ){ - checkAppendMsg(pCheck, "Extends off end of page"); - doCoverageCheck = 0; - continue; - } - - /* Check for integer primary key out of range */ + pCell = findCell(pPage,i); + btreeParseCellPtr(pPage, pCell, &info); + sz = info.nPayload; + /* For intKey pages, check that the keys are in order. + */ if( pPage->intKey ){ - if( keyCanBeEqual ? (info.nKey > maxKey) : (info.nKey >= maxKey) ){ - checkAppendMsg(pCheck, "Rowid %lld out of order", info.nKey); + if( i==0 ){ + nMinKey = nMaxKey = info.nKey; + }else if( info.nKey <= nMaxKey ){ + checkAppendMsg(pCheck, + "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); } - maxKey = info.nKey; + nMaxKey = info.nKey; } - - /* Check the content overflow list */ - if( info.nPayload>info.nLocal ){ - int nPage; /* Number of pages on the overflow chain */ - Pgno pgnoOvfl; /* First page of the overflow chain */ - assert( pc + info.nSize - 4 <= usableSize ); - nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4); - pgnoOvfl = get4byte(&pCell[info.nSize - 4]); + if( (sz>info.nLocal) + && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize]) + ){ + int nPage = (sz - info.nLocal + usableSize - 5)/(usableSize - 4); + Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage); @@ -70514,57 +64715,118 @@ static int checkTreePage( checkList(pCheck, 0, pgnoOvfl, nPage); } + /* Check sanity of left child page. + */ if( !pPage->leaf ){ - /* Check sanity of left child page for internal pages */ pgno = get4byte(pCell); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif - d2 = checkTreePage(pCheck, pgno, &maxKey, maxKey); - keyCanBeEqual = 0; - if( d2!=depth ){ + d2 = checkTreePage(pCheck, pgno, &nMinKey, i==0?NULL:&nMaxKey); + if( i>0 && d2!=depth ){ checkAppendMsg(pCheck, "Child page depth differs"); - depth = d2; } - }else{ - /* Populate the coverage-checking heap for leaf pages */ - btreeHeapInsert(heap, (pc<<16)|(pc+info.nSize-1)); + depth = d2; + } + } + + if( !pPage->leaf ){ + pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + pCheck->zPfx = "On page %d at right child: "; + pCheck->v1 = iPage; +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); + } +#endif + checkTreePage(pCheck, pgno, NULL, !pPage->nCell?NULL:&nMaxKey); + } + + /* For intKey leaf pages, check that the min/max keys are in order + ** with any left/parent/right pages. + */ + pCheck->zPfx = "Page %d: "; + pCheck->v1 = iPage; + if( pPage->leaf && pPage->intKey ){ + /* if we are a left child page */ + if( pnParentMinKey ){ + /* if we are the left most child page */ + if( !pnParentMaxKey ){ + if( nMaxKey > *pnParentMinKey ){ + checkAppendMsg(pCheck, + "Rowid %lld out of order (max larger than parent min of %lld)", + nMaxKey, *pnParentMinKey); + } + }else{ + if( nMinKey <= *pnParentMinKey ){ + checkAppendMsg(pCheck, + "Rowid %lld out of order (min less than parent min of %lld)", + nMinKey, *pnParentMinKey); + } + if( nMaxKey > *pnParentMaxKey ){ + checkAppendMsg(pCheck, + "Rowid %lld out of order (max larger than parent max of %lld)", + nMaxKey, *pnParentMaxKey); + } + *pnParentMinKey = nMaxKey; + } + /* else if we're a right child page */ + } else if( pnParentMaxKey ){ + if( nMinKey <= *pnParentMaxKey ){ + checkAppendMsg(pCheck, + "Rowid %lld out of order (min less than parent max of %lld)", + nMinKey, *pnParentMaxKey); + } } } - *piMinKey = maxKey; /* Check for complete coverage of the page */ + data = pPage->aData; + hdr = pPage->hdrOffset; + heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); pCheck->zPfx = 0; - if( doCoverageCheck && pCheck->mxErr>0 ){ - /* For leaf pages, the min-heap has already been initialized and the - ** cells have already been inserted. But for internal pages, that has - ** not yet been done, so do it now */ - if( !pPage->leaf ){ - heap = pCheck->heap; - heap[0] = 0; - for(i=nCell-1; i>=0; i--){ - u32 size; - pc = get2byteAligned(&data[cellStart+i*2]); - size = pPage->xCellSize(pPage, &data[pc]); + if( heap==0 ){ + pCheck->mallocFailed = 1; + }else{ + int contentOffset = get2byteNotZero(&data[hdr+5]); + assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ + heap[0] = 0; + btreeHeapInsert(heap, contentOffset-1); + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + nCell = get2byte(&data[hdr+3]); + /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page + ** immediately follows the b-tree page header. */ + cellStart = hdr + 12 - 4*pPage->leaf; + /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte + ** integer offsets to the cell contents. */ + for(i=0; i=usableSize ){ + pCheck->zPfx = 0; + checkAppendMsg(pCheck, + "Corruption detected in cell %d on page %d",i,iPage); + }else{ btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); } } - /* Add the freeblocks to the min-heap - ** - ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header + /* EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header ** is the offset of the first freeblock, or zero if there are no - ** freeblocks on the page. - */ + ** freeblocks on the page. */ i = get2byte(&data[hdr+1]); while( i>0 ){ int size, j; - assert( (u32)i<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( i<=usableSize-4 ); /* Enforced by btreeInitPage() */ size = get2byte(&data[i+2]); - assert( (u32)(i+size)<=usableSize ); /* Enforced by btreeInitPage() */ - btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1)); + assert( i+size<=usableSize ); /* Enforced by btreeInitPage() */ + btreeHeapInsert(heap, (i<<16)|(i+size-1)); /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a ** big-endian integer which is the offset in the b-tree page of the next ** freeblock in the chain, or zero if the freeblock is the last on the @@ -70573,50 +64835,39 @@ static int checkTreePage( /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of ** increasing offset. */ assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ - assert( (u32)j<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( j<=usableSize-4 ); /* Enforced by btreeInitPage() */ i = j; } - /* Analyze the min-heap looking for overlap between cells and/or - ** freeblocks, and counting the number of untracked bytes in nFrag. - ** - ** Each min-heap entry is of the form: (start_address<<16)|end_address. - ** There is an implied first entry the covers the page header, the cell - ** pointer index, and the gap between the cell pointer index and the start - ** of cell content. - ** - ** The loop below pulls entries from the min-heap in order and compares - ** the start_address against the previous end_address. If there is an - ** overlap, that means bytes are used multiple times. If there is a gap, - ** that gap is added to the fragmentation count. - */ - nFrag = 0; - prev = contentOffset - 1; /* Implied first min-heap entry */ + cnt = 0; + assert( heap[0]>0 ); + assert( (heap[1]>>16)==0 ); + btreeHeapPull(heap,&prev); while( btreeHeapPull(heap,&x) ){ - if( (prev&0xffff)>=(x>>16) ){ + if( (prev&0xffff)+1>(x>>16) ){ checkAppendMsg(pCheck, "Multiple uses for byte %u of page %d", x>>16, iPage); break; }else{ - nFrag += (x>>16) - (prev&0xffff) - 1; + cnt += (x>>16) - (prev&0xffff) - 1; prev = x; } } - nFrag += usableSize - (prev&0xffff) - 1; + cnt += usableSize - (prev&0xffff) - 1; /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments ** is stored in the fifth field of the b-tree page header. ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the ** number of fragmented free bytes within the cell content area. */ - if( heap[0]==0 && nFrag!=data[hdr+7] ){ + if( heap[0]==0 && cnt!=data[hdr+7] ){ checkAppendMsg(pCheck, "Fragmentation of %d bytes reported as %d on page %d", - nFrag, data[hdr+7], iPage); + cnt, data[hdr+7], iPage); } } + sqlite3PageFree(heap); + releasePage(pPage); end_of_check: - if( !doCoverageCheck ) pPage->isInit = savedIsInit; - releasePage(pPage); pCheck->zPfx = saved_zPfx; pCheck->v1 = saved_v1; pCheck->v2 = saved_v2; @@ -70646,16 +64897,14 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( int *pnErr /* Write number of errors seen to this variable */ ){ Pgno i; + int nRef; IntegrityCk sCheck; BtShared *pBt = p->pBt; - int savedDbFlags = pBt->db->flags; char zErr[100]; - VVA_ONLY( int nRef ); sqlite3BtreeEnter(p); assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); - VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); - assert( nRef>=0 ); + nRef = sqlite3PagerRefcount(pBt->pPager); sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; sCheck.nPage = btreePagecount(sCheck.pBt); @@ -70665,27 +64914,21 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( sCheck.zPfx = 0; sCheck.v1 = 0; sCheck.v2 = 0; - sCheck.aPgRef = 0; - sCheck.heap = 0; - sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); - sCheck.errMsg.printfFlags = SQLITE_PRINTF_INTERNAL; + *pnErr = 0; if( sCheck.nPage==0 ){ - goto integrity_ck_cleanup; + sqlite3BtreeLeave(p); + return 0; } sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); if( !sCheck.aPgRef ){ - sCheck.mallocFailed = 1; - goto integrity_ck_cleanup; - } - sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); - if( sCheck.heap==0 ){ - sCheck.mallocFailed = 1; - goto integrity_ck_cleanup; + *pnErr = 1; + sqlite3BtreeLeave(p); + return 0; } - i = PENDING_BYTE_PAGE(pBt); if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); + sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); /* Check the integrity of the freelist */ @@ -70696,19 +64939,17 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( /* Check all the tables. */ - testcase( pBt->db->flags & SQLITE_CellSizeCk ); - pBt->db->flags &= ~SQLITE_CellSizeCk; for(i=0; (int)iautoVacuum && aRoot[i]>1 ){ checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); } #endif - checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + sCheck.zPfx = "List of tree roots: "; + checkTreePage(&sCheck, aRoot[i], NULL, NULL); + sCheck.zPfx = 0; } - pBt->db->flags = savedDbFlags; /* Make sure every page in the file is referenced */ @@ -70732,20 +64973,28 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( #endif } + /* Make sure this analysis did not leave any unref() pages. + ** This is an internal consistency check; an integrity check + ** of the integrity check. + */ + if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){ + checkAppendMsg(&sCheck, + "Outstanding page count goes from %d to %d during this analysis", + nRef, sqlite3PagerRefcount(pBt->pPager) + ); + } + /* Clean up and report errors. */ -integrity_ck_cleanup: - sqlite3PageFree(sCheck.heap); + sqlite3BtreeLeave(p); sqlite3_free(sCheck.aPgRef); if( sCheck.mallocFailed ){ sqlite3StrAccumReset(&sCheck.errMsg); - sCheck.nErr++; + *pnErr = sCheck.nErr+1; + return 0; } *pnErr = sCheck.nErr; if( sCheck.nErr==0 ) sqlite3StrAccumReset(&sCheck.errMsg); - /* Make sure this analysis did not leave any unref() pages. */ - assert( nRef==sqlite3PagerRefcount(pBt->pPager) ); - sqlite3BtreeLeave(p); return sqlite3StrAccumFinish(&sCheck.errMsg); } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -70908,7 +65157,7 @@ SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ */ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ int rc; - assert( cursorOwnsBtShared(pCsr) ); + assert( cursorHoldsMutex(pCsr) ); assert( sqlite3_mutex_held(pCsr->pBtree->db->mutex) ); assert( pCsr->curFlags & BTCF_Incrblob ); @@ -70956,7 +65205,6 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void */ SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ pCur->curFlags |= BTCF_Incrblob; - pCur->pBtree->hasIncrblobCur = 1; } #endif @@ -70996,6 +65244,15 @@ SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ return rc; } +/* +** set the mask of hint flags for cursor pCsr. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *pCsr, unsigned int mask){ + assert( mask==BTREE_BULKLOAD || mask==BTREE_SEEK_EQ || mask==0 ); + pCsr->hints = mask; +} + +#ifdef SQLITE_DEBUG /* ** Return true if the cursor has a hint specified. This routine is ** only used from within assert() statements @@ -71003,6 +65260,7 @@ SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor *pCsr, unsigned int mask){ return (pCsr->hints & mask)!=0; } +#endif /* ** Return true if the given Btree is read-only. @@ -71016,25 +65274,6 @@ SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ */ SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } -#if !defined(SQLITE_OMIT_SHARED_CACHE) -/* -** Return true if the Btree passed as the only argument is sharable. -*/ -SQLITE_PRIVATE int sqlite3BtreeSharable(Btree *p){ - return p->sharable; -} - -/* -** Return the number of connections to the BtShared object accessed by -** the Btree handle passed as the only argument. For private caches -** this is always 1. For shared caches it may be 1 or greater. -*/ -SQLITE_PRIVATE int sqlite3BtreeConnectionCount(Btree *p){ - testcase( p->sharable ); - return p->pBt->nRef; -} -#endif - /************** End of btree.c ***********************************************/ /************** Begin file backup.c ******************************************/ /* @@ -71051,8 +65290,6 @@ SQLITE_PRIVATE int sqlite3BtreeConnectionCount(Btree *p){ ** This file contains the implementation of the sqlite3_backup_XXX() ** API functions and the related features. */ -/* #include "sqliteInt.h" */ -/* #include "btreeInt.h" */ /* ** Structure allocated for each backup operation. @@ -71122,16 +65359,22 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ int i = sqlite3FindDbName(pDb, zDb); if( i==1 ){ - Parse sParse; + Parse *pParse; int rc = 0; - memset(&sParse, 0, sizeof(sParse)); - sParse.db = pDb; - if( sqlite3OpenTempDatabase(&sParse) ){ - sqlite3ErrorWithMsg(pErrorDb, sParse.rc, "%s", sParse.zErrMsg); - rc = SQLITE_ERROR; + pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse)); + if( pParse==0 ){ + sqlite3ErrorWithMsg(pErrorDb, SQLITE_NOMEM, "out of memory"); + rc = SQLITE_NOMEM; + }else{ + pParse->db = pDb; + if( sqlite3OpenTempDatabase(pParse) ){ + sqlite3ErrorWithMsg(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); + rc = SQLITE_ERROR; + } + sqlite3DbFree(pErrorDb, pParse->zErrMsg); + sqlite3ParserReset(pParse); + sqlite3StackFree(pErrorDb, pParse); } - sqlite3DbFree(pErrorDb, sParse.zErrMsg); - sqlite3ParserReset(&sParse); if( rc ){ return 0; } @@ -71177,7 +65420,7 @@ static int checkReadTransaction(sqlite3 *db, Btree *p){ ** If an error occurs, NULL is returned and an error code and error message ** stored in database handle pDestDb. */ -SQLITE_API sqlite3_backup *sqlite3_backup_init( +SQLITE_API sqlite3_backup *SQLITE_STDCALL sqlite3_backup_init( sqlite3* pDestDb, /* Database to write to */ const char *zDestDb, /* Name of database within pDestDb */ sqlite3* pSrcDb, /* Database connection to read from */ @@ -71215,7 +65458,7 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init( ** sqlite3_backup_finish(). */ p = (sqlite3_backup *)sqlite3MallocZero(sizeof(sqlite3_backup)); if( !p ){ - sqlite3Error(pDestDb, SQLITE_NOMEM_BKPT); + sqlite3Error(pDestDb, SQLITE_NOMEM); } } @@ -71229,6 +65472,7 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init( p->isAttached = 0; if( 0==p->pSrc || 0==p->pDest + || setDestPgsz(p)==SQLITE_NOMEM || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK ){ /* One (or both) of the named databases did not exist or an OOM @@ -71325,7 +65569,7 @@ static int backupOnePage( DbPage *pDestPg = 0; Pgno iDest = (Pgno)(iOff/nDestPgsz)+1; if( iDest==PENDING_BYTE_PAGE(p->pDest->pBt) ) continue; - if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg, 0)) + if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg)) && SQLITE_OK==(rc = sqlite3PagerWrite(pDestPg)) ){ const u8 *zIn = &zSrcData[iOff%nSrcPgsz]; @@ -71384,7 +65628,7 @@ static void attachBackupObject(sqlite3_backup *p){ /* ** Copy nPage pages from the source b-tree to the destination. */ -SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ +SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ int rc; int destMode; /* Destination journal mode */ int pgszSrc = 0; /* Source page size */ @@ -71416,6 +65660,14 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = SQLITE_OK; } + /* Lock the destination database, if it is not locked already. */ + if( SQLITE_OK==rc && p->bDestLocked==0 + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) + ){ + p->bDestLocked = 1; + sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); + } + /* If there is no open read-transaction on the source database, open ** one now. If a transaction is opened here, then it will be closed ** before this function exits. @@ -71425,24 +65677,6 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ bCloseTrans = 1; } - /* If the destination database has not yet been locked (i.e. if this - ** is the first call to backup_step() for the current backup operation), - ** try to set its page size to the same as the source database. This - ** is especially important on ZipVFS systems, as in that case it is - ** not possible to create a database file that uses one page size by - ** writing to it with another. */ - if( p->bDestLocked==0 && rc==SQLITE_OK && setDestPgsz(p)==SQLITE_NOMEM ){ - rc = SQLITE_NOMEM; - } - - /* Lock the destination database, if it is not locked already. */ - if( SQLITE_OK==rc && p->bDestLocked==0 - && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) - ){ - p->bDestLocked = 1; - sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); - } - /* Do not allow backup if the destination database is in WAL mode ** and the page sizes are different between source and destination */ pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); @@ -71461,7 +65695,8 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ const Pgno iSrcPg = p->iNext; /* Source page number */ if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg,PAGER_GET_READONLY); + rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, + PAGER_GET_READONLY); if( rc==SQLITE_OK ){ rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); sqlite3PagerUnref(pSrcPg); @@ -71561,7 +65796,7 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){ if( iPg!=PENDING_BYTE_PAGE(p->pDest->pBt) ){ DbPage *pPg; - rc = sqlite3PagerGet(pDestPager, iPg, &pPg, 0); + rc = sqlite3PagerGet(pDestPager, iPg, &pPg); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg); sqlite3PagerUnref(pPg); @@ -71581,7 +65816,7 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ ){ PgHdr *pSrcPg = 0; const Pgno iSrcPg = (Pgno)((iOff/pgszSrc)+1); - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg, 0); + rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); if( rc==SQLITE_OK ){ u8 *zData = sqlite3PagerGetData(pSrcPg); rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff); @@ -71623,7 +65858,7 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ } if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; } p->rc = rc; } @@ -71638,7 +65873,7 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ /* ** Release all resources associated with an sqlite3_backup* handle. */ -SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ +SQLITE_API int SQLITE_STDCALL sqlite3_backup_finish(sqlite3_backup *p){ sqlite3_backup **pp; /* Ptr to head of pagers backup list */ sqlite3 *pSrcDb; /* Source database connection */ int rc; /* Value to return */ @@ -71690,7 +65925,7 @@ SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ ** Return the number of pages still to be backed up as of the most recent ** call to sqlite3_backup_step(). */ -SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p){ +SQLITE_API int SQLITE_STDCALL sqlite3_backup_remaining(sqlite3_backup *p){ #ifdef SQLITE_ENABLE_API_ARMOR if( p==0 ){ (void)SQLITE_MISUSE_BKPT; @@ -71704,7 +65939,7 @@ SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p){ ** Return the total number of pages in the source database as of the most ** recent call to sqlite3_backup_step(). */ -SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p){ +SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p){ #ifdef SQLITE_ENABLE_API_ARMOR if( p==0 ){ (void)SQLITE_MISUSE_BKPT; @@ -71726,13 +65961,9 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p){ ** corresponding to the source database is held when this function is ** called. */ -static SQLITE_NOINLINE void backupUpdate( - sqlite3_backup *p, - Pgno iPage, - const u8 *aData -){ - assert( p!=0 ); - do{ +SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ + sqlite3_backup *p; /* Iterator variable */ + for(p=pBackup; p; p=p->pNext){ assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); if( !isFatalError(p->rc) && iPageiNext ){ /* The backup process p has already copied page iPage. But now it @@ -71749,10 +65980,7 @@ static SQLITE_NOINLINE void backupUpdate( p->rc = rc; } } - }while( (p = p->pNext)!=0 ); -} -SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ - if( pBackup ) backupUpdate(pBackup, iPage, aData); + } } /* @@ -71810,19 +66038,15 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ b.pDest = pTo; b.iNext = 1; -#ifdef SQLITE_HAS_CODEC - sqlite3PagerAlignReserve(sqlite3BtreePager(pTo), sqlite3BtreePager(pFrom)); -#endif - /* 0x7FFFFFFF is the hard limit for the number of pages in a database ** file. By passing this as the number of pages to copy to ** sqlite3_backup_step(), we can guarantee that the copy finishes ** within a single call (unless an error occurs). The assert() statement ** checks this assumption - (p->rc) should be set to either SQLITE_DONE - ** or an error code. */ + ** or an error code. + */ sqlite3_backup_step(&b, 0x7FFFFFFF); assert( b.rc!=SQLITE_OK ); - rc = sqlite3_backup_finish(&b); if( rc==SQLITE_OK ){ pTo->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; @@ -71857,8 +66081,6 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ ** only within the VDBE. Interface routines refer to a Mem using the ** name sqlite_value */ -/* #include "sqliteInt.h" */ -/* #include "vdbeInt.h" */ #ifdef SQLITE_DEBUG /* @@ -71958,7 +66180,6 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ assert( sqlite3VdbeCheckMemInvariants(pMem) ); assert( (pMem->flags&MEM_RowSet)==0 ); - testcase( pMem->db==0 ); /* If the bPreserve flag is set to true, then the memory cell must already ** contain a valid string or blob value. */ @@ -71980,7 +66201,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPre sqlite3VdbeMemSetNull(pMem); pMem->z = 0; pMem->szMalloc = 0; - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; }else{ pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); } @@ -72031,18 +66252,18 @@ SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ ** Return SQLITE_OK on success or SQLITE_NOMEM if malloc fails. */ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ + int f; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( (pMem->flags&MEM_RowSet)==0 ); - if( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ){ - if( ExpandBlob(pMem) ) return SQLITE_NOMEM; - if( pMem->szMalloc==0 || pMem->z!=pMem->zMalloc ){ - if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ - return SQLITE_NOMEM_BKPT; - } - pMem->z[pMem->n] = 0; - pMem->z[pMem->n+1] = 0; - pMem->flags |= MEM_Term; + ExpandBlob(pMem); + f = pMem->flags; + if( (f&(MEM_Str|MEM_Blob)) && (pMem->szMalloc==0 || pMem->z!=pMem->zMalloc) ){ + if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ + return SQLITE_NOMEM; } + pMem->z[pMem->n] = 0; + pMem->z[pMem->n+1] = 0; + pMem->flags |= MEM_Term; } pMem->flags &= ~MEM_Ephem; #ifdef SQLITE_DEBUG @@ -72058,24 +66279,25 @@ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ */ #ifndef SQLITE_OMIT_INCRBLOB SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ - int nByte; - assert( pMem->flags & MEM_Zero ); - assert( pMem->flags&MEM_Blob ); - assert( (pMem->flags&MEM_RowSet)==0 ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + if( pMem->flags & MEM_Zero ){ + int nByte; + assert( pMem->flags&MEM_Blob ); + assert( (pMem->flags&MEM_RowSet)==0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - /* Set nByte to the number of bytes required to store the expanded blob. */ - nByte = pMem->n + pMem->u.nZero; - if( nByte<=0 ){ - nByte = 1; - } - if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ - return SQLITE_NOMEM_BKPT; - } + /* Set nByte to the number of bytes required to store the expanded blob. */ + nByte = pMem->n + pMem->u.nZero; + if( nByte<=0 ){ + nByte = 1; + } + if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ + return SQLITE_NOMEM; + } - memset(&pMem->z[pMem->n], 0, pMem->u.nZero); - pMem->n += pMem->u.nZero; - pMem->flags &= ~(MEM_Zero|MEM_Term); + memset(&pMem->z[pMem->n], 0, pMem->u.nZero); + pMem->n += pMem->u.nZero; + pMem->flags &= ~(MEM_Zero|MEM_Term); + } return SQLITE_OK; } #endif @@ -72086,7 +66308,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ */ static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; @@ -72135,8 +66357,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ if( sqlite3VdbeMemClearAndResize(pMem, nByte) ){ - pMem->enc = 0; - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 @@ -72417,7 +66638,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ } } assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); - pMem->flags &= ~(MEM_Str|MEM_Blob|MEM_Zero); + pMem->flags &= ~(MEM_Str|MEM_Blob); return SQLITE_OK; } @@ -72431,11 +66652,11 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ if( pMem->flags & MEM_Null ) return; switch( aff ){ - case SQLITE_AFF_BLOB: { /* Really a cast to BLOB */ + case SQLITE_AFF_NONE: { /* Really a cast to BLOB */ if( (pMem->flags & MEM_Blob)==0 ){ sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); - if( pMem->flags & MEM_Str ) MemSetTypeFlag(pMem, MEM_Blob); + MemSetTypeFlag(pMem, MEM_Blob); }else{ pMem->flags &= ~(MEM_TypeMask&~MEM_Blob); } @@ -72562,7 +66783,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem *pMem){ assert( db!=0 ); assert( (pMem->flags & MEM_RowSet)==0 ); sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = sqlite3DbMallocRawNN(db, 64); + pMem->zMalloc = sqlite3DbMallocRaw(db, 64); if( db->mallocFailed ){ pMem->flags = MEM_Null; pMem->szMalloc = 0; @@ -72603,7 +66824,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ int i; Mem *pX; - for(i=0, pX=pVdbe->aMem; inMem; i++, pX++){ + for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ if( pX->pScopyFrom==pMem ){ pX->flags |= MEM_Undefined; pX->pScopyFrom = 0; @@ -72613,6 +66834,10 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ } #endif /* SQLITE_DEBUG */ +/* +** Size of struct Mem not including the Mem.zMalloc member. +*/ +#define MEMCELLSIZE offsetof(Mem,zMalloc) /* ** Make an shallow copy of pFrom into pTo. Prior contents of @@ -72620,15 +66845,10 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ ** pFrom->z is used, then pTo->z points to the same thing as pFrom->z ** and flags gets srcType (either MEM_Ephem or MEM_Static). */ -static SQLITE_NOINLINE void vdbeClrCopy(Mem *pTo, const Mem *pFrom, int eType){ - vdbeMemClearExternAndSetNull(pTo); - assert( !VdbeMemDynamic(pTo) ); - sqlite3VdbeMemShallowCopy(pTo, pFrom, eType); -} SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( (pFrom->flags & MEM_RowSet)==0 ); assert( pTo->db==pFrom->db ); - if( VdbeMemDynamic(pTo) ){ vdbeClrCopy(pTo,pFrom,srcType); return; } + if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); if( (pFrom->flags&MEM_Static)==0 ){ pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); @@ -72644,6 +66864,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int sr SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; + assert( pTo->db==pFrom->db ); assert( (pFrom->flags & MEM_RowSet)==0 ); if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); @@ -72743,7 +66964,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( testcase( nAlloc==31 ); testcase( nAlloc==32 ); if( sqlite3VdbeMemClearAndResize(pMem, MAX(nAlloc,32)) ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } memcpy(pMem->z, z, nAlloc); }else if( xDel==SQLITE_DYNAMIC ){ @@ -72763,7 +66984,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( #ifndef SQLITE_OMIT_UTF16 if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } #endif @@ -72790,32 +67011,6 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( ** If this routine fails for any reason (malloc returns NULL or unable ** to read from the disk) then the pMem is left in an inconsistent state. */ -static SQLITE_NOINLINE int vdbeMemFromBtreeResize( - BtCursor *pCur, /* Cursor pointing at record to retrieve. */ - u32 offset, /* Offset from the start of data to return bytes from. */ - u32 amt, /* Number of bytes to return. */ - int key, /* If true, retrieve from the btree key, not data. */ - Mem *pMem /* OUT: Return data in this Mem structure. */ -){ - int rc; - pMem->flags = MEM_Null; - if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){ - if( key ){ - rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); - }else{ - rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); - } - if( rc==SQLITE_OK ){ - pMem->z[amt] = 0; - pMem->z[amt+1] = 0; - pMem->flags = MEM_Blob|MEM_Term; - pMem->n = (int)amt; - }else{ - sqlite3VdbeMemRelease(pMem); - } - } - return rc; -} SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( BtCursor *pCur, /* Cursor pointing at record to retrieve. */ u32 offset, /* Offset from the start of data to return bytes from. */ @@ -72833,7 +67028,11 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() ** that both the BtShared and database handle mutexes are held. */ assert( (pMem->flags & MEM_RowSet)==0 ); - zData = (char *)sqlite3BtreePayloadFetch(pCur, &available); + if( key ){ + zData = (char *)sqlite3BtreeKeyFetch(pCur, &available); + }else{ + zData = (char *)sqlite3BtreeDataFetch(pCur, &available); + } assert( zData!=0 ); if( offset+amt<=available ){ @@ -72841,7 +67040,22 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( pMem->flags = MEM_Blob|MEM_Ephem; pMem->n = (int)amt; }else{ - rc = vdbeMemFromBtreeResize(pCur, offset, amt, key, pMem); + pMem->flags = MEM_Null; + if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){ + if( key ){ + rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); + }else{ + rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); + } + if( rc==SQLITE_OK ){ + pMem->z[amt] = 0; + pMem->z[amt+1] = 0; + pMem->flags = MEM_Blob|MEM_Term; + pMem->n = (int)amt; + }else{ + sqlite3VdbeMemRelease(pMem); + } + } } return rc; @@ -72860,6 +67074,9 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ assert( (pVal->flags & (MEM_Null))==0 ); if( pVal->flags & (MEM_Blob|MEM_Str) ){ pVal->flags |= MEM_Str; + if( pVal->flags & MEM_Zero ){ + sqlite3VdbeMemExpandBlob(pVal); + } if( pVal->enc != (enc & ~SQLITE_UTF16_ALIGNED) ){ sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); } @@ -72987,7 +67204,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ ** to be a scalar SQL function. If ** ** * all function arguments are SQL literals, -** * one of the SQLITE_FUNC_CONSTANT or _SLOCHNG function flags is set, and +** * the SQLITE_FUNC_CONSTANT function flag is set, and ** * the SQLITE_FUNC_NEEDCOLL function flag is not set, ** ** then this routine attempts to invoke the SQL function. Assuming no @@ -73017,6 +67234,7 @@ static int valueFromFunction( FuncDef *pFunc = 0; /* Function definition */ sqlite3_value *pVal = 0; /* New value */ int rc = SQLITE_OK; /* Return code */ + int nName; /* Size of function name in bytes */ ExprList *pList = 0; /* Function arguments */ int i; /* Iterator variable */ @@ -73024,9 +67242,10 @@ static int valueFromFunction( assert( (p->flags & EP_TokenOnly)==0 ); pList = p->x.pList; if( pList ) nVal = pList->nExpr; - pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0); + nName = sqlite3Strlen30(p->u.zToken); + pFunc = sqlite3FindFunction(db, p->u.zToken, nName, nVal, enc, 0); assert( pFunc ); - if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + if( (pFunc->funcFlags & SQLITE_FUNC_CONSTANT)==0 || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) ){ return SQLITE_OK; @@ -73035,7 +67254,7 @@ static int valueFromFunction( if( pList ){ apVal = (sqlite3_value**)sqlite3DbMallocZero(db, sizeof(apVal[0]) * nVal); if( apVal==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; goto value_from_function_out; } for(i=0; ixSFunc(&ctx, nVal, apVal); + pFunc->xFunc(&ctx, nVal, apVal); if( ctx.isError ){ rc = ctx.isError; sqlite3ErrorMsg(pCtx->pParse, "%s", sqlite3_value_text(pVal)); @@ -73112,8 +67331,11 @@ static int valueFromExpr( const char *zNeg = ""; int rc = SQLITE_OK; - assert( pExpr!=0 ); - while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft; + if( !pExpr ){ + *ppVal = 0; + return SQLITE_OK; + } + while( (op = pExpr->op)==TK_UPLUS ) pExpr = pExpr->pLeft; if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; /* Compressed expressions only appear when parsing the DEFAULT clause @@ -73154,7 +67376,7 @@ static int valueFromExpr( if( zVal==0 ) goto no_mem; sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); } - if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){ + if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){ sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); @@ -73208,7 +67430,7 @@ static int valueFromExpr( return rc; no_mem: - sqlite3OomFault(db); + db->mallocFailed = 1; sqlite3DbFree(db, zVal); assert( *ppVal==0 ); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -73216,7 +67438,7 @@ static int valueFromExpr( #else assert( pCtx==0 ); sqlite3ValueFree(pVal); #endif - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } /* @@ -73236,7 +67458,7 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( u8 affinity, /* Affinity to use */ sqlite3_value **ppVal /* Write the new value here */ ){ - return pExpr ? valueFromExpr(db, pExpr, enc, affinity, ppVal, 0) : 0; + return valueFromExpr(db, pExpr, enc, affinity, ppVal, 0); } #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -73254,20 +67476,21 @@ static void recordFunc( sqlite3_value **argv ){ const int file_format = 1; - u32 iSerial; /* Serial type */ + int iSerial; /* Serial type */ int nSerial; /* Bytes of space for iSerial as varint */ - u32 nVal; /* Bytes of space required for argv[0] */ + int nVal; /* Bytes of space required for argv[0] */ int nRet; sqlite3 *db; u8 *aRet; UNUSED_PARAMETER( argc ); - iSerial = sqlite3VdbeSerialType(argv[0], file_format, &nVal); + iSerial = sqlite3VdbeSerialType(argv[0], file_format); nSerial = sqlite3VarintLen(iSerial); + nVal = sqlite3VdbeSerialTypeLen(iSerial); db = sqlite3_context_db_handle(context); nRet = 1 + nSerial + nVal; - aRet = sqlite3DbMallocRawNN(db, nRet); + aRet = sqlite3DbMallocRaw(db, nRet); if( aRet==0 ){ sqlite3_result_error_nomem(context); }else{ @@ -73283,10 +67506,15 @@ static void recordFunc( ** Register built-in functions used to help read ANALYZE data. */ SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void){ - static FuncDef aAnalyzeTableFuncs[] = { + static SQLITE_WSD FuncDef aAnalyzeTableFuncs[] = { FUNCTION(sqlite_record, 1, 0, 0, recordFunc), }; - sqlite3InsertBuiltinFuncs(aAnalyzeTableFuncs, ArraySize(aAnalyzeTableFuncs)); + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAnalyzeTableFuncs); + for(i=0; iop!=TK_SELECT ){ - int i; - struct ValueNewStat4Ctx alloc; - - alloc.pParse = pParse; - alloc.pIdx = pIdx; - alloc.ppRec = ppRec; + int rc; + sqlite3_value *pVal = 0; + struct ValueNewStat4Ctx alloc; - for(i=0; idb, pIdx, iVal+i); - alloc.iVal = iVal+i; - rc = stat4ValueFromExpr(pParse, pElem, aff, &alloc, &pVal); - if( !pVal ) break; - nExtract++; - } - } + alloc.pParse = pParse; + alloc.pIdx = pIdx; + alloc.ppRec = ppRec; + alloc.iVal = iVal; - *pnExtract = nExtract; + rc = stat4ValueFromExpr(pParse, pExpr, affinity, &alloc, &pVal); + assert( pVal==0 || pVal->db==pParse->db ); + *pbOk = (pVal!=0); return rc; } @@ -73480,7 +67693,7 @@ SQLITE_PRIVATE int sqlite3Stat4Column( if( iField>nRec ) return SQLITE_CORRUPT_BKPT; if( pMem==0 ){ pMem = *ppVal = sqlite3ValueNew(db); - if( pMem==0 ) return SQLITE_NOMEM_BKPT; + if( pMem==0 ) return SQLITE_NOMEM; } sqlite3VdbeSerialGet(&a[iField-szField], t, pMem); pMem->enc = ENC(db); @@ -73530,28 +67743,19 @@ SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value *v){ } /* -** The sqlite3ValueBytes() routine returns the number of bytes in the -** sqlite3_value object assuming that it uses the encoding "enc". -** The valueBytes() routine is a helper function. +** Return the number of bytes in the sqlite3_value object assuming +** that it uses the encoding "enc" */ -static SQLITE_NOINLINE int valueBytes(sqlite3_value *pVal, u8 enc){ - return valueToText(pVal, enc)!=0 ? pVal->n : 0; -} SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ Mem *p = (Mem*)pVal; - assert( (p->flags & MEM_Null)==0 || (p->flags & (MEM_Str|MEM_Blob))==0 ); - if( (p->flags & MEM_Str)!=0 && pVal->enc==enc ){ - return p->n; - } - if( (p->flags & MEM_Blob)!=0 ){ + if( (p->flags & MEM_Blob)!=0 || sqlite3ValueText(pVal, enc) ){ if( p->flags & MEM_Zero ){ return p->n + p->u.nZero; }else{ return p->n; } } - if( p->flags & MEM_Null ) return 0; - return valueBytes(pVal, enc); + return 0; } /************** End of vdbemem.c *********************************************/ @@ -73570,8 +67774,6 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ ** This file contains code used for creating, destroying, and populating ** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) */ -/* #include "sqliteInt.h" */ -/* #include "vdbeInt.h" */ /* ** Create a new virtual database engine. @@ -73579,9 +67781,8 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ sqlite3 *db = pParse->db; Vdbe *p; - p = sqlite3DbMallocRawNN(db, sizeof(Vdbe) ); + p = sqlite3DbMallocZero(db, sizeof(Vdbe) ); if( p==0 ) return 0; - memset(&p->aOp, 0, sizeof(Vdbe)-offsetof(Vdbe,aOp)); p->db = db; if( db->pVdbe ){ db->pVdbe->pPrev = p; @@ -73594,21 +67795,9 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ assert( pParse->aLabel==0 ); assert( pParse->nLabel==0 ); assert( pParse->nOpAlloc==0 ); - assert( pParse->szOpAlloc==0 ); return p; } -/* -** Change the error string stored in Vdbe.zErrMsg -*/ -SQLITE_PRIVATE void sqlite3VdbeError(Vdbe *p, const char *zFormat, ...){ - va_list ap; - sqlite3DbFree(p->db, p->zErrMsg); - va_start(ap, zFormat); - p->zErrMsg = sqlite3VMPrintf(p->db, zFormat, ap); - va_end(ap); -} - /* ** Remember the SQL string for a prepared statement. */ @@ -73623,13 +67812,20 @@ SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepa p->isPrepareV2 = (u8)isPrepareV2; } +/* +** Return the SQL associated with a prepared statement +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_sql(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe *)pStmt; + return (p && p->isPrepareV2) ? p->zSql : 0; +} + /* ** Swap all content between two VDBE structures. */ SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ Vdbe tmp, *pTmp; char *zTmp; - assert( pA->db==pB->db ); tmp = *pA; *pA = *pB; *pB = tmp; @@ -73677,11 +67873,10 @@ static int growOpArray(Vdbe *v, int nOp){ assert( nNew>=(p->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); if( pNew ){ - p->szOpAlloc = sqlite3DbMallocSize(p->db, pNew); - p->nOpAlloc = p->szOpAlloc/sizeof(Op); + p->nOpAlloc = sqlite3DbMallocSize(p->db, pNew)/sizeof(Op); v->aOp = pNew; } - return (pNew ? SQLITE_OK : SQLITE_NOMEM_BKPT); + return (pNew ? SQLITE_OK : SQLITE_NOMEM); } #ifdef SQLITE_DEBUG @@ -73711,21 +67906,17 @@ static void test_addop_breakpoint(void){ ** the sqlite3VdbeChangeP4() function to change the value of the P4 ** operand. */ -static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ - assert( p->pParse->nOpAlloc<=p->nOp ); - if( growOpArray(p, 1) ) return 1; - assert( p->pParse->nOpAlloc>p->nOp ); - return sqlite3VdbeAddOp3(p, op, p1, p2, p3); -} SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ int i; VdbeOp *pOp; i = p->nOp; assert( p->magic==VDBE_MAGIC_INIT ); - assert( op>=0 && op<0xff ); + assert( op>0 && op<0xff ); if( p->pParse->nOpAlloc<=i ){ - return growOp3(p, op, p1, p2, p3); + if( growOpArray(p, 1) ){ + return 1; + } } p->nOp++; pOp = &p->aOp[i]; @@ -73743,8 +67934,9 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ if( p->db->flags & SQLITE_VdbeAddopTrace ){ int jj, kk; Parse *pParse = p->pParse; - for(jj=kk=0; jjnColCache; jj++){ + for(jj=kk=0; jjaColCache + jj; + if( x->iLevel>pParse->iCacheLevel || x->iReg==0 ) continue; printf(" r[%d]={%d:%d}", x->iReg, x->iTable, x->iColumn); kk++; } @@ -73772,43 +67964,6 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ return sqlite3VdbeAddOp3(p, op, p1, p2, 0); } -/* Generate code for an unconditional jump to instruction iDest -*/ -SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe *p, int iDest){ - return sqlite3VdbeAddOp3(p, OP_Goto, 0, iDest, 0); -} - -/* Generate code to cause the string zStr to be loaded into -** register iDest -*/ -SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe *p, int iDest, const char *zStr){ - return sqlite3VdbeAddOp4(p, OP_String8, 0, iDest, 0, zStr, 0); -} - -/* -** Generate code that initializes multiple registers to string or integer -** constants. The registers begin with iDest and increase consecutively. -** One register is initialized for each characgter in zTypes[]. For each -** "s" character in zTypes[], the register is a string if the argument is -** not NULL, or OP_Null if the value is a null pointer. For each "i" character -** in zTypes[], the register is initialized to an integer. -*/ -SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe *p, int iDest, const char *zTypes, ...){ - va_list ap; - int i; - char c; - va_start(ap, zTypes); - for(i=0; (c = zTypes[i])!=0; i++){ - if( c=='s' ){ - const char *z = va_arg(ap, const char*); - sqlite3VdbeAddOp4(p, z==0 ? OP_Null : OP_String8, 0, iDest++, 0, z, 0); - }else{ - assert( c=='i' ); - sqlite3VdbeAddOp2(p, OP_Integer, va_arg(ap, int), iDest++); - } - } - va_end(ap); -} /* ** Add an opcode that includes the p4 value as a pointer. @@ -73827,24 +67982,6 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4( return addr; } -/* -** Add an opcode that includes the p4 value with a P4_INT64 or -** P4_REAL type. -*/ -SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( - Vdbe *p, /* Add the opcode to this VM */ - int op, /* The new opcode */ - int p1, /* The P1 operand */ - int p2, /* The P2 operand */ - int p3, /* The P3 operand */ - const u8 *zP4, /* The P4 operand */ - int p4type /* P4 operand type */ -){ - char *p4copy = sqlite3DbMallocRawNN(sqlite3VdbeDb(p), 8); - if( p4copy ) memcpy(p4copy, zP4, 8); - return sqlite3VdbeAddOp4(p, op, p1, p2, p3, p4copy, p4type); -} - /* ** Add an OP_ParseSchema opcode. This routine is broken out from ** sqlite3VdbeAddOp4() since it needs to also needs to mark all btrees @@ -73855,7 +67992,8 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( */ SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ int j; - sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); + sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); } @@ -73875,21 +68013,6 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( return addr; } -/* Insert the end of a co-routine -*/ -SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe *v, int regYield){ - sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); - - /* Clear the temporary register cache, thereby ensuring that each - ** co-routine has its own independent set of registers, because co-routines - ** might expect their registers to be preserved across an OP_Yield, and - ** that could cause problems if two or more co-routines are using the same - ** temporary register. - */ - v->pParse->nTempReg = 0; - v->pParse->nRangeReg = 0; -} - /* ** Create a new symbolic label for an instruction that has yet to be ** coded. The symbolic label is really just a negative number. The @@ -73915,7 +68038,7 @@ SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ if( p->aLabel ){ p->aLabel[i] = -1; } - return ADDR(i); + return -1-i; } /* @@ -73925,13 +68048,13 @@ SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ */ SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ Parse *p = v->pParse; - int j = ADDR(x); + int j = -1-x; assert( v->magic==VDBE_MAGIC_INIT ); assert( jnLabel ); - assert( j>=0 ); - if( p->aLabel ){ + if( ALWAYS(j>=0) && p->aLabel ){ p->aLabel[j] = v->nOp; } + p->iFixedOp = v->nOp - 1; } /* @@ -73941,13 +68064,6 @@ SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe *p){ p->runOnlyOnce = 1; } -/* -** Mark the VDBE as one that can only be run multiple times. -*/ -SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe *p){ - p->runOnlyOnce = 0; -} - #ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */ /* @@ -74030,7 +68146,6 @@ static Op *opIterNext(VdbeOpIter *p){ ** * OP_VUpdate ** * OP_VRename ** * OP_FkCounter with P2==0 (immediate foreign key constraint) -** * OP_CreateTable and OP_InitCoroutine (for CREATE TABLE AS SELECT ...) ** ** Then check that the value of Parse.mayAbort is true if an ** ABORT may be thrown, or false otherwise. Return true if it does @@ -74042,8 +68157,6 @@ static Op *opIterNext(VdbeOpIter *p){ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ int hasAbort = 0; int hasFkCounter = 0; - int hasCreateTable = 0; - int hasInitCoroutine = 0; Op *pOp; VdbeOpIter sIter; memset(&sIter, 0, sizeof(sIter)); @@ -74058,8 +68171,6 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ hasAbort = 1; break; } - if( opcode==OP_CreateTable ) hasCreateTable = 1; - if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1; #ifndef SQLITE_OMIT_FOREIGN_KEY if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ hasFkCounter = 1; @@ -74073,105 +68184,94 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ ** through all opcodes and hasAbort may be set incorrectly. Return ** true for this case to prevent the assert() in the callers frame ** from failing. */ - return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter - || (hasCreateTable && hasInitCoroutine) ); + return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter ); } #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ /* -** This routine is called after all opcodes have been inserted. It loops -** through all the opcodes and fixes up some details. +** Loop through the program looking for P2 values that are negative +** on jump instructions. Each such value is a label. Resolve the +** label by setting the P2 value to its correct non-zero value. ** -** (1) For each jump instruction with a negative P2 value (a label) -** resolve the P2 value to an actual address. +** This routine is called once after all opcodes have been inserted. ** -** (2) Compute the maximum number of arguments used by any SQL function -** and store that value in *pMaxFuncArgs. +** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument +** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by +** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array. ** -** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately -** indicate what the prepared statement actually does. -** -** (4) Initialize the p4.xAdvance pointer on opcodes that use it. -** -** (5) Reclaim the memory allocated for storing labels. -** -** This routine will only function correctly if the mkopcodeh.tcl generator -** script numbers the opcodes correctly. Changes to this routine must be -** coordinated with changes to mkopcodeh.tcl. +** The Op.opflags field is set on all opcodes. */ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ + int i; int nMaxArgs = *pMaxFuncArgs; Op *pOp; Parse *pParse = p->pParse; int *aLabel = pParse->aLabel; p->readOnly = 1; p->bIsReader = 0; - pOp = &p->aOp[p->nOp-1]; - while(1){ - - /* Only JUMP opcodes and the short list of special opcodes in the switch - ** below need to be considered. The mkopcodeh.tcl generator script groups - ** all these opcodes together near the front of the opcode list. Skip - ** any opcode that does not need processing by virtual of the fact that - ** it is larger than SQLITE_MX_JUMP_OPCODE, as a performance optimization. - */ - if( pOp->opcode<=SQLITE_MX_JUMP_OPCODE ){ - /* NOTE: Be sure to update mkopcodeh.tcl when adding or removing - ** cases from this switch! */ - switch( pOp->opcode ){ - case OP_Transaction: { - if( pOp->p2!=0 ) p->readOnly = 0; - /* fall thru */ - } - case OP_AutoCommit: - case OP_Savepoint: { - p->bIsReader = 1; - break; - } + for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){ + u8 opcode = pOp->opcode; + + /* NOTE: Be sure to update mkopcodeh.awk when adding or removing + ** cases from this switch! */ + switch( opcode ){ + case OP_Function: + case OP_AggStep: { + if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5; + break; + } + case OP_Transaction: { + if( pOp->p2!=0 ) p->readOnly = 0; + /* fall thru */ + } + case OP_AutoCommit: + case OP_Savepoint: { + p->bIsReader = 1; + break; + } #ifndef SQLITE_OMIT_WAL - case OP_Checkpoint: + case OP_Checkpoint: #endif - case OP_Vacuum: - case OP_JournalMode: { - p->readOnly = 0; - p->bIsReader = 1; - break; - } + case OP_Vacuum: + case OP_JournalMode: { + p->readOnly = 0; + p->bIsReader = 1; + break; + } #ifndef SQLITE_OMIT_VIRTUALTABLE - case OP_VUpdate: { - if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; - break; - } - case OP_VFilter: { - int n; - assert( (pOp - p->aOp) >= 3 ); - assert( pOp[-1].opcode==OP_Integer ); - n = pOp[-1].p1; - if( n>nMaxArgs ) nMaxArgs = n; - break; - } + case OP_VUpdate: { + if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; + break; + } + case OP_VFilter: { + int n; + assert( p->nOp - i >= 3 ); + assert( pOp[-1].opcode==OP_Integer ); + n = pOp[-1].p1; + if( n>nMaxArgs ) nMaxArgs = n; + break; + } #endif - case OP_Next: - case OP_NextIfOpen: - case OP_SorterNext: { - pOp->p4.xAdvance = sqlite3BtreeNext; - pOp->p4type = P4_ADVANCE; - break; - } - case OP_Prev: - case OP_PrevIfOpen: { - pOp->p4.xAdvance = sqlite3BtreePrevious; - pOp->p4type = P4_ADVANCE; - break; - } + case OP_Next: + case OP_NextIfOpen: + case OP_SorterNext: { + pOp->p4.xAdvance = sqlite3BtreeNext; + pOp->p4type = P4_ADVANCE; + break; } - if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 && pOp->p2<0 ){ - assert( ADDR(pOp->p2)nLabel ); - pOp->p2 = aLabel[ADDR(pOp->p2)]; + case OP_Prev: + case OP_PrevIfOpen: { + pOp->p4.xAdvance = sqlite3BtreePrevious; + pOp->p4type = P4_ADVANCE; + break; } } - if( pOp==p->aOp ) break; - pOp--; + + pOp->opflags = sqlite3OpcodeProperty[opcode]; + if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){ + assert( -1-pOp->p2nLabel ); + pOp->p2 = aLabel[-1-pOp->p2]; + } } sqlite3DbFree(p->db, pParse->aLabel); pParse->aLabel = 0; @@ -74188,20 +68288,6 @@ SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe *p){ return p->nOp; } -/* -** Verify that at least N opcode slots are available in p without -** having to malloc for more space (except when compiled using -** SQLITE_TEST_REALLOC_STRESS). This interface is used during testing -** to verify that certain calls to sqlite3VdbeAddOpList() can never -** fail due to a OOM fault and hence that the return value from -** sqlite3VdbeAddOpList() will always be non-NULL. -*/ -#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) -SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N){ - assert( p->nOp + N <= p->pParse->nOpAlloc ); -} -#endif - /* ** This function returns a pointer to the array of opcodes associated with ** the Vdbe passed as the first argument. It is the callers responsibility @@ -74227,54 +68313,51 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg) } /* -** Add a whole list of operations to the operation stack. Return a -** pointer to the first operation inserted. -** -** Non-zero P2 arguments to jump instructions are automatically adjusted -** so that the jump target is relative to the first operation inserted. +** Add a whole list of operations to the operation stack. Return the +** address of the first operation added. */ -SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList( - Vdbe *p, /* Add opcodes to the prepared statement */ - int nOp, /* Number of opcodes to add */ - VdbeOpList const *aOp, /* The opcodes to be added */ - int iLineno /* Source-file line number of first opcode */ -){ - int i; - VdbeOp *pOut, *pFirst; - assert( nOp>0 ); +SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ + int addr; assert( p->magic==VDBE_MAGIC_INIT ); if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ return 0; } - pFirst = pOut = &p->aOp[p->nOp]; - for(i=0; iopcode = aOp->opcode; - pOut->p1 = aOp->p1; - pOut->p2 = aOp->p2; - assert( aOp->p2>=0 ); - if( (sqlite3OpcodeProperty[aOp->opcode] & OPFLG_JUMP)!=0 && aOp->p2>0 ){ - pOut->p2 += p->nOp; - } - pOut->p3 = aOp->p3; - pOut->p4type = P4_NOTUSED; - pOut->p4.p = 0; - pOut->p5 = 0; + addr = p->nOp; + if( ALWAYS(nOp>0) ){ + int i; + VdbeOpList const *pIn = aOp; + for(i=0; ip2; + VdbeOp *pOut = &p->aOp[i+addr]; + pOut->opcode = pIn->opcode; + pOut->p1 = pIn->p1; + if( p2<0 ){ + assert( sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP ); + pOut->p2 = addr + ADDR(p2); + }else{ + pOut->p2 = p2; + } + pOut->p3 = pIn->p3; + pOut->p4type = P4_NOTUSED; + pOut->p4.p = 0; + pOut->p5 = 0; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - pOut->zComment = 0; + pOut->zComment = 0; #endif #ifdef SQLITE_VDBE_COVERAGE - pOut->iSrcLine = iLineno+i; + pOut->iSrcLine = iLineno+i; #else - (void)iLineno; + (void)iLineno; #endif #ifdef SQLITE_DEBUG - if( p->db->flags & SQLITE_VdbeAddopTrace ){ - sqlite3VdbePrintOp(0, i+p->nOp, &p->aOp[i+p->nOp]); - } + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); + } #endif + } + p->nOp += nOp; } - p->nOp += nOp; - return pFirst; + return addr; } #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) @@ -74306,24 +68389,49 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( /* -** Change the value of the opcode, or P1, P2, P3, or P5 operands -** for a specific instruction. +** Change the value of the P1 operand for a specific instruction. +** This routine is useful when a large program is loaded from a +** static array using sqlite3VdbeAddOpList but we want to make a +** few minor changes to the program. */ -SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, u32 addr, u8 iNewOpcode){ - sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; -} SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ - sqlite3VdbeGetOp(p,addr)->p1 = val; + assert( p!=0 ); + if( ((u32)p->nOp)>addr ){ + p->aOp[addr].p1 = val; + } } + +/* +** Change the value of the P2 operand for a specific instruction. +** This routine is useful for setting a jump destination. +*/ SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ - sqlite3VdbeGetOp(p,addr)->p2 = val; + assert( p!=0 ); + if( ((u32)p->nOp)>addr ){ + p->aOp[addr].p2 = val; + } } + +/* +** Change the value of the P3 operand for a specific instruction. +*/ SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ - sqlite3VdbeGetOp(p,addr)->p3 = val; + assert( p!=0 ); + if( ((u32)p->nOp)>addr ){ + p->aOp[addr].p3 = val; + } } -SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){ - assert( p->nOp>0 || p->db->mallocFailed ); - if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5; + +/* +** Change the value of the P5 operand for the most recently +** added operation. +*/ +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ + assert( p!=0 ); + if( p->aOp ){ + assert( p->nOp>0 ); + p->aOp[p->nOp-1].p5 = val; + } } /* @@ -74332,6 +68440,7 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){ */ SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ sqlite3VdbeChangeP2(p, addr, p->nOp); + p->pParse->iFixedOp = p->nOp - 1; } @@ -74340,7 +68449,7 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ ** the FuncDef is not ephermal, then do nothing. */ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ - if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ + if( ALWAYS(pDef) && (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ sqlite3DbFree(db, pDef); } } @@ -74350,57 +68459,43 @@ static void vdbeFreeOpArray(sqlite3 *, Op *, int); /* ** Delete a P4 value if necessary. */ -static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){ - if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); - sqlite3DbFree(db, p); -} -static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){ - freeEphemeralFunction(db, p->pFunc); - sqlite3DbFree(db, p); -} static void freeP4(sqlite3 *db, int p4type, void *p4){ - assert( db ); - switch( p4type ){ - case P4_FUNCCTX: { - freeP4FuncCtx(db, (sqlite3_context*)p4); - break; - } - case P4_REAL: - case P4_INT64: - case P4_DYNAMIC: - case P4_INTARRAY: { - sqlite3DbFree(db, p4); - break; - } - case P4_KEYINFO: { - if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4); - break; - } -#ifdef SQLITE_ENABLE_CURSOR_HINTS - case P4_EXPR: { - sqlite3ExprDelete(db, (Expr*)p4); - break; - } -#endif - case P4_MPRINTF: { - if( db->pnBytesFreed==0 ) sqlite3_free(p4); - break; - } - case P4_FUNCDEF: { - freeEphemeralFunction(db, (FuncDef*)p4); - break; - } - case P4_MEM: { - if( db->pnBytesFreed==0 ){ - sqlite3ValueFree((sqlite3_value*)p4); - }else{ - freeP4Mem(db, (Mem*)p4); + if( p4 ){ + assert( db ); + switch( p4type ){ + case P4_REAL: + case P4_INT64: + case P4_DYNAMIC: + case P4_INTARRAY: { + sqlite3DbFree(db, p4); + break; + } + case P4_KEYINFO: { + if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4); + break; + } + case P4_MPRINTF: { + if( db->pnBytesFreed==0 ) sqlite3_free(p4); + break; + } + case P4_FUNCDEF: { + freeEphemeralFunction(db, (FuncDef*)p4); + break; + } + case P4_MEM: { + if( db->pnBytesFreed==0 ){ + sqlite3ValueFree((sqlite3_value*)p4); + }else{ + Mem *p = (Mem*)p4; + if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); + sqlite3DbFree(db, p); + } + break; + } + case P4_VTAB : { + if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); + break; } - break; - } - case P4_VTAB : { - if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); - break; } } } @@ -74414,7 +68509,7 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ if( aOp ){ Op *pOp; for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ - if( pOp->p4type ) freeP4(db, pOp->p4type, pOp->p4.p); + freeP4(db, pOp->p4type, pOp->p4.p); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS sqlite3DbFree(db, pOp->zComment); #endif @@ -74436,16 +68531,15 @@ SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ /* ** Change the opcode at addr into OP_Noop */ -SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ - VdbeOp *pOp; - if( p->db->mallocFailed ) return 0; - assert( addr>=0 && addrnOp ); - pOp = &p->aOp[addr]; - freeP4(p->db, pOp->p4type, pOp->p4.p); - pOp->p4type = P4_NOTUSED; - pOp->p4.z = 0; - pOp->opcode = OP_Noop; - return 1; +SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ + if( addrnOp ){ + VdbeOp *pOp = &p->aOp[addr]; + sqlite3 *db = p->db; + freeP4(db, pOp->p4type, pOp->p4.p); + memset(pOp, 0, sizeof(pOp[0])); + pOp->opcode = OP_Noop; + if( addr==p->nOp-1 ) p->nOp--; + } } /* @@ -74453,8 +68547,9 @@ SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ ** then remove it. Return true if and only if an opcode was removed. */ SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ - if( p->nOp>0 && p->aOp[p->nOp-1].opcode==op ){ - return sqlite3VdbeChangeToNoop(p, p->nOp-1); + if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){ + sqlite3VdbeChangeToNoop(p, p->nOp-1); + return 1; }else{ return 0; } @@ -74477,34 +68572,16 @@ SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ ** ** If addr<0 then change P4 on the most recently inserted instruction. */ -static void SQLITE_NOINLINE vdbeChangeP4Full( - Vdbe *p, - Op *pOp, - const char *zP4, - int n -){ - if( pOp->p4type ){ - freeP4(p->db, pOp->p4type, pOp->p4.p); - pOp->p4type = 0; - pOp->p4.p = 0; - } - if( n<0 ){ - sqlite3VdbeChangeP4(p, (int)(pOp - p->aOp), zP4, n); - }else{ - if( n==0 ) n = sqlite3Strlen30(zP4); - pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); - pOp->p4type = P4_DYNAMIC; - } -} SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ Op *pOp; sqlite3 *db; assert( p!=0 ); db = p->db; assert( p->magic==VDBE_MAGIC_INIT ); - assert( p->aOp!=0 || db->mallocFailed ); - if( db->mallocFailed ){ - if( n!=P4_VTAB ) freeP4(db, n, (void*)*(char**)&zP4); + if( p->aOp==0 || db->mallocFailed ){ + if( n!=P4_VTAB ){ + freeP4(db, n, (void*)*(char**)&zP4); + } return; } assert( p->nOp>0 ); @@ -74513,20 +68590,34 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int addr = p->nOp - 1; } pOp = &p->aOp[addr]; - if( n>=0 || pOp->p4type ){ - vdbeChangeP4Full(p, pOp, zP4, n); - return; - } + assert( pOp->p4type==P4_NOTUSED + || pOp->p4type==P4_INT32 + || pOp->p4type==P4_KEYINFO ); + freeP4(db, pOp->p4type, pOp->p4.p); + pOp->p4.p = 0; if( n==P4_INT32 ){ /* Note: this cast is safe, because the origin data point was an int ** that was cast to a (const char *). */ pOp->p4.i = SQLITE_PTR_TO_INT(zP4); pOp->p4type = P4_INT32; - }else if( zP4!=0 ){ - assert( n<0 ); + }else if( zP4==0 ){ + pOp->p4.p = 0; + pOp->p4type = P4_NOTUSED; + }else if( n==P4_KEYINFO ){ + pOp->p4.p = (void*)zP4; + pOp->p4type = P4_KEYINFO; + }else if( n==P4_VTAB ){ + pOp->p4.p = (void*)zP4; + pOp->p4type = P4_VTAB; + sqlite3VtabLock((VTable *)zP4); + assert( ((VTable *)zP4)->db==p->db ); + }else if( n<0 ){ pOp->p4.p = (void*)zP4; pOp->p4type = (signed char)n; - if( n==P4_VTAB ) sqlite3VtabLock((VTable*)zP4); + }else{ + if( n==0 ) n = sqlite3Strlen30(zP4); + pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); + pOp->p4type = P4_DYNAMIC; } } @@ -74651,21 +68742,12 @@ static int displayComment( const char *zSynopsis; int nOpName; int ii, jj; - char zAlt[50]; zOpName = sqlite3OpcodeName(pOp->opcode); nOpName = sqlite3Strlen30(zOpName); if( zOpName[nOpName+1] ){ int seenCom = 0; char c; zSynopsis = zOpName += nOpName + 1; - if( strncmp(zSynopsis,"IF ",3)==0 ){ - if( pOp->p5 & SQLITE_STOREP2 ){ - sqlite3_snprintf(sizeof(zAlt), zAlt, "r[P2] = (%s)", zSynopsis+3); - }else{ - sqlite3_snprintf(sizeof(zAlt), zAlt, "if %s goto P2", zSynopsis+3); - } - zSynopsis = zAlt; - } for(ii=jj=0; jjop ){ - case TK_STRING: - sqlite3XPrintf(p, "%Q", pExpr->u.zToken); - break; - case TK_INTEGER: - sqlite3XPrintf(p, "%d", pExpr->u.iValue); - break; - case TK_NULL: - sqlite3XPrintf(p, "NULL"); - break; - case TK_REGISTER: { - sqlite3XPrintf(p, "r[%d]", pExpr->iTable); - break; - } - case TK_COLUMN: { - if( pExpr->iColumn<0 ){ - sqlite3XPrintf(p, "rowid"); - }else{ - sqlite3XPrintf(p, "c%d", (int)pExpr->iColumn); - } - break; - } - case TK_LT: zOp = "LT"; break; - case TK_LE: zOp = "LE"; break; - case TK_GT: zOp = "GT"; break; - case TK_GE: zOp = "GE"; break; - case TK_NE: zOp = "NE"; break; - case TK_EQ: zOp = "EQ"; break; - case TK_IS: zOp = "IS"; break; - case TK_ISNOT: zOp = "ISNOT"; break; - case TK_AND: zOp = "AND"; break; - case TK_OR: zOp = "OR"; break; - case TK_PLUS: zOp = "ADD"; break; - case TK_STAR: zOp = "MUL"; break; - case TK_MINUS: zOp = "SUB"; break; - case TK_REM: zOp = "REM"; break; - case TK_BITAND: zOp = "BITAND"; break; - case TK_BITOR: zOp = "BITOR"; break; - case TK_SLASH: zOp = "DIV"; break; - case TK_LSHIFT: zOp = "LSHIFT"; break; - case TK_RSHIFT: zOp = "RSHIFT"; break; - case TK_CONCAT: zOp = "CONCAT"; break; - case TK_UMINUS: zOp = "MINUS"; break; - case TK_UPLUS: zOp = "PLUS"; break; - case TK_BITNOT: zOp = "BITNOT"; break; - case TK_NOT: zOp = "NOT"; break; - case TK_ISNULL: zOp = "ISNULL"; break; - case TK_NOTNULL: zOp = "NOTNULL"; break; - - default: - sqlite3XPrintf(p, "%s", "expr"); - break; - } - if( zOp ){ - sqlite3XPrintf(p, "%s(", zOp); - displayP4Expr(p, pExpr->pLeft); - if( pExpr->pRight ){ - sqlite3StrAccumAppend(p, ",", 1); - displayP4Expr(p, pExpr->pRight); - } - sqlite3StrAccumAppend(p, ")", 1); - } -} -#endif /* VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) */ - - -#if VDBE_DISPLAY_P4 +#if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ + || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) /* ** Compute a string that describes the P4 parameter for an opcode. ** Use zTemp for any required temporary buffer space. */ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ char *zP4 = zTemp; - StrAccum x; assert( nTemp>=20 ); - sqlite3StrAccumInit(&x, 0, zTemp, nTemp, 0); switch( pOp->p4type ){ case P4_KEYINFO: { - int j; + int i, j; KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; assert( pKeyInfo->aSortOrder!=0 ); - sqlite3XPrintf(&x, "k(%d", pKeyInfo->nField); + sqlite3_snprintf(nTemp, zTemp, "k(%d", pKeyInfo->nField); + i = sqlite3Strlen30(zTemp); for(j=0; jnField; j++){ CollSeq *pColl = pKeyInfo->aColl[j]; - const char *zColl = pColl ? pColl->zName : ""; - if( strcmp(zColl, "BINARY")==0 ) zColl = "B"; - sqlite3XPrintf(&x, ",%s%s", pKeyInfo->aSortOrder[j] ? "-" : "", zColl); + const char *zColl = pColl ? pColl->zName : "nil"; + int n = sqlite3Strlen30(zColl); + if( n==6 && memcmp(zColl,"BINARY",6)==0 ){ + zColl = "B"; + n = 1; + } + if( i+n>nTemp-6 ){ + memcpy(&zTemp[i],",...",4); + break; + } + zTemp[i++] = ','; + if( pKeyInfo->aSortOrder[j] ){ + zTemp[i++] = '-'; + } + memcpy(&zTemp[i], zColl, n+1); + i += n; } - sqlite3StrAccumAppend(&x, ")", 1); + zTemp[i++] = ')'; + zTemp[i] = 0; + assert( ip4.pExpr); - break; - } -#endif case P4_COLLSEQ: { CollSeq *pColl = pOp->p4.pColl; - sqlite3XPrintf(&x, "(%.20s)", pColl->zName); + sqlite3_snprintf(nTemp, zTemp, "(%.20s)", pColl->zName); break; } case P4_FUNCDEF: { FuncDef *pDef = pOp->p4.pFunc; - sqlite3XPrintf(&x, "%s(%d)", pDef->zName, pDef->nArg); - break; - } -#ifdef SQLITE_DEBUG - case P4_FUNCCTX: { - FuncDef *pDef = pOp->p4.pCtx->pFunc; - sqlite3XPrintf(&x, "%s(%d)", pDef->zName, pDef->nArg); + sqlite3_snprintf(nTemp, zTemp, "%s(%d)", pDef->zName, pDef->nArg); break; } -#endif case P4_INT64: { - sqlite3XPrintf(&x, "%lld", *pOp->p4.pI64); + sqlite3_snprintf(nTemp, zTemp, "%lld", *pOp->p4.pI64); break; } case P4_INT32: { - sqlite3XPrintf(&x, "%d", pOp->p4.i); + sqlite3_snprintf(nTemp, zTemp, "%d", pOp->p4.i); break; } case P4_REAL: { - sqlite3XPrintf(&x, "%.16g", *pOp->p4.pReal); + sqlite3_snprintf(nTemp, zTemp, "%.16g", *pOp->p4.pReal); break; } case P4_MEM: { @@ -74853,11 +68864,11 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ if( pMem->flags & MEM_Str ){ zP4 = pMem->z; }else if( pMem->flags & MEM_Int ){ - sqlite3XPrintf(&x, "%lld", pMem->u.i); + sqlite3_snprintf(nTemp, zTemp, "%lld", pMem->u.i); }else if( pMem->flags & MEM_Real ){ - sqlite3XPrintf(&x, "%.16g", pMem->u.r); + sqlite3_snprintf(nTemp, zTemp, "%.16g", pMem->u.r); }else if( pMem->flags & MEM_Null ){ - zP4 = "NULL"; + sqlite3_snprintf(nTemp, zTemp, "NULL"); }else{ assert( pMem->flags & MEM_Blob ); zP4 = "(blob)"; @@ -74867,34 +68878,22 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ #ifndef SQLITE_OMIT_VIRTUALTABLE case P4_VTAB: { sqlite3_vtab *pVtab = pOp->p4.pVtab->pVtab; - sqlite3XPrintf(&x, "vtab:%p", pVtab); + sqlite3_snprintf(nTemp, zTemp, "vtab:%p", pVtab); break; } #endif case P4_INTARRAY: { - int i; - int *ai = pOp->p4.ai; - int n = ai[0]; /* The first element of an INTARRAY is always the - ** count of the number of elements to follow */ - for(i=1; ip4.pTab->zName); - break; - } default: { zP4 = pOp->p4.z; if( zP4==0 ){ @@ -74903,11 +68902,10 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ } } } - sqlite3StrAccumFinish(&x); assert( zP4!=0 ); return zP4; } -#endif /* VDBE_DISPLAY_P4 */ +#endif /* ** Declare to the Vdbe that the BTree object at db->aDb[i] is used. @@ -74926,7 +68924,7 @@ SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe *p, int i){ } } -#if !defined(SQLITE_OMIT_SHARED_CACHE) +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 /* ** If SQLite is compiled to support shared-cache mode and to be threadsafe, ** this routine obtains the mutex associated with each BtShared structure @@ -74969,11 +68967,12 @@ SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe *p){ /* ** Unlock all of the btrees previously locked by a call to sqlite3VdbeEnter(). */ -static SQLITE_NOINLINE void vdbeLeave(Vdbe *p){ +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ int i; sqlite3 *db; Db *aDb; int nDb; + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ db = p->db; aDb = db->aDb; nDb = db->nDb; @@ -74983,10 +68982,6 @@ static SQLITE_NOINLINE void vdbeLeave(Vdbe *p){ } } } -SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ - if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ - vdbeLeave(p); -} #endif #if defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) @@ -75016,21 +69011,6 @@ SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){ } #endif -/* -** Initialize an array of N Mem element. -*/ -static void initMemArray(Mem *p, int N, sqlite3 *db, u16 flags){ - while( (N--)>0 ){ - p->db = db; - p->flags = flags; - p->szMalloc = 0; -#ifdef SQLITE_DEBUG - p->pScopyFrom = 0; -#endif - p++; - } -} - /* ** Release an array of N Mem elements */ @@ -75038,6 +69018,7 @@ static void releaseMemArray(Mem *p, int N){ if( p && N ){ Mem *pEnd = &p[N]; sqlite3 *db = p->db; + u8 malloc_failed = db->mallocFailed; if( db->pnBytesFreed ){ do{ if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); @@ -75073,6 +69054,7 @@ static void releaseMemArray(Mem *p, int N){ p->flags = MEM_Undefined; }while( (++p)mallocFailed = malloc_failed; } } @@ -75088,7 +69070,6 @@ SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ sqlite3VdbeFreeCursor(p->v, apCsr[i]); } releaseMemArray(aMem, p->nChildMem); - sqlite3VdbeDeleteAuxData(p->v->db, &p->pAuxData, -1, 0); sqlite3DbFree(p->v->db, p); } @@ -75131,10 +69112,10 @@ SQLITE_PRIVATE int sqlite3VdbeList( releaseMemArray(pMem, 8); p->pResultSet = 0; - if( p->rc==SQLITE_NOMEM_BKPT ){ + if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */ - sqlite3OomFault(db); + db->mallocFailed = 1; return SQLITE_ERROR; } @@ -75173,7 +69154,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( }else if( db->u1.isInterrupted ){ p->rc = SQLITE_INTERRUPT; rc = SQLITE_ERROR; - sqlite3VdbeError(p, sqlite3ErrStr(p->rc)); + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(p->rc)); }else{ char *zP4; Op *pOp; @@ -75235,14 +69216,13 @@ SQLITE_PRIVATE int sqlite3VdbeList( pMem->u.i = pOp->p3; /* P3 */ pMem++; - if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */ + if( sqlite3VdbeMemClearAndResize(pMem, 32) ){ /* P4 */ assert( p->db->mallocFailed ); return SQLITE_ERROR; } pMem->flags = MEM_Str|MEM_Term; - zP4 = displayP4(pOp, pMem->z, pMem->szMalloc); + zP4 = displayP4(pOp, pMem->z, 32); if( zP4!=pMem->z ){ - pMem->n = 0; sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0); }else{ assert( pMem->z!=0 ); @@ -75333,46 +69313,43 @@ SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe *p){ } #endif /* !SQLITE_OMIT_TRACE && SQLITE_ENABLE_IOTRACE */ -/* An instance of this object describes bulk memory available for use -** by subcomponents of a prepared statement. Space is allocated out -** of a ReusableSpace object by the allocSpace() routine below. -*/ -struct ReusableSpace { - u8 *pSpace; /* Available memory */ - int nFree; /* Bytes of available memory */ - int nNeeded; /* Total bytes that could not be allocated */ -}; - -/* Try to allocate nByte bytes of 8-byte aligned bulk memory for pBuf -** from the ReusableSpace object. Return a pointer to the allocated -** memory on success. If insufficient memory is available in the -** ReusableSpace object, increase the ReusableSpace.nNeeded -** value by the amount needed and return NULL. +/* +** Allocate space from a fixed size buffer and return a pointer to +** that space. If insufficient space is available, return NULL. +** +** The pBuf parameter is the initial value of a pointer which will +** receive the new memory. pBuf is normally NULL. If pBuf is not +** NULL, it means that memory space has already been allocated and that +** this routine should not allocate any new memory. When pBuf is not +** NULL simply return pBuf. Only allocate new memory space when pBuf +** is NULL. ** -** If pBuf is not initially NULL, that means that the memory has already -** been allocated by a prior call to this routine, so just return a copy -** of pBuf and leave ReusableSpace unchanged. +** nByte is the number of bytes of space needed. ** -** This allocator is employed to repurpose unused slots at the end of the -** opcode array of prepared state for other memory needs of the prepared -** statement. +** *ppFrom points to available space and pEnd points to the end of the +** available space. When space is allocated, *ppFrom is advanced past +** the end of the allocated space. +** +** *pnByte is a counter of the number of bytes of space that have failed +** to allocate. If there is insufficient space in *ppFrom to satisfy the +** request, then increment *pnByte by the amount of the request. */ static void *allocSpace( - struct ReusableSpace *p, /* Bulk memory available for allocation */ - void *pBuf, /* Pointer to a prior allocation */ - int nByte /* Bytes of memory needed */ -){ - assert( EIGHT_BYTE_ALIGNMENT(p->pSpace) ); - if( pBuf==0 ){ - nByte = ROUND8(nByte); - if( nByte <= p->nFree ){ - p->nFree -= nByte; - pBuf = &p->pSpace[p->nFree]; - }else{ - p->nNeeded += nByte; - } + void *pBuf, /* Where return pointer will be stored */ + int nByte, /* Number of bytes to allocate */ + u8 **ppFrom, /* IN/OUT: Allocate from *ppFrom */ + u8 *pEnd, /* Pointer to 1 byte past the end of *ppFrom buffer */ + int *pnByte /* If allocation cannot be made, increment *pnByte */ +){ + assert( EIGHT_BYTE_ALIGNMENT(*ppFrom) ); + if( pBuf ) return pBuf; + nByte = ROUND8(nByte); + if( &(*ppFrom)[nByte] <= pEnd ){ + pBuf = (void*)*ppFrom; + *ppFrom += nByte; + }else{ + *pnByte += nByte; } - assert( EIGHT_BYTE_ALIGNMENT(pBuf) ); return pBuf; } @@ -75385,7 +69362,7 @@ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ int i; #endif assert( p!=0 ); - assert( p->magic==VDBE_MAGIC_INIT || p->magic==VDBE_MAGIC_RESET ); + assert( p->magic==VDBE_MAGIC_INIT ); /* There should be at least one opcode. */ @@ -75395,13 +69372,14 @@ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ p->magic = VDBE_MAGIC_RUN; #ifdef SQLITE_DEBUG - for(i=0; inMem; i++){ + for(i=1; inMem; i++){ assert( p->aMem[i].db==p->db ); } #endif p->pc = -1; p->rc = SQLITE_OK; p->errorAction = OE_Abort; + p->magic = VDBE_MAGIC_RUN; p->nChange = 0; p->cacheCtr = 1; p->minWriteFileFormat = 255; @@ -75442,8 +69420,11 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( int nMem; /* Number of VM memory registers */ int nCursor; /* Number of cursors required */ int nArg; /* Number of arguments in subprograms */ + int nOnce; /* Number of OP_Once instructions */ int n; /* Loop counter */ - struct ReusableSpace x; /* Reusable bulk memory */ + u8 *zCsr; /* Memory available for allocation */ + u8 *zEnd; /* First byte past allocated memory */ + int nByte; /* How much extra memory is needed */ assert( p!=0 ); assert( p->nOp>0 ); @@ -75456,77 +69437,88 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( nMem = pParse->nMem; nCursor = pParse->nTab; nArg = pParse->nMaxArg; + nOnce = pParse->nOnce; + if( nOnce==0 ) nOnce = 1; /* Ensure at least one byte in p->aOnceFlag[] */ - /* Each cursor uses a memory cell. The first cursor (cursor 0) can - ** use aMem[0] which is not otherwise used by the VDBE program. Allocate - ** space at the end of aMem[] for cursors 1 and greater. + /* For each cursor required, also allocate a memory cell. Memory + ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by + ** the vdbe program. Instead they are used to allocate space for + ** VdbeCursor/BtCursor structures. The blob of memory associated with + ** cursor 0 is stored in memory cell nMem. Memory cell (nMem-1) + ** stores the blob of memory associated with cursor 1, etc. + ** ** See also: allocateCursor(). */ nMem += nCursor; - if( nCursor==0 && nMem>0 ) nMem++; /* Space for aMem[0] even if not used */ - /* Figure out how much reusable memory is available at the end of the - ** opcode array. This extra memory will be reallocated for other elements - ** of the prepared statement. + /* Allocate space for memory registers, SQL variables, VDBE cursors and + ** an array to marshal SQL function arguments in. */ - n = ROUND8(sizeof(Op)*p->nOp); /* Bytes of opcode memory used */ - x.pSpace = &((u8*)p->aOp)[n]; /* Unused opcode memory */ - assert( EIGHT_BYTE_ALIGNMENT(x.pSpace) ); - x.nFree = ROUNDDOWN8(pParse->szOpAlloc - n); /* Bytes of unused memory */ - assert( x.nFree>=0 ); - assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) ); + zCsr = (u8*)&p->aOp[p->nOp]; /* Memory avaliable for allocation */ + zEnd = (u8*)&p->aOp[pParse->nOpAlloc]; /* First byte past end of zCsr[] */ resolveP2Values(p, &nArg); p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); if( pParse->explain && nMem<10 ){ nMem = 10; } + memset(zCsr, 0, zEnd-zCsr); + zCsr += (zCsr - (u8*)0)&7; + assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); p->expired = 0; - /* Memory for registers, parameters, cursor, etc, is allocated in one or two - ** passes. On the first pass, we try to reuse unused memory at the + /* Memory for registers, parameters, cursor, etc, is allocated in two + ** passes. On the first pass, we try to reuse unused space at the ** end of the opcode array. If we are unable to satisfy all memory ** requirements by reusing the opcode array tail, then the second - ** pass will fill in the remainder using a fresh memory allocation. + ** pass will fill in the rest using a fresh allocation. ** ** This two-pass approach that reuses as much memory as possible from - ** the leftover memory at the end of the opcode array. This can significantly + ** the leftover space at the end of the opcode array can significantly ** reduce the amount of memory held by a prepared statement. */ do { - x.nNeeded = 0; - p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); - p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); - p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); - p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); + nByte = 0; + p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); + p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); + p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); + p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); + p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), + &zCsr, zEnd, &nByte); + p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, &zCsr, zEnd, &nByte); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); + p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), &zCsr, zEnd, &nByte); #endif - if( x.nNeeded==0 ) break; - x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); - x.nFree = x.nNeeded; - }while( !db->mallocFailed ); + if( nByte ){ + p->pFree = sqlite3DbMallocZero(db, nByte); + } + zCsr = p->pFree; + zEnd = &zCsr[nByte]; + }while( nByte && !db->mallocFailed ); - p->nzVar = pParse->nzVar; - p->azVar = pParse->azVar; - pParse->nzVar = 0; - pParse->azVar = 0; - p->explain = pParse->explain; - if( db->mallocFailed ){ - p->nVar = 0; - p->nCursor = 0; - p->nMem = 0; - }else{ - p->nCursor = nCursor; + p->nCursor = nCursor; + p->nOnceFlag = nOnce; + if( p->aVar ){ p->nVar = (ynVar)nVar; - initMemArray(p->aVar, nVar, db, MEM_Null); - p->nMem = nMem; - initMemArray(p->aMem, nMem, db, MEM_Undefined); - memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - memset(p->anExec, 0, p->nOp*sizeof(i64)); -#endif + for(n=0; naVar[n].flags = MEM_Null; + p->aVar[n].db = db; + } + } + if( p->azVar && pParse->nzVar>0 ){ + p->nzVar = pParse->nzVar; + memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); + memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); + } + if( p->aMem ){ + p->aMem--; /* aMem[] goes from 1..nMem */ + p->nMem = nMem; /* not from 0..nMem-1 */ + for(n=1; n<=nMem; n++){ + p->aMem[n].flags = MEM_Undefined; + p->aMem[n].db = db; + } } + p->explain = pParse->explain; sqlite3VdbeRewind(p); } @@ -75538,34 +69530,23 @@ SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ if( pCx==0 ){ return; } - assert( pCx->pBt==0 || pCx->eCurType==CURTYPE_BTREE ); - switch( pCx->eCurType ){ - case CURTYPE_SORTER: { - sqlite3VdbeSorterClose(p->db, pCx); - break; - } - case CURTYPE_BTREE: { - if( pCx->pBt ){ - sqlite3BtreeClose(pCx->pBt); - /* The pCx->pCursor will be close automatically, if it exists, by - ** the call above. */ - }else{ - assert( pCx->uc.pCursor!=0 ); - sqlite3BtreeCloseCursor(pCx->uc.pCursor); - } - break; - } + sqlite3VdbeSorterClose(p->db, pCx); + if( pCx->pBt ){ + sqlite3BtreeClose(pCx->pBt); + /* The pCx->pCursor will be close automatically, if it exists, by + ** the call above. */ + }else if( pCx->pCursor ){ + sqlite3BtreeCloseCursor(pCx->pCursor); + } #ifndef SQLITE_OMIT_VIRTUALTABLE - case CURTYPE_VTAB: { - sqlite3_vtab_cursor *pVCur = pCx->uc.pVCur; - const sqlite3_module *pModule = pVCur->pVtab->pModule; - assert( pVCur->pVtab->nRef>0 ); - pVCur->pVtab->nRef--; - pModule->xClose(pVCur); - break; - } -#endif + else if( pCx->pVtabCursor ){ + sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor; + const sqlite3_module *pModule = pVtabCursor->pVtab->pModule; + assert( pVtabCursor->pVtab->nRef>0 ); + pVtabCursor->pVtab->nRef--; + pModule->xClose(pVtabCursor); } +#endif } /* @@ -75595,6 +69576,8 @@ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS v->anExec = pFrame->anExec; #endif + v->aOnceFlag = pFrame->aOnceFlag; + v->nOnceFlag = pFrame->nOnceFlag; v->aOp = pFrame->aOp; v->nOp = pFrame->nOp; v->aMem = pFrame->aMem; @@ -75604,9 +69587,6 @@ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ v->db->lastRowid = pFrame->lastRowid; v->nChange = pFrame->nChange; v->db->nChange = pFrame->nDbChange; - sqlite3VdbeDeleteAuxData(v->db, &v->pAuxData, -1, 0); - v->pAuxData = pFrame->pAuxData; - pFrame->pAuxData = 0; return pFrame->pc; } @@ -75629,7 +69609,7 @@ static void closeAllCursors(Vdbe *p){ assert( p->nFrame==0 ); closeCursorsInFrame(p); if( p->aMem ){ - releaseMemArray(p->aMem, p->nMem); + releaseMemArray(&p->aMem[1], p->nMem); } while( p->pDelFrame ){ VdbeFrame *pDel = p->pDelFrame; @@ -75638,7 +69618,7 @@ static void closeAllCursors(Vdbe *p){ } /* Delete any auxdata allocations made by the VM */ - if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p->db, &p->pAuxData, -1, 0); + if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p, -1, 0); assert( p->pAuxData==0 ); } @@ -75654,7 +69634,7 @@ static void Cleanup(Vdbe *p){ int i; if( p->apCsr ) for(i=0; inCursor; i++) assert( p->apCsr[i]==0 ); if( p->aMem ){ - for(i=0; inMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); + for(i=1; i<=p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); } #endif @@ -75678,9 +69658,13 @@ SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){ sqlite3DbFree(db, p->aColName); n = nResColumn*COLNAME_N; p->nResColumn = (u16)nResColumn; - p->aColName = pColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n ); + p->aColName = pColName = (Mem*)sqlite3DbMallocZero(db, sizeof(Mem)*n ); if( p->aColName==0 ) return; - initMemArray(p->aColName, n, p->db, MEM_Null); + while( n-- > 0 ){ + pColName->flags = MEM_Null; + pColName->db = p->db; + pColName++; + } } /* @@ -75706,7 +69690,7 @@ SQLITE_PRIVATE int sqlite3VdbeSetColName( assert( vardb->mallocFailed ){ assert( !zName || xDel!=SQLITE_DYNAMIC ); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } assert( p->aColName!=0 ); pColName = &(p->aColName[idx+var*p->nResColumn]); @@ -75723,9 +69707,7 @@ SQLITE_PRIVATE int sqlite3VdbeSetColName( */ static int vdbeCommit(sqlite3 *db, Vdbe *p){ int i; - int nTrans = 0; /* Number of databases with an active write-transaction - ** that are candidates for a two-phase commit using a - ** master-journal */ + int nTrans = 0; /* Number of databases with an active write-transaction */ int rc = SQLITE_OK; int needXcommit = 0; @@ -75753,28 +69735,10 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( sqlite3BtreeIsInTrans(pBt) ){ - /* Whether or not a database might need a master journal depends upon - ** its journal mode (among other things). This matrix determines which - ** journal modes use a master journal and which do not */ - static const u8 aMJNeeded[] = { - /* DELETE */ 1, - /* PERSIST */ 1, - /* OFF */ 0, - /* TRUNCATE */ 1, - /* MEMORY */ 0, - /* WAL */ 0 - }; - Pager *pPager; /* Pager associated with pBt */ needXcommit = 1; + if( i!=1 ) nTrans++; sqlite3BtreeEnter(pBt); - pPager = sqlite3BtreePager(pBt); - if( db->aDb[i].safety_level!=PAGER_SYNCHRONOUS_OFF - && aMJNeeded[sqlite3PagerGetJournalMode(pPager)] - ){ - assert( i!=1 ); - nTrans++; - } - rc = sqlite3PagerExclusiveLock(pPager); + rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); sqlite3BtreeLeave(pBt); } } @@ -75832,6 +69796,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ #ifndef SQLITE_OMIT_DISKIO else{ sqlite3_vfs *pVfs = db->pVfs; + int needSync = 0; char *zMaster = 0; /* File-name for the master journal */ char const *zMainFile = sqlite3BtreeGetFilename(db->aDb[0].pBt); sqlite3_file *pMaster = 0; @@ -75843,7 +69808,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* Select a master journal file name */ nMainFile = sqlite3Strlen30(zMainFile); zMaster = sqlite3MPrintf(db, "%s-mjXXXXXX9XXz", zMainFile); - if( zMaster==0 ) return SQLITE_NOMEM_BKPT; + if( zMaster==0 ) return SQLITE_NOMEM; do { u32 iRandom; if( retryCount ){ @@ -75891,6 +69856,9 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ continue; /* Ignore TEMP and :memory: databases */ } assert( zFile[0]!=0 ); + if( !needSync && !sqlite3BtreeSyncDisabled(pBt) ){ + needSync = 1; + } rc = sqlite3OsWrite(pMaster, zFile, sqlite3Strlen30(zFile)+1, offset); offset += sqlite3Strlen30(zFile)+1; if( rc!=SQLITE_OK ){ @@ -75905,7 +69873,8 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* Sync the master journal file. If the IOCAP_SEQUENTIAL device ** flag is set this is not required. */ - if( 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL) + if( needSync + && 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL) && SQLITE_OK!=(rc = sqlite3OsSync(pMaster, SQLITE_SYNC_NORMAL)) ){ sqlite3OsCloseFree(pMaster); @@ -75941,7 +69910,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ ** doing this the directory is synced again before any individual ** transaction files are deleted. */ - rc = sqlite3OsDelete(pVfs, zMaster, 1); + rc = sqlite3OsDelete(pVfs, zMaster, needSync); sqlite3DbFree(db, zMaster); zMaster = 0; if( rc ){ @@ -76088,7 +70057,7 @@ SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ ){ p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; p->errorAction = OE_Abort; - sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); + sqlite3SetString(&p->zErrMsg, db, "FOREIGN KEY constraint failed"); return SQLITE_ERROR; } return SQLITE_OK; @@ -76128,9 +70097,10 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ ** one, or the complete transaction if there is no statement transaction. */ - if( db->mallocFailed ){ - p->rc = SQLITE_NOMEM_BKPT; + if( p->db->mallocFailed ){ + p->rc = SQLITE_NOMEM; } + if( p->aOnceFlag ) memset(p->aOnceFlag, 0, p->nOnceFlag); closeAllCursors(p); if( p->magic!=VDBE_MAGIC_RUN ){ return SQLITE_OK; @@ -76288,8 +70258,8 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ } p->magic = VDBE_MAGIC_HALT; checkActiveVdbeCnt(db); - if( db->mallocFailed ){ - p->rc = SQLITE_NOMEM_BKPT; + if( p->db->mallocFailed ){ + p->rc = SQLITE_NOMEM; } /* If the auto-commit flag is set to true, then any locks that were held @@ -76325,12 +70295,12 @@ SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p){ sqlite3 *db = p->db; int rc = p->rc; if( p->zErrMsg ){ - db->bBenignMalloc++; + u8 mallocFailed = db->mallocFailed; sqlite3BeginBenignMalloc(); if( db->pErr==0 ) db->pErr = sqlite3ValueNew(db); sqlite3ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE_UTF8, SQLITE_TRANSIENT); sqlite3EndBenignMalloc(); - db->bBenignMalloc--; + db->mallocFailed = mallocFailed; db->errCode = rc; }else{ sqlite3Error(db, rc); @@ -76442,7 +70412,7 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ } #endif p->iCurrentTime = 0; - p->magic = VDBE_MAGIC_RESET; + p->magic = VDBE_MAGIC_INIT; return p->rc & db->errMask; } @@ -76476,7 +70446,8 @@ SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe *p){ ** * the corresponding bit in argument mask is clear (where the first ** function parameter corresponds to bit 0 etc.). */ -SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, int mask){ +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(Vdbe *pVdbe, int iOp, int mask){ + AuxData **pp = &pVdbe->pAuxData; while( *pp ){ AuxData *pAux = *pp; if( (iOp<0) @@ -76487,7 +70458,7 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, pAux->xDelete(pAux->pAux); } *pp = pAux->pNext; - sqlite3DbFree(db, pAux); + sqlite3DbFree(pVdbe->db, pAux); }else{ pp= &pAux->pNext; } @@ -76506,21 +70477,18 @@ SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; int i; assert( p->db==0 || p->db==db ); + releaseMemArray(p->aVar, p->nVar); releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); for(pSub=p->pProgram; pSub; pSub=pNext){ pNext = pSub->pNext; vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); sqlite3DbFree(db, pSub); } - if( p->magic!=VDBE_MAGIC_INIT ){ - releaseMemArray(p->aVar, p->nVar); - for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); - sqlite3DbFree(db, p->azVar); - sqlite3DbFree(db, p->pFree); - } + for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); vdbeFreeOpArray(db, p->aOp, p->nOp); sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); + sqlite3DbFree(db, p->pFree); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS for(i=0; inScan; i++){ sqlite3DbFree(db, p->aScan[i].zName); @@ -76565,8 +70533,7 @@ static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ #endif assert( p->deferredMoveto ); assert( p->isTable ); - assert( p->eCurType==CURTYPE_BTREE ); - rc = sqlite3BtreeMovetoUnpacked(p->uc.pCursor, 0, p->movetoTarget, 0, &res); + rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); if( rc ) return rc; if( res!=0 ) return SQLITE_CORRUPT_BKPT; #ifdef SQLITE_TEST @@ -76586,10 +70553,9 @@ static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ */ static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ int isDifferentRow, rc; - assert( p->eCurType==CURTYPE_BTREE ); - assert( p->uc.pCursor!=0 ); - assert( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ); - rc = sqlite3BtreeCursorRestore(p->uc.pCursor, &isDifferentRow); + assert( p->pCursor!=0 ); + assert( sqlite3BtreeCursorHasMoved(p->pCursor) ); + rc = sqlite3BtreeCursorRestore(p->pCursor, &isDifferentRow); p->cacheStatus = CACHE_STALE; if( isDifferentRow ) p->nullRow = 1; return rc; @@ -76600,8 +70566,7 @@ static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ ** if need be. Return any I/O error from the restore operation. */ SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor *p){ - assert( p->eCurType==CURTYPE_BTREE ); - if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ + if( sqlite3BtreeCursorHasMoved(p->pCursor) ){ return handleMovedCursor(p); } return SQLITE_OK; @@ -76620,21 +70585,12 @@ SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor *p){ ** If the cursor is already pointing to the correct row and that row has ** not been deleted out from under the cursor, then this routine is a no-op. */ -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){ - VdbeCursor *p = *pp; - if( p->eCurType==CURTYPE_BTREE ){ - if( p->deferredMoveto ){ - int iMap; - if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 ){ - *pp = p->pAltCursor; - *piCol = iMap - 1; - return SQLITE_OK; - } - return handleDeferredMoveto(p); - } - if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ - return handleMovedCursor(p); - } +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor *p){ + if( p->deferredMoveto ){ + return handleDeferredMoveto(p); + } + if( p->pCursor && sqlite3BtreeCursorHasMoved(p->pCursor) ){ + return handleMovedCursor(p); } return SQLITE_OK; } @@ -76684,13 +70640,11 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){ /* ** Return the serial-type for the value stored in pMem. */ -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ int flags = pMem->flags; u32 n; - assert( pLen!=0 ); if( flags&MEM_Null ){ - *pLen = 0; return 0; } if( flags&MEM_Int ){ @@ -76704,23 +70658,15 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ u = i; } if( u<=127 ){ - if( (i&1)==i && file_format>=4 ){ - *pLen = 0; - return 8+(u32)u; - }else{ - *pLen = 1; - return 1; - } + return ((i&1)==i && file_format>=4) ? 8+(u32)u : 1; } - if( u<=32767 ){ *pLen = 2; return 2; } - if( u<=8388607 ){ *pLen = 3; return 3; } - if( u<=2147483647 ){ *pLen = 4; return 4; } - if( u<=MAX_6BYTE ){ *pLen = 6; return 5; } - *pLen = 8; + if( u<=32767 ) return 2; + if( u<=8388607 ) return 3; + if( u<=2147483647 ) return 4; + if( u<=MAX_6BYTE ) return 5; return 6; } if( flags&MEM_Real ){ - *pLen = 8; return 7; } assert( pMem->db->mallocFailed || flags&(MEM_Str|MEM_Blob) ); @@ -76729,46 +70675,20 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ if( flags & MEM_Zero ){ n += pMem->u.nZero; } - *pLen = n; return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } -/* -** The sizes for serial types less than 128 -*/ -static const u8 sqlite3SmallTypeSizes[] = { - /* 0 1 2 3 4 5 6 7 8 9 */ -/* 0 */ 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, -/* 10 */ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, -/* 20 */ 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, -/* 30 */ 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, -/* 40 */ 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, -/* 50 */ 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, -/* 60 */ 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, -/* 70 */ 29, 29, 30, 30, 31, 31, 32, 32, 33, 33, -/* 80 */ 34, 34, 35, 35, 36, 36, 37, 37, 38, 38, -/* 90 */ 39, 39, 40, 40, 41, 41, 42, 42, 43, 43, -/* 100 */ 44, 44, 45, 45, 46, 46, 47, 47, 48, 48, -/* 110 */ 49, 49, 50, 50, 51, 51, 52, 52, 53, 53, -/* 120 */ 54, 54, 55, 55, 56, 56, 57, 57 -}; - /* ** Return the length of the data corresponding to the supplied serial-type. */ SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32 serial_type){ - if( serial_type>=128 ){ + if( serial_type>=12 ){ return (serial_type-12)/2; }else{ - assert( serial_type<12 - || sqlite3SmallTypeSizes[serial_type]==(serial_type - 12)/2 ); - return sqlite3SmallTypeSizes[serial_type]; + static const u8 aSize[] = { 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, 0, 0 }; + return aSize[serial_type]; } } -SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8 serial_type){ - assert( serial_type<128 ); - return sqlite3SmallTypeSizes[serial_type]; -} /* ** If we are on an architecture with mixed-endian floating @@ -76850,7 +70770,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ }else{ v = pMem->u.i; } - len = i = sqlite3SmallTypeSizes[serial_type]; + len = i = sqlite3VdbeSerialTypeLen(serial_type); assert( i>0 ); do{ buf[--i] = (u8)(v&0xFF); @@ -76864,7 +70784,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ assert( pMem->n + ((pMem->flags & MEM_Zero)?pMem->u.nZero:0) == (int)sqlite3VdbeSerialTypeLen(serial_type) ); len = pMem->n; - if( len>0 ) memcpy(buf, pMem->z, len); + memcpy(buf, pMem->z, len); return len; } @@ -76967,10 +70887,6 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit ** twos-complement integer. */ pMem->u.i = FOUR_BYTE_INT(buf); -#ifdef __HP_cc - /* Work around a sign-extension bug in the HP compiler for HP/UX */ - if( buf[0]&0x80 ) pMem->u.i |= 0xffffffff80000000LL; -#endif pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 4; @@ -77088,7 +71004,6 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( pMem->db = pKeyInfo->db; /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ pMem->szMalloc = 0; - pMem->z = 0; d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); pMem++; if( (++u)>=p->nField ) break; @@ -77140,7 +71055,6 @@ static int vdbeRecordCompareDebug( /* mem1.u.i = 0; // not needed, here to silence compiler warning */ idx1 = getVarint32(aKey1, szHdr1); - if( szHdr1>98307 ) return SQLITE_CORRUPT; d1 = szHdr1; assert( pKeyInfo->nField+pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); assert( pKeyInfo->aSortOrder!=0 ); @@ -77269,86 +71183,24 @@ static int vdbeCompareMemString( v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); n2 = v2==0 ? 0 : c2.n; rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); - if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM_BKPT; sqlite3VdbeMemRelease(&c1); sqlite3VdbeMemRelease(&c2); + if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM; return rc; } } -/* -** The input pBlob is guaranteed to be a Blob that is not marked -** with MEM_Zero. Return true if it could be a zero-blob. -*/ -static int isAllZero(const char *z, int n){ - int i; - for(i=0; in; - int n2 = pB2->n; - - /* It is possible to have a Blob value that has some non-zero content - ** followed by zero content. But that only comes up for Blobs formed - ** by the OP_MakeRecord opcode, and such Blobs never get passed into - ** sqlite3MemCompare(). */ - assert( (pB1->flags & MEM_Zero)==0 || n1==0 ); - assert( (pB2->flags & MEM_Zero)==0 || n2==0 ); - - if( (pB1->flags|pB2->flags) & MEM_Zero ){ - if( pB1->flags & pB2->flags & MEM_Zero ){ - return pB1->u.nZero - pB2->u.nZero; - }else if( pB1->flags & MEM_Zero ){ - if( !isAllZero(pB2->z, pB2->n) ) return -1; - return pB1->u.nZero - n2; - }else{ - if( !isAllZero(pB1->z, pB1->n) ) return +1; - return n1 - pB2->u.nZero; - } - } - c = memcmp(pB1->z, pB2->z, n1>n2 ? n2 : n1); + int c = memcmp(pB1->z, pB2->z, pB1->n>pB2->n ? pB2->n : pB1->n); if( c ) return c; - return n1 - n2; + return pB1->n - pB2->n; } -/* -** Do a comparison between a 64-bit signed integer and a 64-bit floating-point -** number. Return negative, zero, or positive if the first (i64) is less than, -** equal to, or greater than the second (double). -*/ -static int sqlite3IntFloatCompare(i64 i, double r){ - if( sizeof(LONGDOUBLE_TYPE)>8 ){ - LONGDOUBLE_TYPE x = (LONGDOUBLE_TYPE)i; - if( xr ) return +1; - return 0; - }else{ - i64 y; - double s; - if( r<-9223372036854775808.0 ) return +1; - if( r>9223372036854775807.0 ) return -1; - y = (i64)r; - if( iy ){ - if( y==SMALLEST_INT64 && r>0.0 ) return -1; - return +1; - } - s = (double)i; - if( sr ) return +1; - return 0; - } -} /* ** Compare the values contained by the two memory cells, returning @@ -77375,34 +71227,34 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C return (f2&MEM_Null) - (f1&MEM_Null); } - /* At least one of the two values is a number + /* If one value is a number and the other is not, the number is less. + ** If both are numbers, compare as reals if one is a real, or as integers + ** if both values are integers. */ if( combined_flags&(MEM_Int|MEM_Real) ){ + double r1, r2; if( (f1 & f2 & MEM_Int)!=0 ){ if( pMem1->u.i < pMem2->u.i ) return -1; - if( pMem1->u.i > pMem2->u.i ) return +1; - return 0; - } - if( (f1 & f2 & MEM_Real)!=0 ){ - if( pMem1->u.r < pMem2->u.r ) return -1; - if( pMem1->u.r > pMem2->u.r ) return +1; + if( pMem1->u.i > pMem2->u.i ) return 1; return 0; } - if( (f1&MEM_Int)!=0 ){ - if( (f2&MEM_Real)!=0 ){ - return sqlite3IntFloatCompare(pMem1->u.i, pMem2->u.r); - }else{ - return -1; - } - } if( (f1&MEM_Real)!=0 ){ - if( (f2&MEM_Int)!=0 ){ - return -sqlite3IntFloatCompare(pMem2->u.i, pMem1->u.r); - }else{ - return -1; - } + r1 = pMem1->u.r; + }else if( (f1&MEM_Int)!=0 ){ + r1 = (double)pMem1->u.i; + }else{ + return 1; + } + if( (f2&MEM_Real)!=0 ){ + r2 = pMem2->u.r; + }else if( (f2&MEM_Int)!=0 ){ + r2 = (double)pMem2->u.i; + }else{ + return -1; } - return +1; + if( r1r2 ) return 1; + return 0; } /* If one value is a string and the other is a blob, the string is less. @@ -77416,7 +71268,7 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C return -1; } - assert( pMem1->enc==pMem2->enc || pMem1->db->mallocFailed ); + assert( pMem1->enc==pMem2->enc ); assert( pMem1->enc==SQLITE_UTF8 || pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); @@ -77548,13 +71400,18 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( if( pRhs->flags & MEM_Int ){ serial_type = aKey1[idx1]; testcase( serial_type==12 ); - if( serial_type>=10 ){ + if( serial_type>=12 ){ rc = +1; }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ + double rhs = (double)pRhs->u.i; sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); - rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r); + if( mem1.u.rrhs ){ + rc = +1; + } }else{ i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); i64 rhs = pRhs->u.i; @@ -77569,24 +71426,23 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is real */ else if( pRhs->flags & MEM_Real ){ serial_type = aKey1[idx1]; - if( serial_type>=10 ){ - /* Serial types 12 or greater are strings and blobs (greater than - ** numbers). Types 10 and 11 are currently "reserved for future - ** use", so it doesn't really matter what the results of comparing - ** them to numberic values are. */ + if( serial_type>=12 ){ rc = +1; }else if( serial_type==0 ){ rc = -1; }else{ + double rhs = pRhs->u.r; + double lhs; sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); if( serial_type==7 ){ - if( mem1.u.ru.r ){ - rc = -1; - }else if( mem1.u.r>pRhs->u.r ){ - rc = +1; - } + lhs = mem1.u.r; }else{ - rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r); + lhs = (double)mem1.u.i; + } + if( lhsrhs ){ + rc = +1; } } } @@ -77624,7 +71480,6 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is a blob */ else if( pRhs->flags & MEM_Blob ){ - assert( (pRhs->flags & MEM_Zero)==0 || pRhs->n==0 ); getVarint32(&aKey1[idx1], serial_type); testcase( serial_type==12 ); if( serial_type<12 || (serial_type & 0x01) ){ @@ -77636,12 +71491,6 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( if( (d1+nStr) > (unsigned)nKey1 ){ pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ - }else if( pRhs->flags & MEM_Zero ){ - if( !isAllZero((const char*)&aKey1[d1],nStr) ){ - rc = 1; - }else{ - rc = nStr - pRhs->u.nZero; - } }else{ int nCmp = MIN(nStr, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); @@ -77683,7 +71532,6 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) || pKeyInfo->db->mallocFailed ); - pPKey2->eqSeen = 1; return pPKey2->default_rc; } SQLITE_PRIVATE int sqlite3VdbeRecordCompare( @@ -77712,7 +71560,7 @@ static int vdbeRecordCompareInt( int res; u32 y; u64 x; - i64 v; + i64 v = pPKey2->aMem[0].u.i; i64 lhs; vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); @@ -77771,7 +71619,6 @@ static int vdbeRecordCompareInt( return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); } - v = pPKey2->aMem[0].u.i; if( v>lhs ){ res = pPKey2->r1; }else if( vdefault_rc in this case. */ res = pPKey2->default_rc; - pPKey2->eqSeen = 1; } assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) ); @@ -77805,7 +71651,6 @@ static int vdbeRecordCompareString( int serial_type; int res; - assert( pPKey2->aMem[0].flags & MEM_Str ); vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); getVarint32(&aKey1[1], serial_type); if( serial_type<12 ){ @@ -77832,7 +71677,6 @@ static int vdbeRecordCompareString( res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ res = pPKey2->default_rc; - pPKey2->eqSeen = 1; } }else if( res>0 ){ res = pPKey2->r2; @@ -77918,7 +71762,8 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ ** this code can safely assume that nCellKey is 32-bits */ assert( sqlite3BtreeCursorIsValid(pCur) ); - nCellKey = sqlite3BtreePayloadSize(pCur); + VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); + assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey ); /* Read in the complete content of the index entry */ @@ -77950,7 +71795,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ if( unlikely(typeRowid<1 || typeRowid>9 || typeRowid==7) ){ goto idx_rowid_corruption; } - lenRowid = sqlite3SmallTypeSizes[typeRowid]; + lenRowid = sqlite3VdbeSerialTypeLen(typeRowid); testcase( (u32)m.n==szHdr+lenRowid ); if( unlikely((u32)m.npCursor; Mem m; - assert( pC->eCurType==CURTYPE_BTREE ); - pCur = pC->uc.pCursor; assert( sqlite3BtreeCursorIsValid(pCur) ); - nCellKey = sqlite3BtreePayloadSize(pCur); + VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); + assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ /* nCellKey will always be between 0 and 0xffffffff because of the way ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ if( nCellKey<=0 || nCellKey>0x7fffffff ){ @@ -78003,7 +71847,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( return SQLITE_CORRUPT_BKPT; } sqlite3VdbeMemInit(&m, db, 0); - rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, 1, &m); + rc = sqlite3VdbeMemFromBtree(pC->pCursor, 0, (u32)nCellKey, 1, &m); if( rc ){ return rc; } @@ -78099,100 +71943,14 @@ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ ** in memory obtained from sqlite3DbMalloc). */ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ - if( pVtab->zErrMsg ){ - sqlite3 *db = p->db; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); - sqlite3_free(pVtab->zErrMsg); - pVtab->zErrMsg = 0; - } + sqlite3 *db = p->db; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK - -/* -** If the second argument is not NULL, release any allocations associated -** with the memory cells in the p->aMem[] array. Also free the UnpackedRecord -** structure itself, using sqlite3DbFree(). -** -** This function is used to free UnpackedRecord structures allocated by -** the vdbeUnpackRecord() function found in vdbeapi.c. -*/ -static void vdbeFreeUnpacked(sqlite3 *db, UnpackedRecord *p){ - if( p ){ - int i; - for(i=0; inField; i++){ - Mem *pMem = &p->aMem[i]; - if( pMem->zMalloc ) sqlite3VdbeMemRelease(pMem); - } - sqlite3DbFree(db, p); - } -} -#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ - -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK -/* -** Invoke the pre-update hook. If this is an UPDATE or DELETE pre-update call, -** then cursor passed as the second argument should point to the row about -** to be update or deleted. If the application calls sqlite3_preupdate_old(), -** the required value will be read from the row the cursor points to. -*/ -SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( - Vdbe *v, /* Vdbe pre-update hook is invoked by */ - VdbeCursor *pCsr, /* Cursor to grab old.* values from */ - int op, /* SQLITE_INSERT, UPDATE or DELETE */ - const char *zDb, /* Database name */ - Table *pTab, /* Modified table */ - i64 iKey1, /* Initial key value */ - int iReg /* Register for new.* record */ -){ - sqlite3 *db = v->db; - i64 iKey2; - PreUpdate preupdate; - const char *zTbl = pTab->zName; - static const u8 fakeSortOrder = 0; - - assert( db->pPreUpdate==0 ); - memset(&preupdate, 0, sizeof(PreUpdate)); - if( op==SQLITE_UPDATE ){ - iKey2 = v->aMem[iReg].u.i; - }else{ - iKey2 = iKey1; - } - - assert( pCsr->nField==pTab->nCol - || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1) - ); - - preupdate.v = v; - preupdate.pCsr = pCsr; - preupdate.op = op; - preupdate.iNewReg = iReg; - preupdate.keyinfo.db = db; - preupdate.keyinfo.enc = ENC(db); - preupdate.keyinfo.nField = pTab->nCol; - preupdate.keyinfo.aSortOrder = (u8*)&fakeSortOrder; - preupdate.iKey1 = iKey1; - preupdate.iKey2 = iKey2; - preupdate.pTab = pTab; - - db->pPreUpdate = &preupdate; - db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2); - db->pPreUpdate = 0; - sqlite3DbFree(db, preupdate.aRecord); - vdbeFreeUnpacked(db, preupdate.pUnpacked); - vdbeFreeUnpacked(db, preupdate.pNewUnpacked); - if( preupdate.aNew ){ - int i; - for(i=0; inField; i++){ - sqlite3VdbeMemRelease(&preupdate.aNew[i]); - } - sqlite3DbFree(db, preupdate.aNew); - } -} -#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ - /************** End of vdbeaux.c *********************************************/ /************** Begin file vdbeapi.c *****************************************/ /* @@ -78210,8 +71968,6 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( ** This file contains code use to implement APIs that are part of the ** VDBE. */ -/* #include "sqliteInt.h" */ -/* #include "vdbeInt.h" */ #ifndef SQLITE_OMIT_DEPRECATED /* @@ -78222,7 +71978,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( ** collating sequences are registered or if an authorizer function is ** added or changed. */ -SQLITE_API int sqlite3_expired(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_expired(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; return p==0 || p->expired; } @@ -78250,38 +72006,6 @@ static int vdbeSafetyNotNull(Vdbe *p){ } } -#ifndef SQLITE_OMIT_TRACE -/* -** Invoke the profile callback. This routine is only called if we already -** know that the profile callback is defined and needs to be invoked. -*/ -static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ - sqlite3_int64 iNow; - sqlite3_int64 iElapse; - assert( p->startTime>0 ); - assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 ); - assert( db->init.busy==0 ); - assert( p->zSql!=0 ); - sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); - iElapse = (iNow - p->startTime)*1000000; - if( db->xProfile ){ - db->xProfile(db->pProfileArg, p->zSql, iElapse); - } - if( db->mTrace & SQLITE_TRACE_PROFILE ){ - db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse); - } - p->startTime = 0; -} -/* -** The checkProfileCallback(DB,P) macro checks to see if a profile callback -** is needed, and it invokes the callback if it is needed. -*/ -# define checkProfileCallback(DB,P) \ - if( ((P)->startTime)>0 ){ invokeProfileCallback(DB,P); } -#else -# define checkProfileCallback(DB,P) /*no-op*/ -#endif - /* ** The following routine destroys a virtual machine that is created by ** the sqlite3_compile() routine. The integer returned is an SQLITE_ @@ -78291,7 +72015,7 @@ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ ** This routine sets the error code and string returned by ** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). */ -SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_finalize(sqlite3_stmt *pStmt){ int rc; if( pStmt==0 ){ /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL @@ -78302,7 +72026,6 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ sqlite3 *db = v->db; if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; sqlite3_mutex_enter(db->mutex); - checkProfileCallback(db, v); rc = sqlite3VdbeFinalize(v); rc = sqlite3ApiExit(db, rc); sqlite3LeaveMutexAndCloseZombie(db); @@ -78318,20 +72041,18 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ ** This routine sets the error code and string returned by ** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). */ -SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt){ int rc; if( pStmt==0 ){ rc = SQLITE_OK; }else{ Vdbe *v = (Vdbe*)pStmt; - sqlite3 *db = v->db; - sqlite3_mutex_enter(db->mutex); - checkProfileCallback(db, v); + sqlite3_mutex_enter(v->db->mutex); rc = sqlite3VdbeReset(v); sqlite3VdbeRewind(v); - assert( (rc & (db->errMask))==rc ); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); + assert( (rc & (v->db->errMask))==rc ); + rc = sqlite3ApiExit(v->db, rc); + sqlite3_mutex_leave(v->db->mutex); } return rc; } @@ -78339,7 +72060,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt){ /* ** Set all the parameters in the compiled SQL statement to NULL. */ -SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_clear_bindings(sqlite3_stmt *pStmt){ int i; int rc = SQLITE_OK; Vdbe *p = (Vdbe*)pStmt; @@ -78363,49 +72084,42 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ ** The following routines extract information from a Mem or sqlite3_value ** structure. */ -SQLITE_API const void *sqlite3_value_blob(sqlite3_value *pVal){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_blob(sqlite3_value *pVal){ Mem *p = (Mem*)pVal; if( p->flags & (MEM_Blob|MEM_Str) ){ - if( ExpandBlob(p)!=SQLITE_OK ){ - assert( p->flags==MEM_Null && p->z==0 ); - return 0; - } + sqlite3VdbeMemExpandBlob(p); p->flags |= MEM_Blob; return p->n ? p->z : 0; }else{ return sqlite3_value_text(pVal); } } -SQLITE_API int sqlite3_value_bytes(sqlite3_value *pVal){ +SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes(sqlite3_value *pVal){ return sqlite3ValueBytes(pVal, SQLITE_UTF8); } -SQLITE_API int sqlite3_value_bytes16(sqlite3_value *pVal){ +SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes16(sqlite3_value *pVal){ return sqlite3ValueBytes(pVal, SQLITE_UTF16NATIVE); } -SQLITE_API double sqlite3_value_double(sqlite3_value *pVal){ +SQLITE_API double SQLITE_STDCALL sqlite3_value_double(sqlite3_value *pVal){ return sqlite3VdbeRealValue((Mem*)pVal); } -SQLITE_API int sqlite3_value_int(sqlite3_value *pVal){ +SQLITE_API int SQLITE_STDCALL sqlite3_value_int(sqlite3_value *pVal){ return (int)sqlite3VdbeIntValue((Mem*)pVal); } -SQLITE_API sqlite_int64 sqlite3_value_int64(sqlite3_value *pVal){ +SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_value_int64(sqlite3_value *pVal){ return sqlite3VdbeIntValue((Mem*)pVal); } -SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value *pVal){ - Mem *pMem = (Mem*)pVal; - return ((pMem->flags & MEM_Subtype) ? pMem->eSubtype : 0); -} -SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value *pVal){ +SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_value_text(sqlite3_value *pVal){ return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_value_text16(sqlite3_value* pVal){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16(sqlite3_value* pVal){ return sqlite3ValueText(pVal, SQLITE_UTF16NATIVE); } -SQLITE_API const void *sqlite3_value_text16be(sqlite3_value *pVal){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16be(sqlite3_value *pVal){ return sqlite3ValueText(pVal, SQLITE_UTF16BE); } -SQLITE_API const void *sqlite3_value_text16le(sqlite3_value *pVal){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16le(sqlite3_value *pVal){ return sqlite3ValueText(pVal, SQLITE_UTF16LE); } #endif /* SQLITE_OMIT_UTF16 */ @@ -78413,7 +72127,7 @@ SQLITE_API const void *sqlite3_value_text16le(sqlite3_value *pVal){ ** fundamental datatypes: 64-bit signed integer 64-bit IEEE floating ** point number string BLOB NULL */ -SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ +SQLITE_API int SQLITE_STDCALL sqlite3_value_type(sqlite3_value* pVal){ static const u8 aType[] = { SQLITE_BLOB, /* 0x00 */ SQLITE_NULL, /* 0x01 */ @@ -78451,36 +72165,6 @@ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ return aType[pVal->flags&MEM_AffMask]; } -/* Make a copy of an sqlite3_value object -*/ -SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value *pOrig){ - sqlite3_value *pNew; - if( pOrig==0 ) return 0; - pNew = sqlite3_malloc( sizeof(*pNew) ); - if( pNew==0 ) return 0; - memset(pNew, 0, sizeof(*pNew)); - memcpy(pNew, pOrig, MEMCELLSIZE); - pNew->flags &= ~MEM_Dyn; - pNew->db = 0; - if( pNew->flags&(MEM_Str|MEM_Blob) ){ - pNew->flags &= ~(MEM_Static|MEM_Dyn); - pNew->flags |= MEM_Ephem; - if( sqlite3VdbeMemMakeWriteable(pNew)!=SQLITE_OK ){ - sqlite3ValueFree(pNew); - pNew = 0; - } - } - return pNew; -} - -/* Destroy an sqlite3_value object previously obtained from -** sqlite3_value_dup(). -*/ -SQLITE_API void sqlite3_value_free(sqlite3_value *pOld){ - sqlite3ValueFree(pOld); -} - - /**************************** sqlite3_result_ ******************************* ** The following routines are used by user-defined functions to specify ** the function result. @@ -78519,7 +72203,7 @@ static int invokeValueDestructor( if( pCtx ) sqlite3_result_error_toobig(pCtx); return SQLITE_TOOBIG; } -SQLITE_API void sqlite3_result_blob( +SQLITE_API void SQLITE_STDCALL sqlite3_result_blob( sqlite3_context *pCtx, const void *z, int n, @@ -78529,7 +72213,7 @@ SQLITE_API void sqlite3_result_blob( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, 0, xDel); } -SQLITE_API void sqlite3_result_blob64( +SQLITE_API void SQLITE_STDCALL sqlite3_result_blob64( sqlite3_context *pCtx, const void *z, sqlite3_uint64 n, @@ -78543,43 +72227,37 @@ SQLITE_API void sqlite3_result_blob64( setResultStrOrError(pCtx, z, (int)n, 0, xDel); } } -SQLITE_API void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_double(sqlite3_context *pCtx, double rVal){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetDouble(pCtx->pOut, rVal); } -SQLITE_API void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; pCtx->fErrorOrAux = 1; sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; pCtx->fErrorOrAux = 1; sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); } #endif -SQLITE_API void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_int(sqlite3_context *pCtx, int iVal){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetInt64(pCtx->pOut, (i64)iVal); } -SQLITE_API void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetInt64(pCtx->pOut, iVal); } -SQLITE_API void sqlite3_result_null(sqlite3_context *pCtx){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_null(sqlite3_context *pCtx){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); } -SQLITE_API void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ - Mem *pOut = pCtx->pOut; - assert( sqlite3_mutex_held(pOut->db->mutex) ); - pOut->eSubtype = eSubtype & 0xff; - pOut->flags |= MEM_Subtype; -} -SQLITE_API void sqlite3_result_text( +SQLITE_API void SQLITE_STDCALL sqlite3_result_text( sqlite3_context *pCtx, const char *z, int n, @@ -78588,7 +72266,7 @@ SQLITE_API void sqlite3_result_text( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); } -SQLITE_API void sqlite3_result_text64( +SQLITE_API void SQLITE_STDCALL sqlite3_result_text64( sqlite3_context *pCtx, const char *z, sqlite3_uint64 n, @@ -78605,7 +72283,7 @@ SQLITE_API void sqlite3_result_text64( } } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API void sqlite3_result_text16( +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16( sqlite3_context *pCtx, const void *z, int n, @@ -78614,7 +72292,7 @@ SQLITE_API void sqlite3_result_text16( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); } -SQLITE_API void sqlite3_result_text16be( +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16be( sqlite3_context *pCtx, const void *z, int n, @@ -78623,7 +72301,7 @@ SQLITE_API void sqlite3_result_text16be( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); } -SQLITE_API void sqlite3_result_text16le( +SQLITE_API void SQLITE_STDCALL sqlite3_result_text16le( sqlite3_context *pCtx, const void *z, int n, @@ -78633,24 +72311,15 @@ SQLITE_API void sqlite3_result_text16le( setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); } #endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemCopy(pCtx->pOut, pValue); } -SQLITE_API void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetZeroBlob(pCtx->pOut, n); } -SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ - Mem *pOut = pCtx->pOut; - assert( sqlite3_mutex_held(pOut->db->mutex) ); - if( n>(u64)pOut->db->aLimit[SQLITE_LIMIT_LENGTH] ){ - return SQLITE_TOOBIG; - } - sqlite3VdbeMemSetZeroBlob(pCtx->pOut, (int)n); - return SQLITE_OK; -} -SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ pCtx->isError = errCode; pCtx->fErrorOrAux = 1; #ifdef SQLITE_DEBUG @@ -78663,7 +72332,7 @@ SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ } /* Force an SQLITE_TOOBIG error. */ -SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_toobig(sqlite3_context *pCtx){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_TOOBIG; pCtx->fErrorOrAux = 1; @@ -78672,12 +72341,12 @@ SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ } /* An SQLITE_NOMEM error. */ -SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ +SQLITE_API void SQLITE_STDCALL sqlite3_result_error_nomem(sqlite3_context *pCtx){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); - pCtx->isError = SQLITE_NOMEM_BKPT; + pCtx->isError = SQLITE_NOMEM; pCtx->fErrorOrAux = 1; - sqlite3OomFault(pCtx->pOut->db); + pCtx->pOut->db->mallocFailed = 1; } /* @@ -78696,7 +72365,7 @@ static int doWalCallbacks(sqlite3 *db){ nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); sqlite3BtreeLeave(pBt); if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){ - rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zDbSName, nEntry); + rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry); } } } @@ -78704,7 +72373,6 @@ static int doWalCallbacks(sqlite3 *db){ return rc; } - /* ** Execute the statement pStmt, either until a row of data is ready, the ** statement is completely executed or an error occurs. @@ -78751,7 +72419,7 @@ static int sqlite3Step(Vdbe *p){ db = p->db; if( db->mallocFailed ){ p->rc = SQLITE_NOMEM; - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } if( p->pc<=0 && p->expired ){ @@ -78773,11 +72441,8 @@ static int sqlite3Step(Vdbe *p){ ); #ifndef SQLITE_OMIT_TRACE - if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0) - && !db->init.busy && p->zSql ){ + if( db->xProfile && !db->init.busy ){ sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); - }else{ - assert( p->startTime==0 ); } #endif @@ -78801,8 +72466,13 @@ static int sqlite3Step(Vdbe *p){ } #ifndef SQLITE_OMIT_TRACE - /* If the statement completed successfully, invoke the profile callback */ - if( rc!=SQLITE_ROW ) checkProfileCallback(db, p); + /* Invoke the profile callback if there is one + */ + if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){ + sqlite3_int64 iNow; + sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); + db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); + } #endif if( rc==SQLITE_DONE ){ @@ -78815,7 +72485,7 @@ static int sqlite3Step(Vdbe *p){ db->errCode = rc; if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){ - p->rc = SQLITE_NOMEM_BKPT; + p->rc = SQLITE_NOMEM; } end_of_step: /* At this point local variable rc holds the value that should be @@ -78826,7 +72496,7 @@ static int sqlite3Step(Vdbe *p){ ** were called on statement p. */ assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR - || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE + || rc==SQLITE_BUSY || rc==SQLITE_MISUSE ); assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp ); if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ @@ -78844,7 +72514,7 @@ static int sqlite3Step(Vdbe *p){ ** sqlite3Step() to do most of the work. If a schema error occurs, ** call sqlite3Reprepare() and try again. */ -SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt *pStmt){ int rc = SQLITE_OK; /* Result from sqlite3Step() */ int rc2 = SQLITE_OK; /* Result from sqlite3Reprepare() */ Vdbe *v = (Vdbe*)pStmt; /* the prepared statement */ @@ -78882,7 +72552,7 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ v->rc = rc2; } else { v->zErrMsg = 0; - v->rc = rc = SQLITE_NOMEM_BKPT; + v->rc = rc = SQLITE_NOMEM; } } rc = sqlite3ApiExit(db, rc); @@ -78895,7 +72565,7 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ ** Extract the user data from a sqlite3_context structure and return a ** pointer to it. */ -SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ +SQLITE_API void *SQLITE_STDCALL sqlite3_user_data(sqlite3_context *p){ assert( p && p->pFunc ); return p->pFunc->pUserData; } @@ -78910,8 +72580,8 @@ SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ ** sqlite3_create_function16() routines that originally registered the ** application defined function. */ -SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ - assert( p && p->pOut ); +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_context_db_handle(sqlite3_context *p){ + assert( p && p->pFunc ); return p->pOut->db; } @@ -78986,8 +72656,8 @@ static SQLITE_NOINLINE void *createAggContext(sqlite3_context *p, int nByte){ ** context is allocated on the first call. Subsequent calls return the ** same context that was returned on prior calls. */ -SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ - assert( p && p->pFunc && p->pFunc->xFinalize ); +SQLITE_API void *SQLITE_STDCALL sqlite3_aggregate_context(sqlite3_context *p, int nByte){ + assert( p && p->pFunc && p->pFunc->xStep ); assert( sqlite3_mutex_held(p->pOut->db->mutex) ); testcase( nByte<0 ); if( (p->pMem->flags & MEM_Agg)==0 ){ @@ -79001,7 +72671,7 @@ SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ ** Return the auxiliary data pointer, if any, for the iArg'th argument to ** the user-function defined by pCtx. */ -SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ +SQLITE_API void *SQLITE_STDCALL sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ AuxData *pAuxData; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); @@ -79022,7 +72692,7 @@ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ ** argument to the user-function defined by pCtx. Any previous value is ** deleted by calling the delete function specified when it was set. */ -SQLITE_API void sqlite3_set_auxdata( +SQLITE_API void SQLITE_STDCALL sqlite3_set_auxdata( sqlite3_context *pCtx, int iArg, void *pAux, @@ -79077,8 +72747,8 @@ SQLITE_API void sqlite3_set_auxdata( ** implementations should keep their own counts within their aggregate ** context. */ -SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ - assert( p && p->pMem && p->pFunc && p->pFunc->xFinalize ); +SQLITE_API int SQLITE_STDCALL sqlite3_aggregate_count(sqlite3_context *p){ + assert( p && p->pMem && p->pFunc && p->pFunc->xStep ); return p->pMem->n; } #endif @@ -79086,7 +72756,7 @@ SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ /* ** Return the number of columns in the result set for the statement pStmt. */ -SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_column_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; return pVm ? pVm->nResColumn : 0; } @@ -79095,7 +72765,7 @@ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ ** Return the number of values available from the current row of the ** currently executing statement pStmt. */ -SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; if( pVm==0 || pVm->pResultSet==0 ) return 0; return pVm->nResColumn; @@ -79120,19 +72790,18 @@ static const Mem *columnNullValue(void){ #endif = { /* .u = */ {0}, - /* .flags = */ (u16)MEM_Null, - /* .enc = */ (u8)0, - /* .eSubtype = */ (u8)0, - /* .n = */ (int)0, - /* .z = */ (char*)0, - /* .zMalloc = */ (char*)0, - /* .szMalloc = */ (int)0, - /* .uTemp = */ (u32)0, - /* .db = */ (sqlite3*)0, - /* .xDel = */ (void(*)(void*))0, + /* .flags = */ MEM_Null, + /* .enc = */ 0, + /* .n = */ 0, + /* .z = */ 0, + /* .zMalloc = */ 0, + /* .szMalloc = */ 0, + /* .iPadding1 = */ 0, + /* .db = */ 0, + /* .xDel = */ 0, #ifdef SQLITE_DEBUG - /* .pScopyFrom = */ (Mem*)0, - /* .pFiller = */ (void*)0, + /* .pScopyFrom = */ 0, + /* .pFiller = */ 0, #endif }; return &nullMem; @@ -79149,13 +72818,14 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ Mem *pOut; pVm = (Vdbe *)pStmt; - if( pVm==0 ) return (Mem*)columnNullValue(); - assert( pVm->db ); - sqlite3_mutex_enter(pVm->db->mutex); - if( pVm->pResultSet!=0 && inResColumn && i>=0 ){ + if( pVm && pVm->pResultSet!=0 && inResColumn && i>=0 ){ + sqlite3_mutex_enter(pVm->db->mutex); pOut = &pVm->pResultSet[i]; }else{ - sqlite3Error(pVm->db, SQLITE_RANGE); + if( pVm && ALWAYS(pVm->db) ){ + sqlite3_mutex_enter(pVm->db->mutex); + sqlite3Error(pVm->db, SQLITE_RANGE); + } pOut = (Mem*)columnNullValue(); } return pOut; @@ -79188,8 +72858,6 @@ static void columnMallocFailure(sqlite3_stmt *pStmt) */ Vdbe *p = (Vdbe *)pStmt; if( p ){ - assert( p->db!=0 ); - assert( sqlite3_mutex_held(p->db->mutex) ); p->rc = sqlite3ApiExit(p->db, p->rc); sqlite3_mutex_leave(p->db->mutex); } @@ -79199,7 +72867,7 @@ static void columnMallocFailure(sqlite3_stmt *pStmt) ** The following routines are used to access elements of the current row ** in the result set. */ -SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ const void *val; val = sqlite3_value_blob( columnMem(pStmt,i) ); /* Even though there is no encoding conversion, value_blob() might @@ -79209,37 +72877,37 @@ SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ columnMallocFailure(pStmt); return val; } -SQLITE_API int sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ +SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ int val = sqlite3_value_bytes( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt *pStmt, int i){ +SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes16(sqlite3_stmt *pStmt, int i){ int val = sqlite3_value_bytes16( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API double sqlite3_column_double(sqlite3_stmt *pStmt, int i){ +SQLITE_API double SQLITE_STDCALL sqlite3_column_double(sqlite3_stmt *pStmt, int i){ double val = sqlite3_value_double( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API int sqlite3_column_int(sqlite3_stmt *pStmt, int i){ +SQLITE_API int SQLITE_STDCALL sqlite3_column_int(sqlite3_stmt *pStmt, int i){ int val = sqlite3_value_int( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API sqlite_int64 sqlite3_column_int64(sqlite3_stmt *pStmt, int i){ +SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_column_int64(sqlite3_stmt *pStmt, int i){ sqlite_int64 val = sqlite3_value_int64( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt *pStmt, int i){ +SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_column_text(sqlite3_stmt *pStmt, int i){ const unsigned char *val = sqlite3_value_text( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt *pStmt, int i){ +SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_column_value(sqlite3_stmt *pStmt, int i){ Mem *pOut = columnMem(pStmt, i); if( pOut->flags&MEM_Static ){ pOut->flags &= ~MEM_Static; @@ -79249,13 +72917,13 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt *pStmt, int i){ return (sqlite3_value *)pOut; } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt *pStmt, int i){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_text16(sqlite3_stmt *pStmt, int i){ const void *val = sqlite3_value_text16( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } #endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ +SQLITE_API int SQLITE_STDCALL sqlite3_column_type(sqlite3_stmt *pStmt, int i){ int iType = sqlite3_value_type( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return iType; @@ -79307,7 +72975,7 @@ static const void *columnName( ** is the case, clear the mallocFailed flag and return NULL. */ if( db->mallocFailed ){ - sqlite3OomClear(db); + db->mallocFailed = 0; ret = 0; } sqlite3_mutex_leave(db->mutex); @@ -79319,12 +72987,12 @@ static const void *columnName( ** Return the name of the Nth column of the result set returned by SQL ** statement pStmt. */ -SQLITE_API const char *sqlite3_column_name(sqlite3_stmt *pStmt, int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_name(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_NAME); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_NAME); } @@ -79344,12 +73012,12 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ ** Return the column declaration type (if applicable) of the 'i'th column ** of the result set of SQL statement pStmt. */ -SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DECLTYPE); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DECLTYPE); } @@ -79362,12 +73030,12 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ ** NULL is returned if the result column is an expression or constant or ** anything else which is not an unambiguous reference to a database column. */ -SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DATABASE); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DATABASE); } @@ -79378,12 +73046,12 @@ SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N ** NULL is returned if the result column is an expression or constant or ** anything else which is not an unambiguous reference to a database column. */ -SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_TABLE); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_TABLE); } @@ -79394,12 +73062,12 @@ SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ ** NULL is returned if the result column is an expression or constant or ** anything else which is not an unambiguous reference to a database column. */ -SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_COLUMN); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ +SQLITE_API const void *SQLITE_STDCALL sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ return columnName( pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_COLUMN); } @@ -79500,19 +73168,16 @@ static int bindText( /* ** Bind a blob value to an SQL statement variable. */ -SQLITE_API int sqlite3_bind_blob( +SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob( sqlite3_stmt *pStmt, int i, const void *zData, int nData, void (*xDel)(void*) ){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( nData<0 ) return SQLITE_MISUSE_BKPT; -#endif return bindText(pStmt, i, zData, nData, xDel, 0); } -SQLITE_API int sqlite3_bind_blob64( +SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob64( sqlite3_stmt *pStmt, int i, const void *zData, @@ -79526,7 +73191,7 @@ SQLITE_API int sqlite3_bind_blob64( return bindText(pStmt, i, zData, (int)nData, xDel, 0); } } -SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, i); @@ -79536,10 +73201,10 @@ SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ } return rc; } -SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ return sqlite3_bind_int64(p, i, (i64)iValue); } -SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ int rc; Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, i); @@ -79549,7 +73214,7 @@ SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValu } return rc; } -SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ int rc; Vdbe *p = (Vdbe*)pStmt; rc = vdbeUnbind(p, i); @@ -79558,7 +73223,7 @@ SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ } return rc; } -SQLITE_API int sqlite3_bind_text( +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text( sqlite3_stmt *pStmt, int i, const char *zData, @@ -79567,7 +73232,7 @@ SQLITE_API int sqlite3_bind_text( ){ return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); } -SQLITE_API int sqlite3_bind_text64( +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text64( sqlite3_stmt *pStmt, int i, const char *zData, @@ -79584,7 +73249,7 @@ SQLITE_API int sqlite3_bind_text64( } } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API int sqlite3_bind_text16( +SQLITE_API int SQLITE_STDCALL sqlite3_bind_text16( sqlite3_stmt *pStmt, int i, const void *zData, @@ -79594,7 +73259,7 @@ SQLITE_API int sqlite3_bind_text16( return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); } #endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ int rc; switch( sqlite3_value_type((sqlite3_value*)pValue) ){ case SQLITE_INTEGER: { @@ -79625,7 +73290,7 @@ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_valu } return rc; } -SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ int rc; Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, i); @@ -79635,26 +73300,12 @@ SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ } return rc; } -SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i, sqlite3_uint64 n){ - int rc; - Vdbe *p = (Vdbe *)pStmt; - sqlite3_mutex_enter(p->db->mutex); - if( n>(u64)p->db->aLimit[SQLITE_LIMIT_LENGTH] ){ - rc = SQLITE_TOOBIG; - }else{ - assert( (n & 0x7FFFFFFF)==n ); - rc = sqlite3_bind_zeroblob(pStmt, i, n); - } - rc = sqlite3ApiExit(p->db, rc); - sqlite3_mutex_leave(p->db->mutex); - return rc; -} /* ** Return the number of wildcards that can be potentially bound to. ** This routine is added to support DBD::SQLite. */ -SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; return p ? p->nVar : 0; } @@ -79665,7 +73316,7 @@ SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ ** ** The result is always UTF-8. */ -SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ +SQLITE_API const char *SQLITE_STDCALL sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ Vdbe *p = (Vdbe*)pStmt; if( p==0 || i<1 || i>p->nzVar ){ return 0; @@ -79693,7 +73344,7 @@ SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nNa } return 0; } -SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ +SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ return sqlite3VdbeParameterIndex((Vdbe*)pStmt, zName, sqlite3Strlen30(zName)); } @@ -79727,7 +73378,7 @@ SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *pFromStmt, sqlite3_stmt ** an SQLITE_ERROR is returned. Nothing else can go wrong, so otherwise ** SQLITE_OK is returned. */ -SQLITE_API int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ Vdbe *pFrom = (Vdbe*)pFromStmt; Vdbe *pTo = (Vdbe*)pToStmt; if( pFrom->nVar!=pTo->nVar ){ @@ -79749,7 +73400,7 @@ SQLITE_API int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt * ** the first argument to the sqlite3_prepare() that was used to create ** the statement in the first place. */ -SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt *pStmt){ +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_db_handle(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->db : 0; } @@ -79757,16 +73408,16 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt *pStmt){ ** Return true if the prepared statement is guaranteed to not modify the ** database. */ -SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->readOnly : 1; } /* ** Return true if the prepared statement is in need of being reset. */ -SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; - return v!=0 && v->magic==VDBE_MAGIC_RUN && v->pc>=0; + return v!=0 && v->pc>=0 && v->magic==VDBE_MAGIC_RUN; } /* @@ -79775,7 +73426,7 @@ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ ** prepared statement for the database connection. Return NULL if there ** are no more. */ -SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ +SQLITE_API sqlite3_stmt *SQLITE_STDCALL sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ sqlite3_stmt *pNext; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(pDb) ){ @@ -79796,7 +73447,7 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ /* ** Return the value of a status counter for a prepared statement */ -SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ Vdbe *pVdbe = (Vdbe*)pStmt; u32 v; #ifdef SQLITE_ENABLE_API_ARMOR @@ -79810,229 +73461,11 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ return (int)v; } -/* -** Return the SQL associated with a prepared statement -*/ -SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt){ - Vdbe *p = (Vdbe *)pStmt; - return p ? p->zSql : 0; -} - -/* -** Return the SQL associated with a prepared statement with -** bound parameters expanded. Space to hold the returned string is -** obtained from sqlite3_malloc(). The caller is responsible for -** freeing the returned string by passing it to sqlite3_free(). -** -** The SQLITE_TRACE_SIZE_LIMIT puts an upper bound on the size of -** expanded bound parameters. -*/ -SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){ -#ifdef SQLITE_OMIT_TRACE - return 0; -#else - char *z = 0; - const char *zSql = sqlite3_sql(pStmt); - if( zSql ){ - Vdbe *p = (Vdbe *)pStmt; - sqlite3_mutex_enter(p->db->mutex); - z = sqlite3VdbeExpandSql(p, zSql); - sqlite3_mutex_leave(p->db->mutex); - } - return z; -#endif -} - -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK -/* -** Allocate and populate an UnpackedRecord structure based on the serialized -** record in nKey/pKey. Return a pointer to the new UnpackedRecord structure -** if successful, or a NULL pointer if an OOM error is encountered. -*/ -static UnpackedRecord *vdbeUnpackRecord( - KeyInfo *pKeyInfo, - int nKey, - const void *pKey -){ - char *dummy; /* Dummy argument for AllocUnpackedRecord() */ - UnpackedRecord *pRet; /* Return value */ - - pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo, 0, 0, &dummy); - if( pRet ){ - memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nField+1)); - sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet); - } - return pRet; -} - -/* -** This function is called from within a pre-update callback to retrieve -** a field of the row currently being updated or deleted. -*/ -SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ - PreUpdate *p = db->pPreUpdate; - int rc = SQLITE_OK; - - /* Test that this call is being made from within an SQLITE_DELETE or - ** SQLITE_UPDATE pre-update callback, and that iIdx is within range. */ - if( !p || p->op==SQLITE_INSERT ){ - rc = SQLITE_MISUSE_BKPT; - goto preupdate_old_out; - } - if( iIdx>=p->pCsr->nField || iIdx<0 ){ - rc = SQLITE_RANGE; - goto preupdate_old_out; - } - - /* If the old.* record has not yet been loaded into memory, do so now. */ - if( p->pUnpacked==0 ){ - u32 nRec; - u8 *aRec; - - nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor); - aRec = sqlite3DbMallocRaw(db, nRec); - if( !aRec ) goto preupdate_old_out; - rc = sqlite3BtreeData(p->pCsr->uc.pCursor, 0, nRec, aRec); - if( rc==SQLITE_OK ){ - p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec); - if( !p->pUnpacked ) rc = SQLITE_NOMEM; - } - if( rc!=SQLITE_OK ){ - sqlite3DbFree(db, aRec); - goto preupdate_old_out; - } - p->aRecord = aRec; - } - - if( iIdx>=p->pUnpacked->nField ){ - *ppValue = (sqlite3_value *)columnNullValue(); - }else{ - Mem *pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; - *ppValue = &p->pUnpacked->aMem[iIdx]; - if( iIdx==p->pTab->iPKey ){ - sqlite3VdbeMemSetInt64(pMem, p->iKey1); - }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ - if( pMem->flags & MEM_Int ){ - sqlite3VdbeMemRealify(pMem); - } - } - } - - preupdate_old_out: - sqlite3Error(db, rc); - return sqlite3ApiExit(db, rc); -} -#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ - -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK -/* -** This function is called from within a pre-update callback to retrieve -** the number of columns in the row being updated, deleted or inserted. -*/ -SQLITE_API int sqlite3_preupdate_count(sqlite3 *db){ - PreUpdate *p = db->pPreUpdate; - return (p ? p->keyinfo.nField : 0); -} -#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ - -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK -/* -** This function is designed to be called from within a pre-update callback -** only. It returns zero if the change that caused the callback was made -** immediately by a user SQL statement. Or, if the change was made by a -** trigger program, it returns the number of trigger programs currently -** on the stack (1 for a top-level trigger, 2 for a trigger fired by a -** top-level trigger etc.). -** -** For the purposes of the previous paragraph, a foreign key CASCADE, SET NULL -** or SET DEFAULT action is considered a trigger. -*/ -SQLITE_API int sqlite3_preupdate_depth(sqlite3 *db){ - PreUpdate *p = db->pPreUpdate; - return (p ? p->v->nFrame : 0); -} -#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ - -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK -/* -** This function is called from within a pre-update callback to retrieve -** a field of the row currently being updated or inserted. -*/ -SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ - PreUpdate *p = db->pPreUpdate; - int rc = SQLITE_OK; - Mem *pMem; - - if( !p || p->op==SQLITE_DELETE ){ - rc = SQLITE_MISUSE_BKPT; - goto preupdate_new_out; - } - if( iIdx>=p->pCsr->nField || iIdx<0 ){ - rc = SQLITE_RANGE; - goto preupdate_new_out; - } - - if( p->op==SQLITE_INSERT ){ - /* For an INSERT, memory cell p->iNewReg contains the serialized record - ** that is being inserted. Deserialize it. */ - UnpackedRecord *pUnpack = p->pNewUnpacked; - if( !pUnpack ){ - Mem *pData = &p->v->aMem[p->iNewReg]; - rc = ExpandBlob(pData); - if( rc!=SQLITE_OK ) goto preupdate_new_out; - pUnpack = vdbeUnpackRecord(&p->keyinfo, pData->n, pData->z); - if( !pUnpack ){ - rc = SQLITE_NOMEM; - goto preupdate_new_out; - } - p->pNewUnpacked = pUnpack; - } - if( iIdx>=pUnpack->nField ){ - pMem = (sqlite3_value *)columnNullValue(); - }else{ - pMem = &pUnpack->aMem[iIdx]; - if( iIdx==p->pTab->iPKey ){ - sqlite3VdbeMemSetInt64(pMem, p->iKey2); - } - } - }else{ - /* For an UPDATE, memory cell (p->iNewReg+1+iIdx) contains the required - ** value. Make a copy of the cell contents and return a pointer to it. - ** It is not safe to return a pointer to the memory cell itself as the - ** caller may modify the value text encoding. - */ - assert( p->op==SQLITE_UPDATE ); - if( !p->aNew ){ - p->aNew = (Mem *)sqlite3DbMallocZero(db, sizeof(Mem) * p->pCsr->nField); - if( !p->aNew ){ - rc = SQLITE_NOMEM; - goto preupdate_new_out; - } - } - assert( iIdx>=0 && iIdxpCsr->nField ); - pMem = &p->aNew[iIdx]; - if( pMem->flags==0 ){ - if( iIdx==p->pTab->iPKey ){ - sqlite3VdbeMemSetInt64(pMem, p->iKey2); - }else{ - rc = sqlite3VdbeMemCopy(pMem, &p->v->aMem[p->iNewReg+1+iIdx]); - if( rc!=SQLITE_OK ) goto preupdate_new_out; - } - } - } - *ppValue = pMem; - - preupdate_new_out: - sqlite3Error(db, rc); - return sqlite3ApiExit(db, rc); -} -#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ - #ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* ** Return status data for a single loop within query pStmt. */ -SQLITE_API int sqlite3_stmt_scanstatus( +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement being queried */ int idx, /* Index of loop to report on */ int iScanStatusOp, /* Which metric to return */ @@ -80091,7 +73524,7 @@ SQLITE_API int sqlite3_stmt_scanstatus( /* ** Zero all counters associated with the sqlite3_stmt_scanstatus() data. */ -SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ +SQLITE_API void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; memset(p->anExec, 0, p->nOp * sizeof(i64)); } @@ -80116,8 +73549,6 @@ SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ ** ** The Vdbe parse-tree explainer is also found here. */ -/* #include "sqliteInt.h" */ -/* #include "vdbeInt.h" */ #ifndef SQLITE_OMIT_TRACE @@ -80182,13 +73613,10 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( int i; /* Loop counter */ Mem *pVar; /* Value of a host parameter */ StrAccum out; /* Accumulate the output here */ -#ifndef SQLITE_OMIT_UTF16 - Mem utf8; /* Used to convert UTF16 parameters into UTF8 for display */ -#endif char zBase[100]; /* Initial working space */ db = p->db; - sqlite3StrAccumInit(&out, 0, zBase, sizeof(zBase), + sqlite3StrAccumInit(&out, db, zBase, sizeof(zBase), db->aLimit[SQLITE_LIMIT_LENGTH]); if( db->nVdbeExec>1 ){ while( *zRawSql ){ @@ -80232,21 +73660,19 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( if( pVar->flags & MEM_Null ){ sqlite3StrAccumAppend(&out, "NULL", 4); }else if( pVar->flags & MEM_Int ){ - sqlite3XPrintf(&out, "%lld", pVar->u.i); + sqlite3XPrintf(&out, 0, "%lld", pVar->u.i); }else if( pVar->flags & MEM_Real ){ - sqlite3XPrintf(&out, "%!.15g", pVar->u.r); + sqlite3XPrintf(&out, 0, "%!.15g", pVar->u.r); }else if( pVar->flags & MEM_Str ){ int nOut; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 u8 enc = ENC(db); + Mem utf8; if( enc!=SQLITE_UTF8 ){ memset(&utf8, 0, sizeof(utf8)); utf8.db = db; sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); - if( SQLITE_NOMEM==sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8) ){ - out.accError = STRACCUM_NOMEM; - out.nAlloc = 0; - } + sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); pVar = &utf8; } #endif @@ -80257,17 +73683,17 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( while( nOutn && (pVar->z[nOut]&0xc0)==0x80 ){ nOut++; } } #endif - sqlite3XPrintf(&out, "'%.*q'", nOut, pVar->z); + sqlite3XPrintf(&out, 0, "'%.*q'", nOut, pVar->z); #ifdef SQLITE_TRACE_SIZE_LIMIT if( nOutn ){ - sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); + sqlite3XPrintf(&out, 0, "/*+%d bytes*/", pVar->n-nOut); } #endif #ifndef SQLITE_OMIT_UTF16 if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); #endif }else if( pVar->flags & MEM_Zero ){ - sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); + sqlite3XPrintf(&out, 0, "zeroblob(%d)", pVar->u.nZero); }else{ int nOut; /* Number of bytes of the blob to include in output */ assert( pVar->flags & MEM_Blob ); @@ -80277,18 +73703,17 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( if( nOut>SQLITE_TRACE_SIZE_LIMIT ) nOut = SQLITE_TRACE_SIZE_LIMIT; #endif for(i=0; iz[i]&0xff); + sqlite3XPrintf(&out, 0, "%02x", pVar->z[i]&0xff); } sqlite3StrAccumAppend(&out, "'", 1); #ifdef SQLITE_TRACE_SIZE_LIMIT if( nOutn ){ - sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); + sqlite3XPrintf(&out, 0, "/*+%d bytes*/", pVar->n-nOut); } #endif } } } - if( out.accError ) sqlite3StrAccumReset(&out); return sqlite3StrAccumFinish(&out); } @@ -80316,8 +73741,6 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( ** in this file for details. If in doubt, do not deviate from existing ** commenting and indentation practices when changing or adding code. */ -/* #include "sqliteInt.h" */ -/* #include "vdbeInt.h" */ /* ** Invoke this macro on memory cells just prior to changing the @@ -80384,16 +73807,6 @@ static void updateMaxBlobsize(Mem *p){ } #endif -/* -** This macro evaluates to true if either the update hook or the preupdate -** hook are enabled for database connect DB. -*/ -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK -# define HAS_UPDATE_HOOK(DB) ((DB)->xPreUpdateCallback||(DB)->xUpdateCallback) -#else -# define HAS_UPDATE_HOOK(DB) ((DB)->xUpdateCallback) -#endif - /* ** The next global variable is incremented each time the OP_Found opcode ** is executed. This is used to test whether or not the foreign key @@ -80473,7 +73886,7 @@ SQLITE_API int sqlite3_found_count = 0; && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ -#define isSorter(x) ((x)->eCurType==CURTYPE_SORTER) +#define isSorter(x) ((x)->pSorter!=0) /* ** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL @@ -80484,7 +73897,7 @@ static VdbeCursor *allocateCursor( int iCur, /* Index of the new VdbeCursor */ int nField, /* Number of fields in the table or index */ int iDb, /* Database the cursor belongs to, or -1 */ - u8 eCurType /* Type of the new cursor */ + int isBtreeCursor /* True for B-Tree. False for pseudo-table or vtab */ ){ /* Find the memory cell that will be used to store the blob of memory ** required for this VdbeCursor structure. It is convenient to use a @@ -80500,34 +73913,33 @@ static VdbeCursor *allocateCursor( ** be freed lazily via the sqlite3_release_memory() API. This ** minimizes the number of malloc calls made by the system. ** - ** The memory cell for cursor 0 is aMem[0]. The rest are allocated from - ** the top of the register space. Cursor 1 is at Mem[p->nMem-1]. - ** Cursor 2 is at Mem[p->nMem-2]. And so forth. + ** Memory cells for cursors are allocated at the top of the address + ** space. Memory cell (p->nMem) corresponds to cursor 0. Space for + ** cursor 1 is managed by memory cell (p->nMem-1), etc. */ - Mem *pMem = iCur>0 ? &p->aMem[p->nMem-iCur] : p->aMem; + Mem *pMem = &p->aMem[p->nMem-iCur]; int nByte; VdbeCursor *pCx = 0; nByte = ROUND8(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + - (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); + (isBtreeCursor?sqlite3BtreeCursorSize():0); - assert( iCur>=0 && iCurnCursor ); - if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ + assert( iCurnCursor ); + if( p->apCsr[iCur] ){ sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; } if( SQLITE_OK==sqlite3VdbeMemClearAndResize(pMem, nByte) ){ p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z; memset(pCx, 0, sizeof(VdbeCursor)); - pCx->eCurType = eCurType; pCx->iDb = iDb; pCx->nField = nField; pCx->aOffset = &pCx->aType[nField]; - if( eCurType==CURTYPE_BTREE ){ - pCx->uc.pCursor = (BtCursor*) + if( isBtreeCursor ){ + pCx->pCursor = (BtCursor*) &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; - sqlite3BtreeCursorZero(pCx->uc.pCursor); + sqlite3BtreeCursorZero(pCx->pCursor); } } return pCx; @@ -80579,7 +73991,7 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ ** SQLITE_AFF_TEXT: ** Convert pRec to a text representation. ** -** SQLITE_AFF_BLOB: +** SQLITE_AFF_NONE: ** No-op. pRec is unchanged. */ static void applyAffinity( @@ -80590,7 +74002,7 @@ static void applyAffinity( if( affinity>=SQLITE_AFF_NUMERIC ){ assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL || affinity==SQLITE_AFF_NUMERIC ); - if( (pRec->flags & MEM_Int)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + if( (pRec->flags & MEM_Int)==0 ){ if( (pRec->flags & MEM_Real)==0 ){ if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); }else{ @@ -80600,13 +74012,10 @@ static void applyAffinity( }else if( affinity==SQLITE_AFF_TEXT ){ /* Only attempt the conversion to TEXT if there is an integer or real ** representation (blob and NULL do not get converted) but no string - ** representation. It would be harmless to repeat the conversion if - ** there is already a string rep, but it is pointless to waste those - ** CPU cycles. */ - if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/ - if( (pRec->flags&(MEM_Real|MEM_Int)) ){ - sqlite3VdbeMemStringify(pRec, enc, 1); - } + ** representation. + */ + if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ + sqlite3VdbeMemStringify(pRec, enc, 1); } pRec->flags &= ~(MEM_Real|MEM_Int); } @@ -80618,7 +74027,7 @@ static void applyAffinity( ** is appropriate. But only do the conversion if it is possible without ** loss of information and return the revised type of the argument. */ -SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ +SQLITE_API int SQLITE_STDCALL sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; @@ -80782,7 +74191,6 @@ static void memTracePrint(Mem *p){ sqlite3VdbeMemPrettyPrint(p, zBuf); printf(" %s", zBuf); } - if( p->flags & MEM_Subtype ) printf(" subtype=0x%02x", p->eSubtype); } static void registerTrace(int iReg, Mem *p){ printf("REG[%d] = ", iReg); @@ -80821,8 +74229,8 @@ static void registerTrace(int iReg, Mem *p){ ** This file contains inline asm code for retrieving "high-performance" ** counters for x86 class CPUs. */ -#ifndef SQLITE_HWTIME_H -#define SQLITE_HWTIME_H +#ifndef _HWTIME_H_ +#define _HWTIME_H_ /* ** The following routine only works on pentium-class (or newer) processors. @@ -80890,7 +74298,7 @@ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } #endif -#endif /* !defined(SQLITE_HWTIME_H) */ +#endif /* !defined(_HWTIME_H_) */ /************** End of hwtime.h **********************************************/ /************** Continuing where we left off in vdbe.c ***********************/ @@ -80920,24 +74328,16 @@ static int checkSavepointCount(sqlite3 *db){ /* ** Return the register of pOp->p2 after first preparing it to be ** overwritten with an integer value. -*/ -static SQLITE_NOINLINE Mem *out2PrereleaseWithClear(Mem *pOut){ - sqlite3VdbeMemSetNull(pOut); - pOut->flags = MEM_Int; - return pOut; -} +*/ static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){ Mem *pOut; assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p2<=(p->nMem-p->nCursor) ); pOut = &p->aMem[pOp->p2]; memAboutToChange(p, pOut); - if( VdbeMemDynamic(pOut) ){ /*OPTIMIZATION-IF-FALSE*/ - return out2PrereleaseWithClear(pOut); - }else{ - pOut->flags = MEM_Int; - return pOut; - } + if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); + pOut->flags = MEM_Int; + return pOut; } @@ -80952,15 +74352,12 @@ SQLITE_PRIVATE int sqlite3VdbeExec( Op *pOp = aOp; /* Current operation */ #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) Op *pOrigOp; /* Value of pOp at the top of the loop */ -#endif -#ifdef SQLITE_DEBUG - int nExtraDelete = 0; /* Verifies FORDELETE and AUXDELETE flags */ #endif int rc = SQLITE_OK; /* Value to return */ sqlite3 *db = p->db; /* The database */ u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */ u8 encoding = ENC(db); /* The database encoding */ - int iCompare = 0; /* Result of last comparison */ + int iCompare = 0; /* Result of last OP_Compare operation */ unsigned nVmStep = 0; /* Number of virtual machine steps */ #ifndef SQLITE_OMIT_PROGRESS_CALLBACK unsigned nProgressLimit = 0;/* Invoke xProgress() when nVmStep reaches this */ @@ -80984,7 +74381,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** sqlite3_column_text16() failed. */ goto no_mem; } - assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); + assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY ); assert( p->bIsReader || p->readOnly!=0 ); p->rc = SQLITE_OK; p->iCurrentTime = 0; @@ -80995,9 +74392,13 @@ SQLITE_PRIVATE int sqlite3VdbeExec( sqlite3VdbeIOTraceSql(p); #ifndef SQLITE_OMIT_PROGRESS_CALLBACK if( db->xProgress ){ - u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; assert( 0 < db->nProgressOps ); - nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); + nProgressLimit = (unsigned)p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; + if( nProgressLimit==0 ){ + nProgressLimit = db->nProgressOps; + }else{ + nProgressLimit %= (unsigned)db->nProgressOps; + } } #endif #ifdef SQLITE_DEBUG @@ -81027,12 +74428,9 @@ SQLITE_PRIVATE int sqlite3VdbeExec( } sqlite3EndBenignMalloc(); #endif - for(pOp=&aOp[p->pc]; 1; pOp++){ - /* Errors are detected by individual opcodes, with an immediate - ** jumps to abort_due_to_error. */ - assert( rc==SQLITE_OK ); - + for(pOp=&aOp[p->pc]; rc==SQLITE_OK; pOp++){ assert( pOp>=aOp && pOp<&aOp[p->nOp]); + if( db->mallocFailed ) goto no_mem; #ifdef VDBE_PROFILE start = sqlite3Hwtime(); #endif @@ -81064,39 +74462,37 @@ SQLITE_PRIVATE int sqlite3VdbeExec( /* Sanity checking on other operands */ #ifdef SQLITE_DEBUG - { - u8 opProperty = sqlite3OpcodeProperty[pOp->opcode]; - if( (opProperty & OPFLG_IN1)!=0 ){ - assert( pOp->p1>0 ); - assert( pOp->p1<=(p->nMem+1 - p->nCursor) ); - assert( memIsValid(&aMem[pOp->p1]) ); - assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) ); - REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); - } - if( (opProperty & OPFLG_IN2)!=0 ){ - assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); - assert( memIsValid(&aMem[pOp->p2]) ); - assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) ); - REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); - } - if( (opProperty & OPFLG_IN3)!=0 ){ - assert( pOp->p3>0 ); - assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); - assert( memIsValid(&aMem[pOp->p3]) ); - assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) ); - REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); - } - if( (opProperty & OPFLG_OUT2)!=0 ){ - assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); - memAboutToChange(p, &aMem[pOp->p2]); - } - if( (opProperty & OPFLG_OUT3)!=0 ){ - assert( pOp->p3>0 ); - assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); - memAboutToChange(p, &aMem[pOp->p3]); - } + assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); + if( (pOp->opflags & OPFLG_IN1)!=0 ){ + assert( pOp->p1>0 ); + assert( pOp->p1<=(p->nMem-p->nCursor) ); + assert( memIsValid(&aMem[pOp->p1]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) ); + REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); + } + if( (pOp->opflags & OPFLG_IN2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem-p->nCursor) ); + assert( memIsValid(&aMem[pOp->p2]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) ); + REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); + } + if( (pOp->opflags & OPFLG_IN3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( memIsValid(&aMem[pOp->p3]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) ); + REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); + } + if( (pOp->opflags & OPFLG_OUT2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem-p->nCursor) ); + memAboutToChange(p, &aMem[pOp->p2]); + } + if( (pOp->opflags & OPFLG_OUT3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); + memAboutToChange(p, &aMem[pOp->p3]); } #endif #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) @@ -81180,7 +74576,7 @@ case OP_Goto: { /* jump */ nProgressLimit = nVmStep + db->nProgressOps - (nVmStep%db->nProgressOps); if( db->xProgress(db->pProgressArg) ){ rc = SQLITE_INTERRUPT; - goto abort_due_to_error; + goto vdbe_error_halt; } } #endif @@ -81194,7 +74590,7 @@ case OP_Goto: { /* jump */ ** and then jump to address P2. */ case OP_Gosub: { /* jump */ - assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); pIn1 = &aMem[pOp->p1]; assert( VdbeMemDynamic(pIn1)==0 ); memAboutToChange(p, pIn1); @@ -81234,7 +74630,7 @@ case OP_Return: { /* in1 */ ** See also: EndCoroutine */ case OP_InitCoroutine: { /* jump */ - assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); assert( pOp->p2>=0 && pOp->p2nOp ); assert( pOp->p3>=0 && pOp->p3nOp ); pOut = &aMem[pOp->p1]; @@ -81292,7 +74688,7 @@ case OP_Yield: { /* in1, jump */ } /* Opcode: HaltIfNull P1 P2 P3 P4 P5 -** Synopsis: if r[P3]=null halt +** Synopsis: if r[P3]=null halt ** ** Check the value in register P3. If it is NULL then Halt using ** parameter P1, P2, and P4 as if this were a Halt instruction. If the @@ -81336,6 +74732,8 @@ case OP_HaltIfNull: { /* in3 */ ** is the same as executing Halt. */ case OP_Halt: { + const char *zType; + const char *zLogFmt; VdbeFrame *pFrame; int pcx; @@ -81364,28 +74762,35 @@ case OP_Halt: { p->rc = pOp->p1; p->errorAction = (u8)pOp->p2; p->pc = pcx; - assert( pOp->p5>=0 && pOp->p5<=4 ); if( p->rc ){ if( pOp->p5 ){ static const char * const azType[] = { "NOT NULL", "UNIQUE", "CHECK", "FOREIGN KEY" }; + assert( pOp->p5>=1 && pOp->p5<=4 ); testcase( pOp->p5==1 ); testcase( pOp->p5==2 ); testcase( pOp->p5==3 ); testcase( pOp->p5==4 ); - sqlite3VdbeError(p, "%s constraint failed", azType[pOp->p5-1]); - if( pOp->p4.z ){ - p->zErrMsg = sqlite3MPrintf(db, "%z: %s", p->zErrMsg, pOp->p4.z); - } + zType = azType[pOp->p5-1]; }else{ - sqlite3VdbeError(p, "%s", pOp->p4.z); + zType = 0; } - sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pcx, p->zSql, p->zErrMsg); + assert( zType!=0 || pOp->p4.z!=0 ); + zLogFmt = "abort at %d in [%s]: %s"; + if( zType && pOp->p4.z ){ + sqlite3SetString(&p->zErrMsg, db, "%s constraint failed: %s", + zType, pOp->p4.z); + }else if( pOp->p4.z ){ + sqlite3SetString(&p->zErrMsg, db, "%s", pOp->p4.z); + }else{ + sqlite3SetString(&p->zErrMsg, db, "%s constraint failed", zType); + } + sqlite3_log(pOp->p1, zLogFmt, pcx, p->zSql, p->zErrMsg); } rc = sqlite3VdbeHalt(p); assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); if( rc==SQLITE_BUSY ){ - p->rc = SQLITE_BUSY; + p->rc = rc = SQLITE_BUSY; }else{ assert( rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT ); assert( rc==SQLITE_OK || db->nDeferredCons>0 || db->nDeferredImmCons>0 ); @@ -81451,7 +74856,7 @@ case OP_String8: { /* same as TK_STRING, out2 */ #ifndef SQLITE_OMIT_UTF16 if( encoding!=SQLITE_UTF8 ){ rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); - assert( rc==SQLITE_OK || rc==SQLITE_TOOBIG ); + if( rc==SQLITE_TOOBIG ) goto too_big; if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z ); assert( VdbeMemDynamic(pOut)==0 ); @@ -81464,12 +74869,10 @@ case OP_String8: { /* same as TK_STRING, out2 */ pOp->p4.z = pOut->z; pOp->p1 = pOut->n; } - testcase( rc==SQLITE_TOOBIG ); #endif if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } - assert( rc==SQLITE_OK ); /* Fall through to the next case, OP_String */ } @@ -81478,12 +74881,10 @@ case OP_String8: { /* same as TK_STRING, out2 */ ** ** The string value P4 of length P1 (bytes) is stored in register P2. ** -** If P3 is not zero and the content of register P3 is equal to P5, then +** If P5!=0 and the content of register P3 is greater than zero, then ** the datatype of the register P2 is converted to BLOB. The content is ** the same sequence of bytes, it is merely interpreted as a BLOB instead -** of a string, as if it had been CAST. In other words: -** -** if( P3!=0 and reg[P3]==P5 ) reg[P2] := CAST(reg[P2] as BLOB) +** of a string, as if it had been CAST. */ case OP_String: { /* out2 */ assert( pOp->p4.z!=0 ); @@ -81493,19 +74894,18 @@ case OP_String: { /* out2 */ pOut->n = pOp->p1; pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); -#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS - if( pOp->p3>0 ){ - assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); + if( pOp->p5 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); pIn3 = &aMem[pOp->p3]; assert( pIn3->flags & MEM_Int ); - if( pIn3->u.i==pOp->p5 ) pOut->flags = MEM_Blob|MEM_Static|MEM_Term; + if( pIn3->u.i ) pOut->flags = MEM_Blob|MEM_Static|MEM_Term; } -#endif break; } /* Opcode: Null P1 P2 P3 * * -** Synopsis: r[P2..P3]=NULL +** Synopsis: r[P2..P3]=NULL ** ** Write a NULL into registers P2. If P3 greater than P2, then also write ** NULL into register P3 and every register in between P2 and P3. If P3 @@ -81521,22 +74921,20 @@ case OP_Null: { /* out2 */ u16 nullFlag; pOut = out2Prerelease(p, pOp); cnt = pOp->p3-pOp->p2; - assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; - pOut->n = 0; while( cnt>0 ){ pOut++; memAboutToChange(p, pOut); sqlite3VdbeMemSetNull(pOut); pOut->flags = nullFlag; - pOut->n = 0; cnt--; } break; } /* Opcode: SoftNull P1 * * * * -** Synopsis: r[P1]=NULL +** Synopsis: r[P1]=NULL ** ** Set register P1 to have the value NULL as seen by the OP_MakeRecord ** instruction, but do not free any string or blob memory associated with @@ -81544,7 +74942,7 @@ case OP_Null: { /* out2 */ ** previously copied using OP_SCopy, the copies will continue to be valid. */ case OP_SoftNull: { - assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); pOut = &aMem[pOp->p1]; pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined; break; @@ -81589,7 +74987,7 @@ case OP_Variable: { /* out2 */ } /* Opcode: Move P1 P2 P3 * * -** Synopsis: r[P2@P3]=r[P1@P3] +** Synopsis: r[P2@P3]=r[P1@P3] ** ** Move the P3 values in register P1..P1+P3-1 over into ** registers P2..P2+P3-1. Registers P1..P1+P3-1 are @@ -81611,8 +75009,8 @@ case OP_Move: { pIn1 = &aMem[p1]; pOut = &aMem[p2]; do{ - assert( pOut<=&aMem[(p->nMem+1 - p->nCursor)] ); - assert( pIn1<=&aMem[(p->nMem+1 - p->nCursor)] ); + assert( pOut<=&aMem[(p->nMem-p->nCursor)] ); + assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); sqlite3VdbeMemMove(pOut, pIn1); @@ -81682,24 +75080,8 @@ case OP_SCopy: { /* out2 */ break; } -/* Opcode: IntCopy P1 P2 * * * -** Synopsis: r[P2]=r[P1] -** -** Transfer the integer value held in register P1 into register P2. -** -** This is an optimized version of SCopy that works only for integer -** values. -*/ -case OP_IntCopy: { /* out2 */ - pIn1 = &aMem[pOp->p1]; - assert( (pIn1->flags & MEM_Int)!=0 ); - pOut = &aMem[pOp->p2]; - sqlite3VdbeMemSetInt64(pOut, pIn1->u.i); - break; -} - /* Opcode: ResultRow P1 P2 * * * -** Synopsis: output=r[P1@P2] +** Synopsis: output=r[P1@P2] ** ** The registers P1 through P1+P2-1 contain a single row of ** results. This opcode causes the sqlite3_step() call to terminate @@ -81712,7 +75094,7 @@ case OP_ResultRow: { int i; assert( p->nResColumn==pOp->p2 ); assert( pOp->p1>0 ); - assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); + assert( pOp->p1+pOp->p2<=(p->nMem-p->nCursor)+1 ); #ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* Run the progress counter just before returning. @@ -81722,7 +75104,7 @@ case OP_ResultRow: { && db->xProgress(db->pProgressArg)!=0 ){ rc = SQLITE_INTERRUPT; - goto abort_due_to_error; + goto vdbe_error_halt; } #endif @@ -81732,7 +75114,7 @@ case OP_ResultRow: { if( SQLITE_OK!=(rc = sqlite3VdbeCheckFk(p, 0)) ){ assert( db->flags&SQLITE_CountRows ); assert( p->usesStmtJournal ); - goto abort_due_to_error; + break; } /* If the SQLITE_CountRows flag is set in sqlite3.flags mask, then @@ -81752,7 +75134,9 @@ case OP_ResultRow: { */ assert( p->iStatement==0 || db->flags&SQLITE_CountRows ); rc = sqlite3VdbeCloseStatement(p, SAVEPOINT_RELEASE); - assert( rc==SQLITE_OK ); + if( NEVER(rc!=SQLITE_OK) ){ + break; + } /* Invalidate all ephemeral cursor row caches */ p->cacheCtr = (p->cacheCtr + 2)|1; @@ -81772,10 +75156,6 @@ case OP_ResultRow: { } if( db->mallocFailed ) goto no_mem; - if( db->mTrace & SQLITE_TRACE_ROW ){ - db->xTrace(SQLITE_TRACE_ROW, db->pTraceArg, p, 0); - } - /* Return SQLITE_ROW */ p->pc = (int)(pOp - aOp) + 1; @@ -81832,14 +75212,14 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ } /* Opcode: Add P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]+r[P2] +** Synopsis: r[P3]=r[P1]+r[P2] ** ** Add the value in register P1 to the value in register P2 ** and store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: Multiply P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]*r[P2] +** Synopsis: r[P3]=r[P1]*r[P2] ** ** ** Multiply the value in register P1 by the value in register P2 @@ -81847,14 +75227,14 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ ** If either input is NULL, the result is NULL. */ /* Opcode: Subtract P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]-r[P1] +** Synopsis: r[P3]=r[P2]-r[P1] ** ** Subtract the value in register P1 from the value in register P2 ** and store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: Divide P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]/r[P1] +** Synopsis: r[P3]=r[P2]/r[P1] ** ** Divide the value in register P1 by the value in register P2 ** and store the result in register P3 (P3=P2/P1). If the value in @@ -81862,7 +75242,7 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ ** NULL, the result is NULL. */ /* Opcode: Remainder P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]%r[P1] +** Synopsis: r[P3]=r[P2]%r[P1] ** ** Compute the remainder after integer register P2 is divided by ** register P1 and store the result in register P3. @@ -81981,10 +75361,10 @@ case OP_CollSeq: { break; } -/* Opcode: Function0 P1 P2 P3 P4 P5 +/* Opcode: Function P1 P2 P3 P4 P5 ** Synopsis: r[P3]=func(r[P2@P5]) ** -** Invoke a user function (P4 is a pointer to a FuncDef object that +** Invoke a user function (P4 is a pointer to a Function structure that ** defines the function) with P5 arguments taken from register P2 and ** successors. The result of the function is stored in register P3. ** Register P3 must not be one of the function inputs. @@ -81996,120 +75376,78 @@ case OP_CollSeq: { ** sqlite3_set_auxdata() API may be safely retained until the next ** invocation of this opcode. ** -** See also: Function, AggStep, AggFinal +** See also: AggStep and AggFinal */ -/* Opcode: Function P1 P2 P3 P4 P5 -** Synopsis: r[P3]=func(r[P2@P5]) -** -** Invoke a user function (P4 is a pointer to an sqlite3_context object that -** contains a pointer to the function to be run) with P5 arguments taken -** from register P2 and successors. The result of the function is stored -** in register P3. Register P3 must not be one of the function inputs. -** -** P1 is a 32-bit bitmask indicating whether or not each argument to the -** function was determined to be constant at compile time. If the first -** argument was constant then bit 0 of P1 is set. This is used to determine -** whether meta data associated with a user function argument using the -** sqlite3_set_auxdata() API may be safely retained until the next -** invocation of this opcode. -** -** SQL functions are initially coded as OP_Function0 with P4 pointing -** to a FuncDef object. But on first evaluation, the P4 operand is -** automatically converted into an sqlite3_context object and the operation -** changed to this OP_Function opcode. In this way, the initialization of -** the sqlite3_context object occurs only once, rather than once for each -** evaluation of the function. -** -** See also: Function0, AggStep, AggFinal -*/ -case OP_Function0: { - int n; - sqlite3_context *pCtx; - - assert( pOp->p4type==P4_FUNCDEF ); - n = pOp->p5; - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); - assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); - if( pCtx==0 ) goto no_mem; - pCtx->pOut = 0; - pCtx->pFunc = pOp->p4.pFunc; - pCtx->iOp = (int)(pOp - aOp); - pCtx->pVdbe = p; - pCtx->argc = n; - pOp->p4type = P4_FUNCCTX; - pOp->p4.pCtx = pCtx; - pOp->opcode = OP_Function; - /* Fall through into OP_Function */ -} case OP_Function: { int i; - sqlite3_context *pCtx; + Mem *pArg; + sqlite3_context ctx; + sqlite3_value **apVal; + int n; - assert( pOp->p4type==P4_FUNCCTX ); - pCtx = pOp->p4.pCtx; + n = pOp->p5; + apVal = p->apArg; + assert( apVal || n==0 ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + ctx.pOut = &aMem[pOp->p3]; + memAboutToChange(p, ctx.pOut); - /* If this function is inside of a trigger, the register array in aMem[] - ** might change from one evaluation to the next. The next block of code - ** checks to see if the register array has changed, and if so it - ** reinitializes the relavant parts of the sqlite3_context object */ - pOut = &aMem[pOp->p3]; - if( pCtx->pOut != pOut ){ - pCtx->pOut = pOut; - for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); + assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); + pArg = &aMem[pOp->p2]; + for(i=0; ip2+i, pArg); } - memAboutToChange(p, pCtx->pOut); -#ifdef SQLITE_DEBUG - for(i=0; iargc; i++){ - assert( memIsValid(pCtx->argv[i]) ); - REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); - } -#endif - MemSetTypeFlag(pCtx->pOut, MEM_Null); - pCtx->fErrorOrAux = 0; + assert( pOp->p4type==P4_FUNCDEF ); + ctx.pFunc = pOp->p4.pFunc; + ctx.iOp = (int)(pOp - aOp); + ctx.pVdbe = p; + MemSetTypeFlag(ctx.pOut, MEM_Null); + ctx.fErrorOrAux = 0; db->lastRowid = lastRowid; - (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ - lastRowid = db->lastRowid; /* Remember rowid changes made by xSFunc */ + (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ + lastRowid = db->lastRowid; /* Remember rowid changes made by xFunc */ /* If the function returned an error, throw an exception */ - if( pCtx->fErrorOrAux ){ - if( pCtx->isError ){ - sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); - rc = pCtx->isError; + if( ctx.fErrorOrAux ){ + if( ctx.isError ){ + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(ctx.pOut)); + rc = ctx.isError; } - sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1); - if( rc ) goto abort_due_to_error; + sqlite3VdbeDeleteAuxData(p, (int)(pOp - aOp), pOp->p1); } /* Copy the result of the function into register P3 */ - if( pOut->flags & (MEM_Str|MEM_Blob) ){ - sqlite3VdbeChangeEncoding(pCtx->pOut, encoding); - if( sqlite3VdbeMemTooBig(pCtx->pOut) ) goto too_big; + sqlite3VdbeChangeEncoding(ctx.pOut, encoding); + if( sqlite3VdbeMemTooBig(ctx.pOut) ){ + goto too_big; } - REGISTER_TRACE(pOp->p3, pCtx->pOut); - UPDATE_MAX_BLOBSIZE(pCtx->pOut); + REGISTER_TRACE(pOp->p3, ctx.pOut); + UPDATE_MAX_BLOBSIZE(ctx.pOut); break; } /* Opcode: BitAnd P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]&r[P2] +** Synopsis: r[P3]=r[P1]&r[P2] ** ** Take the bit-wise AND of the values in register P1 and P2 and ** store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: BitOr P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]|r[P2] +** Synopsis: r[P3]=r[P1]|r[P2] ** ** Take the bit-wise OR of the values in register P1 and P2 and ** store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: ShiftLeft P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]<>r[P1] +** Synopsis: r[P3]=r[P2]>>r[P1] ** ** Shift the integer value in register P2 to the right by the ** number of bits specified by the integer in register P1. @@ -82177,7 +75515,7 @@ case OP_ShiftRight: { /* same as TK_RSHIFT, in1, in2, out3 */ } /* Opcode: AddImm P1 P2 * * * -** Synopsis: r[P1]=r[P1]+P2 +** Synopsis: r[P1]=r[P1]+P2 ** ** Add the constant P2 to the value in register P1. ** The result is always an integer. @@ -82253,9 +75591,9 @@ case OP_RealAffinity: { /* in1 */ ** A NULL value is not changed by this routine. It remains NULL. */ case OP_Cast: { /* in1 */ - assert( pOp->p2>=SQLITE_AFF_BLOB && pOp->p2<=SQLITE_AFF_REAL ); + assert( pOp->p2>=SQLITE_AFF_NONE && pOp->p2<=SQLITE_AFF_REAL ); testcase( pOp->p2==SQLITE_AFF_TEXT ); - testcase( pOp->p2==SQLITE_AFF_BLOB ); + testcase( pOp->p2==SQLITE_AFF_NONE ); testcase( pOp->p2==SQLITE_AFF_NUMERIC ); testcase( pOp->p2==SQLITE_AFF_INTEGER ); testcase( pOp->p2==SQLITE_AFF_REAL ); @@ -82264,17 +75602,19 @@ case OP_Cast: { /* in1 */ rc = ExpandBlob(pIn1); sqlite3VdbeMemCast(pIn1, pOp->p2, encoding); UPDATE_MAX_BLOBSIZE(pIn1); - if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_CAST */ -/* Opcode: Eq P1 P2 P3 P4 P5 -** Synopsis: IF r[P3]==r[P1] +/* Opcode: Lt P1 P2 P3 P4 P5 +** Synopsis: if r[P1]r[P1] +** Synopsis: if r[P1]>r[P3] goto P2 ** ** This works just like the Lt opcode except that the jump is taken if ** the content of register P3 is greater than the content of ** register P1. See the Lt opcode for additional information. */ /* Opcode: Ge P1 P2 P3 P4 P5 -** Synopsis: IF r[P3]>=r[P1] +** Synopsis: if r[P1]>=r[P3] goto P2 ** ** This works just like the Lt opcode except that the jump is taken if ** the content of register P3 is greater than or equal to the content of @@ -82371,7 +75694,7 @@ case OP_Lt: /* same as TK_LT, jump, in1, in3 */ case OP_Le: /* same as TK_LE, jump, in1, in3 */ case OP_Gt: /* same as TK_GT, jump, in1, in3 */ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ - int res, res2; /* Result of the comparison of pIn1 against pIn3 */ + int res; /* Result of the comparison of pIn1 against pIn3 */ char affinity; /* Affinity to use for comparison */ u16 flags1; /* Copy of initial value of pIn1->flags */ u16 flags3; /* Copy of initial value of pIn3->flags */ @@ -82394,9 +75717,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ && (flags3&MEM_Null)!=0 && (flags3&MEM_Cleared)==0 ){ - res = 0; /* Operands are equal */ + res = 0; /* Results are equal */ }else{ - res = 1; /* Operands are not equal */ + res = 1; /* Results are not equal */ } }else{ /* SQLITE_NULLEQ is clear and at least one operand is NULL, @@ -82405,8 +75728,6 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ */ if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; - iCompare = 1; /* Operands are not equal */ - memAboutToChange(p, pOut); MemSetTypeFlag(pOut, MEM_Null); REGISTER_TRACE(pOp->p2, pOut); }else{ @@ -82421,34 +75742,21 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ /* Neither operand is NULL. Do a comparison. */ affinity = pOp->p5 & SQLITE_AFF_MASK; if( affinity>=SQLITE_AFF_NUMERIC ){ - if( (flags1 | flags3)&MEM_Str ){ - if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ - applyNumericAffinity(pIn1,0); - testcase( flags3!=pIn3->flags ); /* Possible if pIn1==pIn3 */ - flags3 = pIn3->flags; - } - if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ - applyNumericAffinity(pIn3,0); - } + if( (pIn1->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + applyNumericAffinity(pIn1,0); } - /* Handle the common case of integer comparison here, as an - ** optimization, to avoid a call to sqlite3MemCompare() */ - if( (pIn1->flags & pIn3->flags & MEM_Int)!=0 ){ - if( pIn3->u.i > pIn1->u.i ){ res = +1; goto compare_op; } - if( pIn3->u.i < pIn1->u.i ){ res = -1; goto compare_op; } - res = 0; - goto compare_op; + if( (pIn3->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + applyNumericAffinity(pIn3,0); } }else if( affinity==SQLITE_AFF_TEXT ){ - if( (flags1 & MEM_Str)==0 && (flags1 & (MEM_Int|MEM_Real))!=0 ){ + if( (pIn1->flags & MEM_Str)==0 && (pIn1->flags & (MEM_Int|MEM_Real))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); sqlite3VdbeMemStringify(pIn1, encoding, 1); testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); - assert( pIn1!=pIn3 ); } - if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){ + if( (pIn3->flags & MEM_Str)==0 && (pIn3->flags & (MEM_Int|MEM_Real))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); sqlite3VdbeMemStringify(pIn3, encoding, 1); @@ -82457,16 +75765,24 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ } } assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); + if( pIn1->flags & MEM_Zero ){ + sqlite3VdbeMemExpandBlob(pIn1); + flags1 &= ~MEM_Zero; + } + if( pIn3->flags & MEM_Zero ){ + sqlite3VdbeMemExpandBlob(pIn3); + flags3 &= ~MEM_Zero; + } + if( db->mallocFailed ) goto no_mem; res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); } -compare_op: switch( pOp->opcode ){ - case OP_Eq: res2 = res==0; break; - case OP_Ne: res2 = res; break; - case OP_Lt: res2 = res<0; break; - case OP_Le: res2 = res<=0; break; - case OP_Gt: res2 = res>0; break; - default: res2 = res>=0; break; + case OP_Eq: res = res==0; break; + case OP_Ne: res = res!=0; break; + case OP_Lt: res = res<0; break; + case OP_Le: res = res<=0; break; + case OP_Gt: res = res>0; break; + default: res = res>=0; break; } /* Undo any changes made by applyAffinity() to the input registers. */ @@ -82477,55 +75793,19 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; - iCompare = res; - res2 = res2!=0; /* For this path res2 must be exactly 0 or 1 */ - if( (pOp->p5 & SQLITE_KEEPNULL)!=0 ){ - /* The KEEPNULL flag prevents OP_Eq from overwriting a NULL with 1 - ** and prevents OP_Ne from overwriting NULL with 0. This flag - ** is only used in contexts where either: - ** (1) op==OP_Eq && (r[P2]==NULL || r[P2]==0) - ** (2) op==OP_Ne && (r[P2]==NULL || r[P2]==1) - ** Therefore it is not necessary to check the content of r[P2] for - ** NULL. */ - assert( pOp->opcode==OP_Ne || pOp->opcode==OP_Eq ); - assert( res2==0 || res2==1 ); - testcase( res2==0 && pOp->opcode==OP_Eq ); - testcase( res2==1 && pOp->opcode==OP_Eq ); - testcase( res2==0 && pOp->opcode==OP_Ne ); - testcase( res2==1 && pOp->opcode==OP_Ne ); - if( (pOp->opcode==OP_Eq)==res2 ) break; - } memAboutToChange(p, pOut); MemSetTypeFlag(pOut, MEM_Int); - pOut->u.i = res2; + pOut->u.i = res; REGISTER_TRACE(pOp->p2, pOut); }else{ VdbeBranchTaken(res!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); - if( res2 ){ + if( res ){ goto jump_to_p2; } } break; } -/* Opcode: ElseNotEq * P2 * * * -** -** This opcode must immediately follow an OP_Lt or OP_Gt comparison operator. -** If result of an OP_Eq comparison on the same two operands -** would have be NULL or false (0), then then jump to P2. -** If the result of an OP_Eq comparison on the two previous operands -** would have been true (1), then fall through. -*/ -case OP_ElseNotEq: { /* same as TK_ESCAPE, jump */ - assert( pOp>aOp ); - assert( pOp[-1].opcode==OP_Lt || pOp[-1].opcode==OP_Gt ); - assert( pOp[-1].p5 & SQLITE_STOREP2 ); - VdbeBranchTaken(iCompare!=0, 2); - if( iCompare!=0 ) goto jump_to_p2; - break; -} - - /* Opcode: Permutation * * * P4 * ** ** Set the permutation used by the OP_Compare operator to be the array @@ -82534,14 +75814,11 @@ case OP_ElseNotEq: { /* same as TK_ESCAPE, jump */ ** The permutation is only valid until the next OP_Compare that has ** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should ** occur immediately prior to the OP_Compare. -** -** The first integer in the P4 integer array is the length of the array -** and does not become part of the permutation. */ case OP_Permutation: { assert( pOp->p4type==P4_INTARRAY ); assert( pOp->p4.ai ); - aPermute = pOp->p4.ai + 1; + aPermute = pOp->p4.ai; break; } @@ -82586,11 +75863,11 @@ case OP_Compare: { if( aPermute ){ int k, mx = 0; for(k=0; kmx ) mx = aPermute[k]; - assert( p1>0 && p1+mx<=(p->nMem+1 - p->nCursor)+1 ); - assert( p2>0 && p2+mx<=(p->nMem+1 - p->nCursor)+1 ); + assert( p1>0 && p1+mx<=(p->nMem-p->nCursor)+1 ); + assert( p2>0 && p2+mx<=(p->nMem-p->nCursor)+1 ); }else{ - assert( p1>0 && p1+n<=(p->nMem+1 - p->nCursor)+1 ); - assert( p2>0 && p2+n<=(p->nMem+1 - p->nCursor)+1 ); + assert( p1>0 && p1+n<=(p->nMem-p->nCursor)+1 ); + assert( p2>0 && p2+n<=(p->nMem-p->nCursor)+1 ); } #endif /* SQLITE_DEBUG */ for(i=0; iaOp[0].opcode==OP_Init ); - VdbeBranchTaken(p->aOp[0].p1==pOp->p1, 2); - if( p->aOp[0].p1==pOp->p1 ){ + assert( pOp->p1nOnceFlag ); + VdbeBranchTaken(p->aOnceFlag[pOp->p1]!=0, 2); + if( p->aOnceFlag[pOp->p1] ){ goto jump_to_p2; }else{ - pOp->p1 = p->aOp[0].p1; + p->aOnceFlag[pOp->p1] = 1; } break; } @@ -82771,7 +76052,7 @@ case OP_IfNot: { /* jump, in1 */ } /* Opcode: IsNull P1 P2 * * * -** Synopsis: if r[P1]==NULL goto P2 +** Synopsis: if r[P1]==NULL goto P2 ** ** Jump to P2 if the value in register P1 is NULL. */ @@ -82799,7 +76080,7 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ } /* Opcode: Column P1 P2 P3 P4 P5 -** Synopsis: r[P3]=PX +** Synopsis: r[P3]=PX ** ** Interpret the data that cursor P1 points to as a structure built using ** the MakeRecord instruction. (See the MakeRecord opcode for additional @@ -82824,6 +76105,7 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** skipped for length() and all content loading can be skipped for typeof(). */ case OP_Column: { + i64 payloadSize64; /* Number of bytes in the record */ int p2; /* column number to retrieve */ VdbeCursor *pC; /* The VDBE cursor */ BtCursor *pCrsr; /* The BTree cursor */ @@ -82836,35 +76118,36 @@ case OP_Column: { const u8 *zHdr; /* Next unparsed byte of the header */ const u8 *zEndHdr; /* Pointer to first byte after the header */ u32 offset; /* Offset into the data */ - u64 offset64; /* 64-bit offset */ + u32 szField; /* Number of bytes in the content of a field */ u32 avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ + u16 fx; /* pDest->flags value */ Mem *pReg; /* PseudoTable input register */ - pC = p->apCsr[pOp->p1]; p2 = pOp->p2; - - /* If the cursor cache is stale, bring it up-to-date */ - rc = sqlite3VdbeCursorMoveto(&pC, &p2); - if( rc ) goto abort_due_to_error; - - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( p2nField ); aOffset = pC->aOffset; - assert( pC->eCurType!=CURTYPE_VTAB ); - assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow ); - assert( pC->eCurType!=CURTYPE_SORTER ); - pCrsr = pC->uc.pCursor; +#ifndef SQLITE_OMIT_VIRTUALTABLE + assert( pC->pVtabCursor==0 ); /* OP_Column never called on virtual table */ +#endif + pCrsr = pC->pCursor; + assert( pCrsr!=0 || pC->pseudoTableReg>0 ); /* pCrsr NULL on PseudoTables */ + assert( pCrsr!=0 || pC->nullRow ); /* pC->nullRow on PseudoTables */ - if( pC->cacheStatus!=p->cacheCtr ){ /*OPTIMIZATION-IF-FALSE*/ + /* If the cursor cache is stale, bring it up-to-date */ + rc = sqlite3VdbeCursorMoveto(pC); + if( rc ) goto abort_due_to_error; + if( pC->cacheStatus!=p->cacheCtr ){ if( pC->nullRow ){ - if( pC->eCurType==CURTYPE_PSEUDO ){ - assert( pC->uc.pseudoTableReg>0 ); - pReg = &aMem[pC->uc.pseudoTableReg]; + if( pCrsr==0 ){ + assert( pC->pseudoTableReg>0 ); + pReg = &aMem[pC->pseudoTableReg]; assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); pC->payloadSize = pC->szRow = avail = pReg->n; @@ -82874,56 +76157,67 @@ case OP_Column: { goto op_column_out; } }else{ - assert( pC->eCurType==CURTYPE_BTREE ); assert( pCrsr ); - assert( sqlite3BtreeCursorIsValid(pCrsr) ); - pC->payloadSize = sqlite3BtreePayloadSize(pCrsr); - pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &avail); + if( pC->isTable==0 ){ + assert( sqlite3BtreeCursorIsValid(pCrsr) ); + VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &payloadSize64); + assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ + /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the + ** payload size, so it is impossible for payloadSize64 to be + ** larger than 32 bits. */ + assert( (payloadSize64 & SQLITE_MAX_U32)==(u64)payloadSize64 ); + pC->aRow = sqlite3BtreeKeyFetch(pCrsr, &avail); + pC->payloadSize = (u32)payloadSize64; + }else{ + assert( sqlite3BtreeCursorIsValid(pCrsr) ); + VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &pC->payloadSize); + assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ + pC->aRow = sqlite3BtreeDataFetch(pCrsr, &avail); + } assert( avail<=65536 ); /* Maximum page size is 64KiB */ if( pC->payloadSize <= (u32)avail ){ pC->szRow = pC->payloadSize; - }else if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; }else{ pC->szRow = avail; } + if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } } pC->cacheStatus = p->cacheCtr; pC->iHdrOffset = getVarint32(pC->aRow, offset); pC->nHdrParsed = 0; aOffset[0] = offset; + /* Make sure a corrupt database has not given us an oversize header. + ** Do this now to avoid an oversize memory allocation. + ** + ** Type entries can be between 1 and 5 bytes each. But 4 and 5 byte + ** types use so much data space that there can only be 4096 and 32 of + ** them, respectively. So the maximum header length results from a + ** 3-byte type for each of the maximum of 32768 columns plus three + ** extra bytes for the header length itself. 32768*3 + 3 = 98307. + */ + if( offset > 98307 || offset > pC->payloadSize ){ + rc = SQLITE_CORRUPT_BKPT; + goto op_column_error; + } - if( availaRow does not have to hold the entire row, but it does at least ** need to cover the header of the record. If pC->aRow does not contain ** the complete header, then set it to zero, forcing the header to be ** dynamically allocated. */ pC->aRow = 0; pC->szRow = 0; - - /* Make sure a corrupt database has not given us an oversize header. - ** Do this now to avoid an oversize memory allocation. - ** - ** Type entries can be between 1 and 5 bytes each. But 4 and 5 byte - ** types use so much data space that there can only be 4096 and 32 of - ** them, respectively. So the maximum header length results from a - ** 3-byte type for each of the maximum of 32768 columns plus three - ** extra bytes for the header length itself. 32768*3 + 3 = 98307. - */ - if( offset > 98307 || offset > pC->payloadSize ){ - rc = SQLITE_CORRUPT_BKPT; - goto abort_due_to_error; - } - }else if( offset>0 ){ /*OPTIMIZATION-IF-TRUE*/ - /* The following goto is an optimization. It can be omitted and - ** everything will still work. But OP_Column is measurably faster - ** by skipping the subsequent conditional, which is always true. - */ - zData = pC->aRow; - assert( pC->nHdrParsed<=p2 ); /* Conditional skipped */ - goto op_column_read_header; } + + /* The following goto is an optimization. It can be omitted and + ** everything will still work. But OP_Column is measurably faster + ** by skipping the subsequent conditional, which is always true. + */ + assert( pC->nHdrParsed<=p2 ); /* Conditional skipped */ + goto op_column_read_header; } /* Make sure at least the first p2+1 entries of the header have been @@ -82933,53 +76227,65 @@ case OP_Column: { /* If there is more header available for parsing in the record, try ** to extract additional fields up through the p2+1-th field */ + op_column_read_header: if( pC->iHdrOffsetaRow==0 ){ memset(&sMem, 0, sizeof(sMem)); - rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], !pC->isTable, &sMem); - if( rc!=SQLITE_OK ) goto abort_due_to_error; + rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], + !pC->isTable, &sMem); + if( rc!=SQLITE_OK ){ + goto op_column_error; + } zData = (u8*)sMem.z; }else{ zData = pC->aRow; } /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */ - op_column_read_header: i = pC->nHdrParsed; - offset64 = aOffset[i]; + offset = aOffset[i]; zHdr = zData + pC->iHdrOffset; zEndHdr = zData + aOffset[0]; + assert( i<=p2 && zHdraType[i++] = t; - aOffset[i] = (u32)(offset64 & 0xffffffff); + pC->aType[i] = t; + szField = sqlite3VdbeSerialTypeLen(t); + offset += szField; + if( offsetnHdrParsed = i; + pC->iHdrOffset = (u32)(zHdr - zData); + if( pC->aRow==0 ){ + sqlite3VdbeMemRelease(&sMem); + sMem.flags = MEM_Null; + } + /* The record is corrupt if any of the following are true: ** (1) the bytes of the header extend past the declared header size + ** (zHdr>zEndHdr) ** (2) the entire header was used but not all data was used + ** (zHdr==zEndHdr && offset!=pC->payloadSize) ** (3) the end of the data extends beyond the end of the record. + ** (offset > pC->payloadSize) */ - if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset64!=pC->payloadSize)) - || (offset64 > pC->payloadSize) + if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset!=pC->payloadSize)) + || (offset > pC->payloadSize) ){ - if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); rc = SQLITE_CORRUPT_BKPT; - goto abort_due_to_error; + goto op_column_error; } - - pC->nHdrParsed = i; - pC->iHdrOffset = (u32)(zHdr - zData); - if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); - }else{ - t = 0; } /* If after trying to extract new entries from the header, nHdrParsed is @@ -82994,8 +76300,6 @@ case OP_Column: { } goto op_column_out; } - }else{ - t = pC->aType[p2]; } /* Extract the content for the p2+1-th column. Control can only @@ -83005,37 +76309,13 @@ case OP_Column: { assert( p2nHdrParsed ); assert( rc==SQLITE_OK ); assert( sqlite3VdbeCheckMemInvariants(pDest) ); - if( VdbeMemDynamic(pDest) ){ - sqlite3VdbeMemSetNull(pDest); - } - assert( t==pC->aType[p2] ); + if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest); + t = pC->aType[p2]; if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ - zData = pC->aRow + aOffset[p2]; - if( t<12 ){ - sqlite3VdbeSerialGet(zData, t, pDest); - }else{ - /* If the column value is a string, we need a persistent value, not - ** a MEM_Ephem value. This branch is a fast short-cut that is equivalent - ** to calling sqlite3VdbeSerialGet() and sqlite3VdbeDeephemeralize(). - */ - static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term }; - pDest->n = len = (t-12)/2; - pDest->enc = encoding; - if( pDest->szMalloc < len+2 ){ - pDest->flags = MEM_Null; - if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem; - }else{ - pDest->z = pDest->zMalloc; - } - memcpy(pDest->z, zData, len); - pDest->z[len] = 0; - pDest->z[len+1] = 0; - pDest->flags = aFlag[t&1]; - } + sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], t, pDest); }else{ - pDest->enc = encoding; /* This branch happens only when content is on overflow pages */ if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) @@ -83046,19 +76326,39 @@ case OP_Column: { ** 2. the length(X) function if X is a blob, and ** 3. if the content length is zero. ** So we might as well use bogus content rather than reading - ** content from disk. */ - static u8 aZero[8]; /* This is the bogus content */ - sqlite3VdbeSerialGet(aZero, t, pDest); + ** content from disk. NULL will work for the value for strings + ** and blobs and whatever is in the payloadSize64 variable + ** will work for everything else. */ + sqlite3VdbeSerialGet(t<=13 ? (u8*)&payloadSize64 : 0, t, pDest); }else{ rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable, pDest); - if( rc!=SQLITE_OK ) goto abort_due_to_error; + if( rc!=SQLITE_OK ){ + goto op_column_error; + } sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); pDest->flags &= ~MEM_Ephem; } } + pDest->enc = encoding; op_column_out: + /* If the column value is an ephemeral string, go ahead and persist + ** that string in case the cursor moves before the column value is + ** used. The following code does the equivalent of Deephemeralize() + ** but does it faster. */ + if( (pDest->flags & MEM_Ephem)!=0 && pDest->z ){ + fx = pDest->flags & (MEM_Str|MEM_Blob); + assert( fx!=0 ); + zData = (const u8*)pDest->z; + len = pDest->n; + if( sqlite3VdbeMemClearAndResize(pDest, len+2) ) goto no_mem; + memcpy(pDest->z, zData, len); + pDest->z[len] = 0; + pDest->z[len+1] = 0; + pDest->flags = fx|MEM_Term; + } +op_column_error: UPDATE_MAX_BLOBSIZE(pDest); REGISTER_TRACE(pOp->p3, pDest); break; @@ -83082,7 +76382,7 @@ case OP_Affinity: { assert( zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; while( (cAff = *(zAffinity++))!=0 ){ - assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); + assert( pIn1 <= &p->aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); applyAffinity(pIn1, cAff, encoding); pIn1++; @@ -83104,7 +76404,7 @@ case OP_Affinity: { ** The mapping from character to affinity is given by the SQLITE_AFF_ ** macros defined in sqliteInt.h. ** -** If P4 is NULL then all index fields have the affinity BLOB. +** If P4 is NULL then all index fields have the affinity NONE. */ case OP_MakeRecord: { u8 *zNewRecord; /* A buffer to hold the data for the new record */ @@ -83122,7 +76422,7 @@ case OP_MakeRecord: { int file_format; /* File format to use for encoding */ int i; /* Space used in zNewRecord[] header */ int j; /* Space used in zNewRecord[] content */ - u32 len; /* Length of a field */ + int len; /* Length of a field */ /* Assuming the record contains N fields, the record format looks ** like this: @@ -83144,7 +76444,7 @@ case OP_MakeRecord: { nZero = 0; /* Number of zero bytes at the end of the record */ nField = pOp->p1; zAffinity = pOp->p4.z; - assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem+1 - p->nCursor)+1 ); + assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem-p->nCursor)+1 ); pData0 = &aMem[nField]; nField = pOp->p2; pLast = &pData0[nField-1]; @@ -83172,10 +76472,11 @@ case OP_MakeRecord: { pRec = pLast; do{ assert( memIsValid(pRec) ); - pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format, &len); + pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format); + len = sqlite3VdbeSerialTypeLen(serial_type); if( pRec->flags & MEM_Zero ){ if( nData ){ - if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; + sqlite3VdbeMemExpandBlob(pRec); }else{ nZero += pRec->u.nZero; len -= pRec->u.nZero; @@ -83185,9 +76486,7 @@ case OP_MakeRecord: { testcase( serial_type==127 ); testcase( serial_type==128 ); nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type); - if( pRec==pData0 ) break; - pRec--; - }while(1); + }while( (--pRec)>=pData0 ); /* EVIDENCE-OF: R-22564-11647 The header begins with a single varint ** which determines the total number of bytes in the header. The varint @@ -83236,7 +76535,7 @@ case OP_MakeRecord: { assert( i==nHdr ); assert( j==nByte ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pOut->n = (int)nByte; pOut->flags = MEM_Blob; if( nZero ){ @@ -83260,12 +76559,10 @@ case OP_Count: { /* out2 */ i64 nEntry; BtCursor *pCrsr; - assert( p->apCsr[pOp->p1]->eCurType==CURTYPE_BTREE ); - pCrsr = p->apCsr[pOp->p1]->uc.pCursor; + pCrsr = p->apCsr[pOp->p1]->pCursor; assert( pCrsr ); nEntry = 0; /* Not needed. Only used to silence a warning. */ rc = sqlite3BtreeCount(pCrsr, &nEntry); - if( rc ) goto abort_due_to_error; pOut = out2Prerelease(p, pOp); pOut->u.i = nEntry; break; @@ -83305,7 +76602,8 @@ case OP_Savepoint: { /* A new savepoint cannot be created if there are active write ** statements (i.e. open read/write incremental blob handles). */ - sqlite3VdbeError(p, "cannot open savepoint - SQL statements in progress"); + sqlite3SetString(&p->zErrMsg, db, "cannot open savepoint - " + "SQL statements in progress"); rc = SQLITE_BUSY; }else{ nName = sqlite3Strlen30(zName); @@ -83322,7 +76620,7 @@ case OP_Savepoint: { #endif /* Create a new savepoint structure. */ - pNew = sqlite3DbMallocRawNN(db, sizeof(Savepoint)+nName+1); + pNew = sqlite3DbMallocRaw(db, sizeof(Savepoint)+nName+1); if( pNew ){ pNew->zName = (char *)&pNew[1]; memcpy(pNew->zName, zName, nName+1); @@ -83335,7 +76633,7 @@ case OP_Savepoint: { }else{ db->nSavepoint++; } - + /* Link the new savepoint into the database handle's list. */ pNew->pNext = db->pSavepoint; db->pSavepoint = pNew; @@ -83356,14 +76654,15 @@ case OP_Savepoint: { iSavepoint++; } if( !pSavepoint ){ - sqlite3VdbeError(p, "no such savepoint: %s", zName); + sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", zName); rc = SQLITE_ERROR; }else if( db->nVdbeWrite>0 && p1==SAVEPOINT_RELEASE ){ /* It is not possible to release (commit) a savepoint if there are ** active write statements. */ - sqlite3VdbeError(p, "cannot release savepoint - " - "SQL statements in progress"); + sqlite3SetString(&p->zErrMsg, db, + "cannot release savepoint - SQL statements in progress" + ); rc = SQLITE_BUSY; }else{ @@ -83443,7 +76742,6 @@ case OP_Savepoint: { } } } - if( rc ) goto abort_due_to_error; break; } @@ -83460,37 +76758,49 @@ case OP_Savepoint: { case OP_AutoCommit: { int desiredAutoCommit; int iRollback; + int turnOnAC; desiredAutoCommit = pOp->p1; iRollback = pOp->p2; + turnOnAC = desiredAutoCommit && !db->autoCommit; assert( desiredAutoCommit==1 || desiredAutoCommit==0 ); assert( desiredAutoCommit==1 || iRollback==0 ); assert( db->nVdbeActive>0 ); /* At least this one VM is active */ assert( p->bIsReader ); - if( desiredAutoCommit!=db->autoCommit ){ +#if 0 + if( turnOnAC && iRollback && db->nVdbeActive>1 ){ + /* If this instruction implements a ROLLBACK and other VMs are + ** still running, and a transaction is active, return an error indicating + ** that the other VMs must complete first. + */ + sqlite3SetString(&p->zErrMsg, db, "cannot rollback transaction - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + }else +#endif + if( turnOnAC && !iRollback && db->nVdbeWrite>0 ){ + /* If this instruction implements a COMMIT and other VMs are writing + ** return an error indicating that the other VMs must complete first. + */ + sqlite3SetString(&p->zErrMsg, db, "cannot commit transaction - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + }else if( desiredAutoCommit!=db->autoCommit ){ if( iRollback ){ assert( desiredAutoCommit==1 ); sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); db->autoCommit = 1; - }else if( desiredAutoCommit && db->nVdbeWrite>0 ){ - /* If this instruction implements a COMMIT and other VMs are writing - ** return an error indicating that the other VMs must complete first. - */ - sqlite3VdbeError(p, "cannot commit transaction - " - "SQL statements in progress"); - rc = SQLITE_BUSY; - goto abort_due_to_error; }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; - } - if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ - p->pc = (int)(pOp - aOp); - db->autoCommit = (u8)(1-desiredAutoCommit); - p->rc = rc = SQLITE_BUSY; - goto vdbe_return; + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + db->autoCommit = (u8)(1-desiredAutoCommit); + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; + } } assert( db->nStatement==0 ); sqlite3CloseSavepoints(db); @@ -83501,13 +76811,12 @@ case OP_AutoCommit: { } goto vdbe_return; }else{ - sqlite3VdbeError(p, + sqlite3SetString(&p->zErrMsg, db, (!desiredAutoCommit)?"cannot start a transaction within a transaction":( (iRollback)?"cannot rollback - no transaction is active": "cannot commit - no transaction is active")); rc = SQLITE_ERROR; - goto abort_due_to_error; } break; } @@ -83563,14 +76872,12 @@ case OP_Transaction: { if( pBt ){ rc = sqlite3BtreeBeginTrans(pBt, pOp->p2); - testcase( rc==SQLITE_BUSY_SNAPSHOT ); - testcase( rc==SQLITE_BUSY_RECOVERY ); + if( rc==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; + } if( rc!=SQLITE_OK ){ - if( (rc&0xff)==SQLITE_BUSY ){ - p->pc = (int)(pOp - aOp); - p->rc = rc; - goto vdbe_return; - } goto abort_due_to_error; } @@ -83597,9 +76904,10 @@ case OP_Transaction: { } /* Gather the schema version number for checking: - ** IMPLEMENTATION-OF: R-03189-51135 As each SQL statement runs, the schema - ** version is checked to ensure that the schema has not changed since the - ** SQL statement was prepared. + ** IMPLEMENTATION-OF: R-32195-19465 The schema version is used by SQLite + ** each time a query is executed to ensure that the internal cache of the + ** schema used when compiling the SQL query matches the schema of the + ** database against which the compiled query is actually executed. */ sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&iMeta); iGen = db->aDb[pOp->p1].pSchema->iGeneration; @@ -83629,7 +76937,6 @@ case OP_Transaction: { p->expired = 1; rc = SQLITE_SCHEMA; } - if( rc ) goto abort_due_to_error; break; } @@ -83666,15 +76973,15 @@ case OP_ReadCookie: { /* out2 */ /* Opcode: SetCookie P1 P2 P3 * * ** -** Write the integer value P3 into cookie number P2 of database P1. -** P2==1 is the schema version. P2==2 is the database format. -** P2==3 is the recommended pager cache +** Write the content of register P3 (interpreted as an integer) +** into cookie number P2 of database P1. P2==1 is the schema version. +** P2==2 is the database format. P2==3 is the recommended pager cache ** size, and so forth. P1==0 is the main database file and P1==1 is the ** database file used to store temporary tables. ** ** A transaction must be started before executing this opcode. */ -case OP_SetCookie: { +case OP_SetCookie: { /* in3 */ Db *pDb; assert( pOp->p2p1>=0 && pOp->p1nDb ); @@ -83683,15 +76990,17 @@ case OP_SetCookie: { pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); + pIn3 = &aMem[pOp->p3]; + sqlite3VdbeMemIntegerify(pIn3); /* See note about index shifting on OP_ReadCookie */ - rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3); + rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, (int)pIn3->u.i); if( pOp->p2==BTREE_SCHEMA_VERSION ){ /* When the schema cookie changes, record the new cookie internally */ - pDb->pSchema->schema_cookie = pOp->p3; + pDb->pSchema->schema_cookie = (int)pIn3->u.i; db->flags |= SQLITE_InternChanges; }else if( pOp->p2==BTREE_FILE_FORMAT ){ /* Record changes in the file format */ - pDb->pSchema->file_format = pOp->p3; + pDb->pSchema->file_format = (u8)pIn3->u.i; } if( pOp->p1==1 ){ /* Invalidate all prepared statements whenever the TEMP database @@ -83699,7 +77008,6 @@ case OP_SetCookie: { sqlite3ExpirePreparedStatements(db); p->expired = 0; } - if( rc ) goto abort_due_to_error; break; } @@ -83790,6 +77098,7 @@ case OP_ReopenIdx: { case OP_OpenRead: case OP_OpenWrite: + assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR|OPFLAG_SEEKEQ))==pOp->p5 ); assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); assert( p->bIsReader ); assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx @@ -83797,7 +77106,7 @@ case OP_OpenWrite: if( p->expired ){ rc = SQLITE_ABORT_ROLLBACK; - goto abort_due_to_error; + break; } nField = 0; @@ -83810,8 +77119,7 @@ case OP_OpenWrite: pX = pDb->pBt; assert( pX!=0 ); if( pOp->opcode==OP_OpenWrite ){ - assert( OPFLAG_FORDELETE==BTREE_FORDELETE ); - wrFlag = BTREE_WRCSR | (pOp->p5 & OPFLAG_FORDELETE); + wrFlag = 1; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( pDb->pSchema->file_format < p->minWriteFileFormat ){ p->minWriteFileFormat = pDb->pSchema->file_format; @@ -83821,7 +77129,7 @@ case OP_OpenWrite: } if( pOp->p5 & OPFLAG_P2ISREG ){ assert( p2>0 ); - assert( p2<=(p->nMem+1 - p->nCursor) ); + assert( p2<=(p->nMem-p->nCursor) ); pIn2 = &aMem[p2]; assert( memIsValid(pIn2) ); assert( (pIn2->flags & MEM_Int)!=0 ); @@ -83831,7 +77139,10 @@ case OP_OpenWrite: ** that opcode will always set the p2 value to 2 or more or else fail. ** If there were a failure, the prepared statement would have halted ** before reaching this instruction. */ - assert( p2>=2 ); + if( NEVER(p2<2) ) { + rc = SQLITE_CORRUPT_BKPT; + goto abort_due_to_error; + } } if( pOp->p4type==P4_KEYINFO ){ pKeyInfo = pOp->p4.pKeyInfo; @@ -83844,15 +77155,12 @@ case OP_OpenWrite: assert( pOp->p1>=0 ); assert( nField>=0 ); testcase( nField==0 ); /* Table with INTEGER PRIMARY KEY and nothing else */ - pCur = allocateCursor(p, pOp->p1, nField, iDb, CURTYPE_BTREE); + pCur = allocateCursor(p, pOp->p1, nField, iDb, 1); if( pCur==0 ) goto no_mem; pCur->nullRow = 1; pCur->isOrdered = 1; pCur->pgnoRoot = p2; -#ifdef SQLITE_DEBUG - pCur->wrFlag = wrFlag; -#endif - rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->uc.pCursor); + rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); pCur->pKeyInfo = pKeyInfo; /* Set the VdbeCursor.isTable variable. Previous versions of ** SQLite used to check if the root-page flags were sane at this point @@ -83863,13 +77171,8 @@ case OP_OpenWrite: open_cursor_set_hints: assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); assert( OPFLAG_SEEKEQ==BTREE_SEEK_EQ ); - testcase( pOp->p5 & OPFLAG_BULKCSR ); -#ifdef SQLITE_ENABLE_CURSOR_HINTS - testcase( pOp->p2 & OPFLAG_SEEKEQ ); -#endif - sqlite3BtreeCursorHintFlags(pCur->uc.pCursor, - (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ))); - if( rc ) goto abort_due_to_error; + sqlite3BtreeCursorHints(pCur->pCursor, + (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ))); break; } @@ -83912,7 +77215,7 @@ case OP_OpenEphemeral: { SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_BTREE); + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; pCx->isEphemeral = 1; @@ -83936,17 +77239,14 @@ case OP_OpenEphemeral: { assert( pKeyInfo->db==db ); assert( pKeyInfo->enc==ENC(db) ); pCx->pKeyInfo = pKeyInfo; - rc = sqlite3BtreeCursor(pCx->pBt, pgno, BTREE_WRCSR, - pKeyInfo, pCx->uc.pCursor); + rc = sqlite3BtreeCursor(pCx->pBt, pgno, 1, pKeyInfo, pCx->pCursor); } pCx->isTable = 0; }else{ - rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, BTREE_WRCSR, - 0, pCx->uc.pCursor); + rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, 1, 0, pCx->pCursor); pCx->isTable = 1; } } - if( rc ) goto abort_due_to_error; pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); break; } @@ -83966,13 +77266,12 @@ case OP_SorterOpen: { assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_SORTER); + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; assert( pCx->pKeyInfo->db==db ); assert( pCx->pKeyInfo->enc==ENC(db) ); rc = sqlite3VdbeSorterInit(db, pOp->p3, pCx); - if( rc ) goto abort_due_to_error; break; } @@ -83987,7 +77286,7 @@ case OP_SequenceTest: { VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; - assert( isSorter(pC) ); + assert( pC->pSorter ); if( (pC->seqCount++)==0 ){ goto jump_to_p2; } @@ -84015,10 +77314,10 @@ case OP_OpenPseudo: { assert( pOp->p1>=0 ); assert( pOp->p3>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, CURTYPE_PSEUDO); + pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, 0); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; - pCx->uc.pseudoTableReg = pOp->p2; + pCx->pseudoTableReg = pOp->p2; pCx->isTable = 1; assert( pOp->p5==0 ); break; @@ -84036,26 +77335,6 @@ case OP_Close: { break; } -#ifdef SQLITE_ENABLE_COLUMN_USED_MASK -/* Opcode: ColumnsUsed P1 * * P4 * -** -** This opcode (which only exists if SQLite was compiled with -** SQLITE_ENABLE_COLUMN_USED_MASK) identifies which columns of the -** table or index for cursor P1 are used. P4 is a 64-bit integer -** (P4_INT64) in which the first 63 bits are one for each of the -** first 63 columns of the table or index that are actually used -** by the cursor. The high-order bit is set if any column after -** the 64th is used. -*/ -case OP_ColumnsUsed: { - VdbeCursor *pC; - pC = p->apCsr[pOp->p1]; - assert( pC->eCurType==CURTYPE_BTREE ); - pC->maskUsed = *(u64*)pOp->p4.pI64; - break; -} -#endif - /* Opcode: SeekGE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** @@ -84068,13 +77347,6 @@ case OP_ColumnsUsed: { ** is greater than or equal to the key value. If there are no records ** greater than or equal to the key and P2 is not zero, then jump to P2. ** -** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this -** opcode will always land on a record that equally equals the key, or -** else jump immediately to P2. When the cursor is OPFLAG_SEEKEQ, this -** opcode must be followed by an IdxLE opcode with the same arguments. -** The IdxLE opcode will be skipped if this opcode succeeds, but the -** IdxLE opcode will be used on subsequent loop iterations. -** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. @@ -84133,48 +77405,51 @@ case OP_ColumnsUsed: { ** from the end toward the beginning. In other words, the cursor is ** configured to use Prev, not Next. ** -** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this -** opcode will always land on a record that equally equals the key, or -** else jump immediately to P2. When the cursor is OPFLAG_SEEKEQ, this -** opcode must be followed by an IdxGE opcode with the same arguments. -** The IdxGE opcode will be skipped if this opcode succeeds, but the -** IdxGE opcode will be used on subsequent loop iterations. -** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ case OP_SeekLT: /* jump, in3 */ case OP_SeekLE: /* jump, in3 */ case OP_SeekGE: /* jump, in3 */ case OP_SeekGT: { /* jump, in3 */ - int res; /* Comparison result */ - int oc; /* Opcode */ - VdbeCursor *pC; /* The cursor to seek */ - UnpackedRecord r; /* The key to seek for */ - int nField; /* Number of columns or fields in the key */ - i64 iKey; /* The rowid we are to seek to */ - int eqOnly; /* Only interested in == results */ + int res; + int oc; + VdbeCursor *pC; + UnpackedRecord r; + int nField; + i64 iKey; /* The rowid we are to seek to */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pOp->p2!=0 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->pseudoTableReg==0 ); assert( OP_SeekLE == OP_SeekLT+1 ); assert( OP_SeekGE == OP_SeekLT+2 ); assert( OP_SeekGT == OP_SeekLT+3 ); assert( pC->isOrdered ); - assert( pC->uc.pCursor!=0 ); + assert( pC->pCursor!=0 ); oc = pOp->opcode; - eqOnly = 0; pC->nullRow = 0; #ifdef SQLITE_DEBUG pC->seekOp = pOp->opcode; #endif + /* For a cursor with the BTREE_SEEK_EQ hint, only the OP_SeekGE and + ** OP_SeekLE opcodes are allowed, and these must be immediately followed + ** by an OP_IdxGT or OP_IdxLT opcode, respectively, with the same key. + */ +#ifdef SQLITE_DEBUG + if( sqlite3BtreeCursorHasHint(pC->pCursor, BTREE_SEEK_EQ) ){ + assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE ); + assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); + assert( pOp[1].p1==pOp[0].p1 ); + assert( pOp[1].p2==pOp[0].p2 ); + assert( pOp[1].p3==pOp[0].p3 ); + assert( pOp[1].p4.i==pOp[0].p4.i ); + } +#endif + if( pC->isTable ){ - /* The BTREE_SEEK_EQ flag is only set on index cursors */ - assert( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ)==0 ); - /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do ** the seek, so convert it. */ @@ -84217,26 +77492,12 @@ case OP_SeekGT: { /* jump, in3 */ if( (oc & 0x0001)==(OP_SeekLT & 0x0001) ) oc++; } } - rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)iKey, 0, &res); + rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)iKey, 0, &res); pC->movetoTarget = iKey; /* Used by OP_Delete */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } }else{ - /* For a cursor with the BTREE_SEEK_EQ hint, only the OP_SeekGE and - ** OP_SeekLE opcodes are allowed, and these must be immediately followed - ** by an OP_IdxGT or OP_IdxLT opcode, respectively, with the same key. - */ - if( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ) ){ - eqOnly = 1; - assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE ); - assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); - assert( pOp[1].p1==pOp[0].p1 ); - assert( pOp[1].p2==pOp[0].p2 ); - assert( pOp[1].p3==pOp[0].p3 ); - assert( pOp[1].p4.i==pOp[0].p4.i ); - } - nField = pOp->p4.i; assert( pOp->p4type==P4_INT32 ); assert( nField>0 ); @@ -84260,15 +77521,11 @@ case OP_SeekGT: { /* jump, in3 */ #ifdef SQLITE_DEBUG { int i; for(i=0; iuc.pCursor, &r, 0, 0, &res); + ExpandBlob(r.aMem); + rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, &r, 0, 0, &res); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - if( eqOnly && r.eqSeen==0 ){ - assert( res!=0 ); - goto seek_not_found; - } } pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -84278,7 +77535,7 @@ case OP_SeekGT: { /* jump, in3 */ if( oc>=OP_SeekGE ){ assert( oc==OP_SeekGE || oc==OP_SeekGT ); if( res<0 || (res==0 && oc==OP_SeekGT) ){ res = 0; - rc = sqlite3BtreeNext(pC->uc.pCursor, &res); + rc = sqlite3BtreeNext(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; }else{ res = 0; @@ -84287,27 +77544,49 @@ case OP_SeekGT: { /* jump, in3 */ assert( oc==OP_SeekLT || oc==OP_SeekLE ); if( res>0 || (res==0 && oc==OP_SeekLT) ){ res = 0; - rc = sqlite3BtreePrevious(pC->uc.pCursor, &res); + rc = sqlite3BtreePrevious(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; }else{ /* res might be negative because the table is empty. Check to ** see if this is the case. */ - res = sqlite3BtreeEof(pC->uc.pCursor); + res = sqlite3BtreeEof(pC->pCursor); } } -seek_not_found: assert( pOp->p2>0 ); VdbeBranchTaken(res!=0,2); if( res ){ goto jump_to_p2; - }else if( eqOnly ){ - assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); - pOp++; /* Skip the OP_IdxLt or OP_IdxGT that follows */ } break; } +/* Opcode: Seek P1 P2 * * * +** Synopsis: intkey=r[P2] +** +** P1 is an open table cursor and P2 is a rowid integer. Arrange +** for P1 to move so that it points to the rowid given by P2. +** +** This is actually a deferred seek. Nothing actually happens until +** the cursor is used to read a record. That way, if no reads +** occur, no unnecessary I/O happens. +*/ +case OP_Seek: { /* in2 */ + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->pCursor!=0 ); + assert( pC->isTable ); + pC->nullRow = 0; + pIn2 = &aMem[pOp->p2]; + pC->movetoTarget = sqlite3VdbeIntValue(pIn2); + pC->deferredMoveto = 1; + break; +} + + /* Opcode: Found P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** @@ -84392,21 +77671,20 @@ case OP_Found: { /* jump, in3 */ pC->seekOp = pOp->opcode; #endif pIn3 = &aMem[pOp->p3]; - assert( pC->eCurType==CURTYPE_BTREE ); - assert( pC->uc.pCursor!=0 ); + assert( pC->pCursor!=0 ); assert( pC->isTable==0 ); pFree = 0; if( pOp->p4.i>0 ){ r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p4.i; r.aMem = pIn3; -#ifdef SQLITE_DEBUG for(ii=0; iip3+ii, &r.aMem[ii]); - } #endif + } pIdxKey = &r; }else{ pIdxKey = sqlite3VdbeAllocUnpackedRecord( @@ -84414,7 +77692,7 @@ case OP_Found: { /* jump, in3 */ ); if( pIdxKey==0 ) goto no_mem; assert( pIn3->flags & MEM_Blob ); - (void)ExpandBlob(pIn3); + ExpandBlob(pIn3); sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); } pIdxKey->default_rc = 0; @@ -84430,10 +77708,10 @@ case OP_Found: { /* jump, in3 */ } } } - rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, pIdxKey, 0, 0, &res); + rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, pIdxKey, 0, 0, &res); sqlite3DbFree(db, pFree); if( rc!=SQLITE_OK ){ - goto abort_due_to_error; + break; } pC->seekResult = res; alreadyExists = (res==0); @@ -84450,43 +77728,14 @@ case OP_Found: { /* jump, in3 */ break; } -/* Opcode: SeekRowid P1 P2 P3 * * -** Synopsis: intkey=r[P3] -** -** P1 is the index of a cursor open on an SQL table btree (with integer -** keys). If register P3 does not contain an integer or if P1 does not -** contain a record with rowid P3 then jump immediately to P2. -** Or, if P2 is 0, raise an SQLITE_CORRUPT error. If P1 does contain -** a record with rowid P3 then -** leave the cursor pointing at that record and fall through to the next -** instruction. -** -** The OP_NotExists opcode performs the same operation, but with OP_NotExists -** the P3 register must be guaranteed to contain an integer value. With this -** opcode, register P3 might not contain an integer. -** -** The OP_NotFound opcode performs the same operation on index btrees -** (with arbitrary multi-value keys). -** -** This opcode leaves the cursor in a state where it cannot be advanced -** in either direction. In other words, the Next and Prev opcodes will -** not work following this opcode. -** -** See also: Found, NotFound, NoConflict, SeekRowid -*/ /* Opcode: NotExists P1 P2 P3 * * ** Synopsis: intkey=r[P3] ** ** P1 is the index of a cursor open on an SQL table btree (with integer ** keys). P3 is an integer rowid. If P1 does not contain a record with -** rowid P3 then jump immediately to P2. Or, if P2 is 0, raise an -** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then -** leave the cursor pointing at that record and fall through to the next -** instruction. -** -** The OP_SeekRowid opcode performs the same operation but also allows the -** P3 register to contain a non-integer value, in which case the jump is -** always taken. This opcode requires that P3 always contain an integer. +** rowid P3 then jump immediately to P2. If P1 does contain a record +** with rowid P3 then leave the cursor pointing at that record and fall +** through to the next instruction. ** ** The OP_NotFound opcode performs the same operation on index btrees ** (with arbitrary multi-value keys). @@ -84495,21 +77744,14 @@ case OP_Found: { /* jump, in3 */ ** in either direction. In other words, the Next and Prev opcodes will ** not work following this opcode. ** -** See also: Found, NotFound, NoConflict, SeekRowid +** See also: Found, NotFound, NoConflict */ -case OP_SeekRowid: { /* jump, in3 */ +case OP_NotExists: { /* jump, in3 */ VdbeCursor *pC; BtCursor *pCrsr; int res; u64 iKey; - pIn3 = &aMem[pOp->p3]; - if( (pIn3->flags & MEM_Int)==0 ){ - applyAffinity(pIn3, SQLITE_AFF_NUMERIC, encoding); - if( (pIn3->flags & MEM_Int)==0 ) goto jump_to_p2; - } - /* Fall through into OP_NotExists */ -case OP_NotExists: /* jump, in3 */ pIn3 = &aMem[pOp->p3]; assert( pIn3->flags & MEM_Int ); assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -84519,28 +77761,19 @@ case OP_NotExists: /* jump, in3 */ pC->seekOp = 0; #endif assert( pC->isTable ); - assert( pC->eCurType==CURTYPE_BTREE ); - pCrsr = pC->uc.pCursor; + assert( pC->pseudoTableReg==0 ); + pCrsr = pC->pCursor; assert( pCrsr!=0 ); res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); - assert( rc==SQLITE_OK || res==0 ); pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; pC->deferredMoveto = 0; VdbeBranchTaken(res!=0,2); pC->seekResult = res; - if( res!=0 ){ - assert( rc==SQLITE_OK ); - if( pOp->p2==0 ){ - rc = SQLITE_CORRUPT_BKPT; - }else{ - goto jump_to_p2; - } - } - if( rc ) goto abort_due_to_error; + if( res!=0 ) goto jump_to_p2; break; } @@ -84555,7 +77788,6 @@ case OP_NotExists: /* jump, in3 */ case OP_Sequence: { /* out2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( p->apCsr[pOp->p1]!=0 ); - assert( p->apCsr[pOp->p1]->eCurType!=CURTYPE_VTAB ); pOut = out2Prerelease(p, pOp); pOut->u.i = p->apCsr[pOp->p1]->seqCount++; break; @@ -84591,9 +77823,9 @@ case OP_NewRowid: { /* out2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->eCurType==CURTYPE_BTREE ); - assert( pC->uc.pCursor!=0 ); - { + if( NEVER(pC->pCursor==0) ){ + /* The zero initialization above is all that is needed */ + }else{ /* The next rowid or record number (different terms for the same ** thing) is obtained in a two-step algorithm. ** @@ -84620,15 +77852,16 @@ case OP_NewRowid: { /* out2 */ #endif if( !pC->useRandomRowid ){ - rc = sqlite3BtreeLast(pC->uc.pCursor, &res); + rc = sqlite3BtreeLast(pC->pCursor, &res); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } if( res ){ v = 1; /* IMP: R-61914-48074 */ }else{ - assert( sqlite3BtreeCursorIsValid(pC->uc.pCursor) ); - v = sqlite3BtreeIntegerKey(pC->uc.pCursor); + assert( sqlite3BtreeCursorIsValid(pC->pCursor) ); + rc = sqlite3BtreeKeySize(pC->pCursor, &v); + assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ if( v>=MAX_ROWID ){ pC->useRandomRowid = 1; }else{ @@ -84648,7 +77881,7 @@ case OP_NewRowid: { /* out2 */ pMem = &pFrame->aMem[pOp->p3]; }else{ /* Assert that P3 is a valid memory cell. */ - assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); pMem = &aMem[pOp->p3]; memAboutToChange(p, pMem); } @@ -84678,12 +77911,11 @@ case OP_NewRowid: { /* out2 */ do{ sqlite3_randomness(sizeof(v), &v); v &= (MAX_ROWID>>1); v++; /* Ensure that v is greater than zero */ - }while( ((rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)v, + }while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, 0, &res))==SQLITE_OK) && (res==0) && (++cnt<100)); - if( rc ) goto abort_due_to_error; - if( res==0 ){ + if( rc==SQLITE_OK && res==0 ){ rc = SQLITE_FULL; /* IMP: R-38219-53002 */ goto abort_due_to_error; } @@ -84711,12 +77943,10 @@ case OP_NewRowid: { /* out2 */ ** sqlite3_last_insert_rowid() function (otherwise it is unmodified). ** ** If the OPFLAG_USESEEKRESULT flag of P5 is set and if the result of -** the last seek operation (OP_NotExists or OP_SeekRowid) was a success, -** then this +** the last seek operation (OP_NotExists) was a success, then this ** operation will not attempt to find the appropriate row before doing ** the insert but will instead overwrite the row that the cursor is -** currently pointing to. Presumably, the prior OP_NotExists or -** OP_SeekRowid opcode +** currently pointing to. Presumably, the prior OP_NotExists opcode ** has already positioned the cursor correctly. This is an optimization ** that boosts performance by avoiding redundant seeks. ** @@ -84725,9 +77955,9 @@ case OP_NewRowid: { /* out2 */ ** is part of an INSERT operation. The difference is only important to ** the update hook. ** -** Parameter P4 may point to a Table structure, or may be NULL. If it is -** not NULL, then the update-hook (sqlite3.xUpdateCallback) is invoked -** following a successful insert. +** Parameter P4 may point to a string containing the table-name, or +** may be NULL. If it is not NULL, then the update-hook +** (sqlite3.xUpdateCallback) is invoked following a successful insert. ** ** (WARNING/TODO: If P1 is a pseudo-cursor and P2 is dynamically ** allocated, then ownership of P2 is transferred to the pseudo-cursor @@ -84739,7 +77969,7 @@ case OP_NewRowid: { /* out2 */ ** for indices is OP_IdxInsert. */ /* Opcode: InsertInt P1 P2 P3 P4 P5 -** Synopsis: intkey=P3 data=r[P2] +** Synopsis: intkey=P3 data=r[P2] ** ** This works exactly like OP_Insert except that the key is the ** integer value P3, not the value of the integer stored in register P3. @@ -84748,23 +77978,22 @@ case OP_Insert: case OP_InsertInt: { Mem *pData; /* MEM cell holding data for the record to be inserted */ Mem *pKey; /* MEM cell holding key for the record */ + i64 iKey; /* The integer ROWID or key for the record to be inserted */ VdbeCursor *pC; /* Cursor to table into which insert is written */ + int nZero; /* Number of zero-bytes to append */ int seekResult; /* Result of prior seek or 0 if no USESEEKRESULT flag */ const char *zDb; /* database name - used by the update hook */ - Table *pTab; /* Table structure - used by update and pre-update hooks */ + const char *zTbl; /* Table name - used by the opdate hook */ int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */ - BtreePayload x; /* Payload to be inserted */ - op = 0; pData = &aMem[pOp->p2]; assert( pOp->p1>=0 && pOp->p1nCursor ); assert( memIsValid(pData) ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->eCurType==CURTYPE_BTREE ); - assert( pC->uc.pCursor!=0 ); + assert( pC->pCursor!=0 ); + assert( pC->pseudoTableReg==0 ); assert( pC->isTable ); - assert( pOp->p4type==P4_TABLE || pOp->p4type>=P4_STATIC ); REGISTER_TRACE(pOp->p2, pData); if( pOp->opcode==OP_Insert ){ @@ -84772,188 +78001,95 @@ case OP_InsertInt: { assert( pKey->flags & MEM_Int ); assert( memIsValid(pKey) ); REGISTER_TRACE(pOp->p3, pKey); - x.nKey = pKey->u.i; + iKey = pKey->u.i; }else{ assert( pOp->opcode==OP_InsertInt ); - x.nKey = pOp->p3; + iKey = pOp->p3; } - if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ - assert( pC->isTable ); - assert( pC->iDb>=0 ); - zDb = db->aDb[pC->iDb].zDbSName; - pTab = pOp->p4.pTab; - assert( HasRowid(pTab) ); - op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); - }else{ - pTab = 0; /* Not needed. Silence a comiler warning. */ - zDb = 0; /* Not needed. Silence a compiler warning. */ - } - -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK - /* Invoke the pre-update hook, if any */ - if( db->xPreUpdateCallback - && pOp->p4type==P4_TABLE - && !(pOp->p5 & OPFLAG_ISUPDATE) - ){ - sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, x.nKey, pOp->p2); - } -#endif - if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = x.nKey; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = iKey; if( pData->flags & MEM_Null ){ - x.pData = 0; - x.nData = 0; + pData->z = 0; + pData->n = 0; }else{ assert( pData->flags & (MEM_Blob|MEM_Str) ); - x.pData = pData->z; - x.nData = pData->n; } seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0); if( pData->flags & MEM_Zero ){ - x.nZero = pData->u.nZero; + nZero = pData->u.nZero; }else{ - x.nZero = 0; + nZero = 0; } - x.pKey = 0; - rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, + rc = sqlite3BtreeInsert(pC->pCursor, 0, iKey, + pData->z, pData->n, nZero, (pOp->p5 & OPFLAG_APPEND)!=0, seekResult ); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; /* Invoke the update-hook if required. */ - if( rc ) goto abort_due_to_error; - if( db->xUpdateCallback && op ){ - db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, x.nKey); + if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z ){ + zDb = db->aDb[pC->iDb].zName; + zTbl = pOp->p4.z; + op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); + assert( pC->isTable ); + db->xUpdateCallback(db->pUpdateArg, op, zDb, zTbl, iKey); + assert( pC->iDb>=0 ); } break; } -/* Opcode: Delete P1 P2 P3 P4 P5 +/* Opcode: Delete P1 P2 * P4 * ** ** Delete the record at which the P1 cursor is currently pointing. ** -** If the OPFLAG_SAVEPOSITION bit of the P5 parameter is set, then -** the cursor will be left pointing at either the next or the previous +** The cursor will be left pointing at either the next or the previous ** record in the table. If it is left pointing at the next record, then -** the next Next instruction will be a no-op. As a result, in this case -** it is ok to delete a record from within a Next loop. If -** OPFLAG_SAVEPOSITION bit of P5 is clear, then the cursor will be -** left in an undefined state. -** -** If the OPFLAG_AUXDELETE bit is set on P5, that indicates that this -** delete one of several associated with deleting a table row and all its -** associated index entries. Exactly one of those deletes is the "primary" -** delete. The others are all on OPFLAG_FORDELETE cursors or else are -** marked with the AUXDELETE flag. +** the next Next instruction will be a no-op. Hence it is OK to delete +** a record from within a Next loop. ** -** If the OPFLAG_NCHANGE flag of P2 (NB: P2 not P5) is set, then the row -** change count is incremented (otherwise not). +** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is +** incremented (otherwise not). ** ** P1 must not be pseudo-table. It has to be a real table with ** multiple rows. ** -** If P4 is not NULL then it points to a Table object. In this case either -** the update or pre-update hook, or both, may be invoked. The P1 cursor must -** have been positioned using OP_NotFound prior to invoking this opcode in -** this case. Specifically, if one is configured, the pre-update hook is -** invoked if P4 is not NULL. The update-hook is invoked if one is configured, -** P4 is not NULL, and the OPFLAG_NCHANGE flag is set in P2. -** -** If the OPFLAG_ISUPDATE flag is set in P2, then P3 contains the address -** of the memory cell that contains the value that the rowid of the row will -** be set to by the update. +** If P4 is not NULL, then it is the name of the table that P1 is +** pointing to. The update hook will be invoked, if it exists. +** If P4 is not NULL then the P1 cursor must have been positioned +** using OP_NotFound prior to invoking this opcode. */ case OP_Delete: { VdbeCursor *pC; - const char *zDb; - Table *pTab; - int opflags; - opflags = pOp->p2; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->eCurType==CURTYPE_BTREE ); - assert( pC->uc.pCursor!=0 ); + assert( pC->pCursor!=0 ); /* Only valid for real tables, no pseudotables */ assert( pC->deferredMoveto==0 ); #ifdef SQLITE_DEBUG - if( pOp->p4type==P4_TABLE && HasRowid(pOp->p4.pTab) && pOp->p5==0 ){ - /* If p5 is zero, the seek operation that positioned the cursor prior to - ** OP_Delete will have also set the pC->movetoTarget field to the rowid of - ** the row that is being deleted */ - i64 iKey = sqlite3BtreeIntegerKey(pC->uc.pCursor); - assert( pC->movetoTarget==iKey ); - } -#endif - - /* If the update-hook or pre-update-hook will be invoked, set zDb to - ** the name of the db to pass as to it. Also set local pTab to a copy - ** of p4.pTab. Finally, if p5 is true, indicating that this cursor was - ** last moved with OP_Next or OP_Prev, not Seek or NotFound, set - ** VdbeCursor.movetoTarget to the current rowid. */ - if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ - assert( pC->iDb>=0 ); - assert( pOp->p4.pTab!=0 ); - zDb = db->aDb[pC->iDb].zDbSName; - pTab = pOp->p4.pTab; - if( (pOp->p5 & OPFLAG_SAVEPOSITION)!=0 && pC->isTable ){ - pC->movetoTarget = sqlite3BtreeIntegerKey(pC->uc.pCursor); - } - }else{ - zDb = 0; /* Not needed. Silence a compiler warning. */ - pTab = 0; /* Not needed. Silence a compiler warning. */ + /* The seek operation that positioned the cursor prior to OP_Delete will + ** have also set the pC->movetoTarget field to the rowid of the row that + ** is being deleted */ + if( pOp->p4.z && pC->isTable ){ + i64 iKey = 0; + sqlite3BtreeKeySize(pC->pCursor, &iKey); + assert( pC->movetoTarget==iKey ); } - -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK - /* Invoke the pre-update-hook if required. */ - if( db->xPreUpdateCallback && pOp->p4.pTab && HasRowid(pTab) ){ - assert( !(opflags & OPFLAG_ISUPDATE) || (aMem[pOp->p3].flags & MEM_Int) ); - sqlite3VdbePreUpdateHook(p, pC, - (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, - zDb, pTab, pC->movetoTarget, - pOp->p3 - ); - } - if( opflags & OPFLAG_ISNOOP ) break; #endif - /* Only flags that can be set are SAVEPOISTION and AUXDELETE */ - assert( (pOp->p5 & ~(OPFLAG_SAVEPOSITION|OPFLAG_AUXDELETE))==0 ); - assert( OPFLAG_SAVEPOSITION==BTREE_SAVEPOSITION ); - assert( OPFLAG_AUXDELETE==BTREE_AUXDELETE ); - -#ifdef SQLITE_DEBUG - if( p->pFrame==0 ){ - if( pC->isEphemeral==0 - && (pOp->p5 & OPFLAG_AUXDELETE)==0 - && (pC->wrFlag & OPFLAG_FORDELETE)==0 - ){ - nExtraDelete++; - } - if( pOp->p2 & OPFLAG_NCHANGE ){ - nExtraDelete--; - } - } -#endif - - rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5); + rc = sqlite3BtreeDelete(pC->pCursor); pC->cacheStatus = CACHE_STALE; - if( rc ) goto abort_due_to_error; /* Invoke the update-hook if required. */ - if( opflags & OPFLAG_NCHANGE ){ - p->nChange++; - if( db->xUpdateCallback && HasRowid(pTab) ){ - db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, zDb, pTab->zName, - pC->movetoTarget); - assert( pC->iDb>=0 ); - } + if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){ + db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, + db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget); + assert( pC->iDb>=0 ); } - + if( pOp->p2 & OPFLAG_NCHANGE ) p->nChange++; break; } /* Opcode: ResetCount * * * * * @@ -84970,7 +78106,7 @@ case OP_ResetCount: { } /* Opcode: SorterCompare P1 P2 P3 P4 -** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2 +** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2 ** ** P1 is a sorter cursor. This instruction compares a prefix of the ** record blob in register P3 against a prefix of the entry that @@ -84997,7 +78133,6 @@ case OP_SorterCompare: { res = 0; rc = sqlite3VdbeSorterCompare(pC, pIn3, nKeyCol, &res); VdbeBranchTaken(res!=0,2); - if( rc ) goto abort_due_to_error; if( res ) goto jump_to_p2; break; }; @@ -85023,7 +78158,6 @@ case OP_SorterData: { rc = sqlite3VdbeSorterRowkey(pC, pOut); assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); assert( pOp->p1>=0 && pOp->p1nCursor ); - if( rc ) goto abort_due_to_error; p->apCsr[pOp->p3]->cacheStatus = CACHE_STALE; break; } @@ -85055,6 +78189,7 @@ case OP_RowData: { VdbeCursor *pC; BtCursor *pCrsr; u32 n; + i64 n64; pOut = &aMem[pOp->p2]; memAboutToChange(p, pOut); @@ -85062,19 +78197,18 @@ case OP_RowData: { /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; - assert( pC!=0 ); - assert( pC->eCurType==CURTYPE_BTREE ); assert( isSorter(pC)==0 ); assert( pC->isTable || pOp->opcode!=OP_RowData ); assert( pC->isTable==0 || pOp->opcode==OP_RowData ); + assert( pC!=0 ); assert( pC->nullRow==0 ); - assert( pC->uc.pCursor!=0 ); - pCrsr = pC->uc.pCursor; + assert( pC->pseudoTableReg==0 ); + assert( pC->pCursor!=0 ); + pCrsr = pC->pCursor; /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or - ** OP_SeekRowid or OP_Rewind/Op_Next with no intervening instructions - ** that might invalidate the cursor. - ** If this where not the case, on of the following assert()s + ** OP_Rewind/Op_Next with no intervening instructions that might invalidate + ** the cursor. If this where not the case, on of the following assert()s ** would fail. Should this ever change (because of changes in the code ** generator) then the fix would be to insert a call to ** sqlite3VdbeCursorMoveto(). @@ -85086,9 +78220,20 @@ case OP_RowData: { if( rc!=SQLITE_OK ) goto abort_due_to_error; #endif - n = sqlite3BtreePayloadSize(pCrsr); - if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; + if( pC->isTable==0 ){ + assert( !pC->isTable ); + VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &n64); + assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ + if( n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + n = (u32)n64; + }else{ + VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &n); + assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ + if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } } testcase( n==0 ); if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){ @@ -85101,7 +78246,6 @@ case OP_RowData: { }else{ rc = sqlite3BtreeData(pCrsr, 0, n, pOut->z); } - if( rc ) goto abort_due_to_error; pOut->enc = SQLITE_UTF8; /* In case the blob is ever cast to text */ UPDATE_MAX_BLOBSIZE(pOut); REGISTER_TRACE(pOp->p2, pOut); @@ -85128,32 +78272,30 @@ case OP_Rowid: { /* out2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow ); + assert( pC->pseudoTableReg==0 || pC->nullRow ); if( pC->nullRow ){ pOut->flags = MEM_Null; break; }else if( pC->deferredMoveto ){ v = pC->movetoTarget; #ifndef SQLITE_OMIT_VIRTUALTABLE - }else if( pC->eCurType==CURTYPE_VTAB ){ - assert( pC->uc.pVCur!=0 ); - pVtab = pC->uc.pVCur->pVtab; + }else if( pC->pVtabCursor ){ + pVtab = pC->pVtabCursor->pVtab; pModule = pVtab->pModule; assert( pModule->xRowid ); - rc = pModule->xRowid(pC->uc.pVCur, &v); + rc = pModule->xRowid(pC->pVtabCursor, &v); sqlite3VtabImportErrmsg(p, pVtab); - if( rc ) goto abort_due_to_error; #endif /* SQLITE_OMIT_VIRTUALTABLE */ }else{ - assert( pC->eCurType==CURTYPE_BTREE ); - assert( pC->uc.pCursor!=0 ); + assert( pC->pCursor!=0 ); rc = sqlite3VdbeCursorRestore(pC); if( rc ) goto abort_due_to_error; if( pC->nullRow ){ pOut->flags = MEM_Null; break; } - v = sqlite3BtreeIntegerKey(pC->uc.pCursor); + rc = sqlite3BtreeKeySize(pC->pCursor, &v); + assert( rc==SQLITE_OK ); /* Always so because of CursorRestore() above */ } pOut->u.i = v; break; @@ -85173,9 +78315,8 @@ case OP_NullRow: { assert( pC!=0 ); pC->nullRow = 1; pC->cacheStatus = CACHE_STALE; - if( pC->eCurType==CURTYPE_BTREE ){ - assert( pC->uc.pCursor!=0 ); - sqlite3BtreeClearCursor(pC->uc.pCursor); + if( pC->pCursor ){ + sqlite3BtreeClearCursor(pC->pCursor); } break; } @@ -85200,8 +78341,7 @@ case OP_Last: { /* jump */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->eCurType==CURTYPE_BTREE ); - pCrsr = pC->uc.pCursor; + pCrsr = pC->pCursor; res = 0; assert( pCrsr!=0 ); rc = sqlite3BtreeLast(pCrsr, &res); @@ -85212,7 +78352,6 @@ case OP_Last: { /* jump */ #ifdef SQLITE_DEBUG pC->seekOp = OP_Last; #endif - if( rc ) goto abort_due_to_error; if( pOp->p2>0 ){ VdbeBranchTaken(res!=0,2); if( res ) goto jump_to_p2; @@ -85270,14 +78409,12 @@ case OP_Rewind: { /* jump */ if( isSorter(pC) ){ rc = sqlite3VdbeSorterRewind(pC, &res); }else{ - assert( pC->eCurType==CURTYPE_BTREE ); - pCrsr = pC->uc.pCursor; + pCrsr = pC->pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } - if( rc ) goto abort_due_to_error; pC->nullRow = (u8)res; assert( pOp->p2>0 && pOp->p2nOp ); VdbeBranchTaken(res!=0,2); @@ -85369,7 +78506,7 @@ case OP_Next: /* jump */ res = pOp->p3; assert( pC!=0 ); assert( pC->deferredMoveto==0 ); - assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->pCursor ); assert( res==0 || (res==1 && pC->isTable==0) ); testcase( res==1 ); assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); @@ -85386,11 +78523,10 @@ case OP_Next: /* jump */ || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE || pC->seekOp==OP_Last ); - rc = pOp->p4.xAdvance(pC->uc.pCursor, &res); + rc = pOp->p4.xAdvance(pC->pCursor, &res); next_tail: pC->cacheStatus = CACHE_STALE; VdbeBranchTaken(res==0,2); - if( rc ) goto abort_due_to_error; if( res==0 ){ pC->nullRow = 0; p->aCounter[pOp->p5]++; @@ -85428,7 +78564,9 @@ case OP_Next: /* jump */ case OP_SorterInsert: /* in2 */ case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; - BtreePayload x; + BtCursor *pCrsr; + int nKey; + const char *zKey; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -85436,23 +78574,24 @@ case OP_IdxInsert: { /* in2 */ assert( isSorter(pC)==(pOp->opcode==OP_SorterInsert) ); pIn2 = &aMem[pOp->p2]; assert( pIn2->flags & MEM_Blob ); + pCrsr = pC->pCursor; if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - assert( pC->eCurType==CURTYPE_BTREE || pOp->opcode==OP_SorterInsert ); + assert( pCrsr!=0 ); assert( pC->isTable==0 ); rc = ExpandBlob(pIn2); - if( rc ) goto abort_due_to_error; - if( pOp->opcode==OP_SorterInsert ){ - rc = sqlite3VdbeSorterWrite(pC, pIn2); - }else{ - x.nKey = pIn2->n; - x.pKey = pIn2->z; - rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, pOp->p3, - ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) - ); - assert( pC->deferredMoveto==0 ); - pC->cacheStatus = CACHE_STALE; + if( rc==SQLITE_OK ){ + if( isSorter(pC) ){ + rc = sqlite3VdbeSorterWrite(pC, pIn2); + }else{ + nKey = pIn2->n; + zKey = pIn2->z; + rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3, + ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) + ); + assert( pC->deferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; + } } - if( rc) goto abort_due_to_error; break; } @@ -85470,48 +78609,29 @@ case OP_IdxDelete: { UnpackedRecord r; assert( pOp->p3>0 ); - assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem+1 - p->nCursor)+1 ); + assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem-p->nCursor)+1 ); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->eCurType==CURTYPE_BTREE ); - pCrsr = pC->uc.pCursor; + pCrsr = pC->pCursor; assert( pCrsr!=0 ); assert( pOp->p5==0 ); r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p3; r.default_rc = 0; r.aMem = &aMem[pOp->p2]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ideferredMoveto==0 ); pC->cacheStatus = CACHE_STALE; break; } -/* Opcode: Seek P1 * P3 P4 * -** Synopsis: Move P3 to P1.rowid -** -** P1 is an open index cursor and P3 is a cursor on the corresponding -** table. This opcode does a deferred seek of the P3 table cursor -** to the row that corresponds to the current row of P1. -** -** This is a deferred seek. Nothing actually happens until -** the cursor is used to read a record. That way, if no reads -** occur, no unnecessary I/O happens. -** -** P4 may be an array of integers (type P4_INTARRAY) containing -** one entry for each column in the P3 table. If array entry a(i) -** is non-zero, then reading column a(i)-1 from cursor P3 is -** equivalent to performing the deferred seek and then reading column i -** from P1. This information is stored in P3 and used to redirect -** reads against P3 over to P1, thus possibly avoiding the need to -** seek and read cursor P3. -*/ /* Opcode: IdxRowid P1 P2 * * * ** Synopsis: r[P2]=rowid ** @@ -85521,57 +78641,36 @@ case OP_IdxDelete: { ** ** See also: Rowid, MakeRecord. */ -case OP_Seek: case OP_IdxRowid: { /* out2 */ - VdbeCursor *pC; /* The P1 index cursor */ - VdbeCursor *pTabCur; /* The P2 table cursor (OP_Seek only) */ - i64 rowid; /* Rowid that P1 current points to */ + BtCursor *pCrsr; + VdbeCursor *pC; + i64 rowid; + pOut = out2Prerelease(p, pOp); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->eCurType==CURTYPE_BTREE ); - assert( pC->uc.pCursor!=0 ); + pCrsr = pC->pCursor; + assert( pCrsr!=0 ); + pOut->flags = MEM_Null; assert( pC->isTable==0 ); assert( pC->deferredMoveto==0 ); - assert( !pC->nullRow || pOp->opcode==OP_IdxRowid ); - - /* The IdxRowid and Seek opcodes are combined because of the commonality - ** of sqlite3VdbeCursorRestore() and sqlite3VdbeIdxRowid(). */ - rc = sqlite3VdbeCursorRestore(pC); /* sqlite3VbeCursorRestore() can only fail if the record has been deleted - ** out from under the cursor. That will never happens for an IdxRowid - ** or Seek opcode */ + ** out from under the cursor. That will never happend for an IdxRowid + ** opcode, hence the NEVER() arround the check of the return value. + */ + rc = sqlite3VdbeCursorRestore(pC); if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; if( !pC->nullRow ){ rowid = 0; /* Not needed. Only used to silence a warning. */ - rc = sqlite3VdbeIdxRowid(db, pC->uc.pCursor, &rowid); + rc = sqlite3VdbeIdxRowid(db, pCrsr, &rowid); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - if( pOp->opcode==OP_Seek ){ - assert( pOp->p3>=0 && pOp->p3nCursor ); - pTabCur = p->apCsr[pOp->p3]; - assert( pTabCur!=0 ); - assert( pTabCur->eCurType==CURTYPE_BTREE ); - assert( pTabCur->uc.pCursor!=0 ); - assert( pTabCur->isTable ); - pTabCur->nullRow = 0; - pTabCur->movetoTarget = rowid; - pTabCur->deferredMoveto = 1; - assert( pOp->p4type==P4_INTARRAY || pOp->p4.ai==0 ); - pTabCur->aAltMap = pOp->p4.ai; - pTabCur->pAltCursor = pC; - }else{ - pOut = out2Prerelease(p, pOp); - pOut->u.i = rowid; - pOut->flags = MEM_Int; - } - }else{ - assert( pOp->opcode==OP_IdxRowid ); - sqlite3VdbeMemSetNull(&aMem[pOp->p2]); + pOut->u.i = rowid; + pOut->flags = MEM_Int; } break; } @@ -85632,8 +78731,7 @@ case OP_IdxGE: { /* jump */ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->isOrdered ); - assert( pC->eCurType==CURTYPE_BTREE ); - assert( pC->uc.pCursor!=0); + assert( pC->pCursor!=0); assert( pC->deferredMoveto==0 ); assert( pOp->p5==0 || pOp->p5==1 ); assert( pOp->p4type==P4_INT32 ); @@ -85661,7 +78759,6 @@ case OP_IdxGE: { /* jump */ res++; } VdbeBranchTaken(res>0,2); - if( rc ) goto abort_due_to_error; if( res>0 ) goto jump_to_p2; break; } @@ -85691,13 +78788,11 @@ case OP_Destroy: { /* out2 */ int iDb; assert( p->readOnly==0 ); - assert( pOp->p1>1 ); pOut = out2Prerelease(p, pOp); pOut->flags = MEM_Null; if( db->nVdbeRead > db->nVDestroy+1 ){ rc = SQLITE_LOCKED; p->errorAction = OE_Abort; - goto abort_due_to_error; }else{ iDb = pOp->p3; assert( DbMaskTest(p->btreeMask, iDb) ); @@ -85705,9 +78800,8 @@ case OP_Destroy: { /* out2 */ rc = sqlite3BtreeDropTable(db->aDb[iDb].pBt, pOp->p1, &iMoved); pOut->flags = MEM_Int; pOut->u.i = iMoved; - if( rc ) goto abort_due_to_error; #ifndef SQLITE_OMIT_AUTOVACUUM - if( iMoved!=0 ){ + if( rc==SQLITE_OK && iMoved!=0 ){ sqlite3RootPageMoved(db, iDb, iMoved, pOp->p1); /* All OP_Destroy operations occur on the same btree */ assert( resetSchemaOnFault==0 || resetSchemaOnFault==iDb+1 ); @@ -85753,7 +78847,6 @@ case OP_Clear: { aMem[pOp->p3].u.i += nChange; } } - if( rc ) goto abort_due_to_error; break; } @@ -85771,13 +78864,11 @@ case OP_ResetSorter: { assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - if( isSorter(pC) ){ - sqlite3VdbeSorterReset(db, pC->uc.pSorter); + if( pC->pSorter ){ + sqlite3VdbeSorterReset(db, pC->pSorter); }else{ - assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->isEphemeral ); - rc = sqlite3BtreeClearTableOfCursor(pC->uc.pCursor); - if( rc ) goto abort_due_to_error; + rc = sqlite3BtreeClearTableOfCursor(pC->pCursor); } break; } @@ -85826,7 +78917,6 @@ case OP_CreateTable: { /* out2 */ flags = BTREE_BLOBKEY; } rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, flags); - if( rc ) goto abort_due_to_error; pOut->u.i = pgno; break; } @@ -85865,9 +78955,9 @@ case OP_ParseSchema: { initData.pzErrMsg = &p->zErrMsg; zSql = sqlite3MPrintf(db, "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid", - db->aDb[iDb].zDbSName, zMaster, pOp->p4.z); + db->aDb[iDb].zName, zMaster, pOp->p4.z); if( zSql==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; }else{ assert( db->init.busy==0 ); db->init.busy = 1; @@ -85879,12 +78969,9 @@ case OP_ParseSchema: { db->init.busy = 0; } } - if( rc ){ - sqlite3ResetAllSchemasOfConnection(db); - if( rc==SQLITE_NOMEM ){ - goto no_mem; - } - goto abort_due_to_error; + if( rc ) sqlite3ResetAllSchemasOfConnection(db); + if( rc==SQLITE_NOMEM ){ + goto no_mem; } break; } @@ -85899,7 +78986,6 @@ case OP_ParseSchema: { case OP_LoadAnalysis: { assert( pOp->p1>=0 && pOp->p1nDb ); rc = sqlite3AnalysisLoad(db, pOp->p1); - if( rc ) goto abort_due_to_error; break; } #endif /* !defined(SQLITE_OMIT_ANALYZE) */ @@ -85945,7 +79031,7 @@ case OP_DropTrigger: { #ifndef SQLITE_OMIT_INTEGRITY_CHECK -/* Opcode: IntegrityCk P1 P2 P3 P4 P5 +/* Opcode: IntegrityCk P1 P2 P3 * P5 ** ** Do an analysis of the currently open database. Store in ** register P1 the text of an error message describing any problems. @@ -85956,8 +79042,9 @@ case OP_DropTrigger: { ** In other words, the analysis stops as soon as reg(P1) errors are ** seen. Reg(P1) is updated with the number of errors remaining. ** -** The root page numbers of all tables in the database are integers -** stored in P4_INTARRAY argument. +** The root page numbers of all tables in the database are integer +** stored in reg(P1), reg(P1+1), reg(P1+2), .... There are P2 tables +** total. ** ** If P5 is not zero, the check is done on the auxiliary database ** file, not the main database file. @@ -85967,24 +79054,30 @@ case OP_DropTrigger: { case OP_IntegrityCk: { int nRoot; /* Number of tables to check. (Number of root pages.) */ int *aRoot; /* Array of rootpage numbers for tables to be checked */ + int j; /* Loop counter */ int nErr; /* Number of errors reported */ char *z; /* Text of the error report */ Mem *pnErr; /* Register keeping track of errors remaining */ assert( p->bIsReader ); nRoot = pOp->p2; - aRoot = pOp->p4.ai; assert( nRoot>0 ); - assert( aRoot[nRoot]==0 ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(nRoot+1) ); + if( aRoot==0 ) goto no_mem; + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pnErr = &aMem[pOp->p3]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); pIn1 = &aMem[pOp->p1]; + for(j=0; jp5nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot, (int)pnErr->u.i, &nErr); + sqlite3DbFree(db, aRoot); pnErr->u.i -= nErr; sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ @@ -86001,7 +79094,7 @@ case OP_IntegrityCk: { #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ /* Opcode: RowSetAdd P1 P2 * * * -** Synopsis: rowset(P1)=r[P2] +** Synopsis: rowset(P1)=r[P2] ** ** Insert the integer value held by register P2 into a boolean index ** held in register P1. @@ -86021,7 +79114,7 @@ case OP_RowSetAdd: { /* in1, in2 */ } /* Opcode: RowSetRead P1 P2 P3 * * -** Synopsis: r[P3]=rowset(P1) +** Synopsis: r[P3]=rowset(P1) ** ** Extract the smallest value from boolean index P1 and put that value into ** register P3. Or, if boolean index P1 is initially empty, leave P3 @@ -86151,8 +79244,8 @@ case OP_Program: { /* jump */ if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){ rc = SQLITE_ERROR; - sqlite3VdbeError(p, "too many levels of trigger recursion"); - goto abort_due_to_error; + sqlite3SetString(&p->zErrMsg, db, "too many levels of trigger recursion"); + break; } /* Register pRt is used to store the memory required to save the state @@ -86166,11 +79259,10 @@ case OP_Program: { /* jump */ ** variable nMem (and later, VdbeFrame.nChildMem) to this value. */ nMem = pProgram->nMem + pProgram->nCsr; - assert( nMem>0 ); - if( pProgram->nCsr==0 ) nMem++; nByte = ROUND8(sizeof(VdbeFrame)) + nMem * sizeof(Mem) - + pProgram->nCsr * sizeof(VdbeCursor *); + + pProgram->nCsr * sizeof(VdbeCursor *) + + pProgram->nOnce * sizeof(u8); pFrame = sqlite3DbMallocZero(db, nByte); if( !pFrame ){ goto no_mem; @@ -86190,6 +79282,8 @@ case OP_Program: { /* jump */ pFrame->aOp = p->aOp; pFrame->nOp = p->nOp; pFrame->token = pProgram->token; + pFrame->aOnceFlag = p->aOnceFlag; + pFrame->nOnceFlag = p->nOnceFlag; #ifdef SQLITE_ENABLE_STMT_SCANSTATUS pFrame->anExec = p->anExec; #endif @@ -86201,8 +79295,7 @@ case OP_Program: { /* jump */ } }else{ pFrame = pRt->u.pFrame; - assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem - || (pProgram->nCsr==0 && pProgram->nMem+1==pFrame->nChildMem) ); + assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem ); assert( pProgram->nCsr==pFrame->nChildCsr ); assert( (int)(pOp - aOp)==pFrame->pc ); } @@ -86212,21 +79305,21 @@ case OP_Program: { /* jump */ pFrame->lastRowid = lastRowid; pFrame->nChange = p->nChange; pFrame->nDbChange = p->db->nChange; - assert( pFrame->pAuxData==0 ); - pFrame->pAuxData = p->pAuxData; - p->pAuxData = 0; p->nChange = 0; p->pFrame = pFrame; - p->aMem = aMem = VdbeFrameMem(pFrame); + p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; p->nMem = pFrame->nChildMem; p->nCursor = (u16)pFrame->nChildCsr; - p->apCsr = (VdbeCursor **)&aMem[p->nMem]; + p->apCsr = (VdbeCursor **)&aMem[p->nMem+1]; p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; + p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; + p->nOnceFlag = pProgram->nOnce; #ifdef SQLITE_ENABLE_STMT_SCANSTATUS p->anExec = 0; #endif pOp = &aOp[-1]; + memset(p->aOnceFlag, 0, p->nOnceFlag); break; } @@ -86330,12 +79423,12 @@ case OP_MemMax: { /* in2 */ } #endif /* SQLITE_OMIT_AUTOINCREMENT */ -/* Opcode: IfPos P1 P2 P3 * * -** Synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 +/* Opcode: IfPos P1 P2 * * * +** Synopsis: if r[P1]>0 goto P2 ** ** Register P1 must contain an integer. -** If the value of register P1 is 1 or greater, subtract P3 from the -** value in P1 and jump to P2. +** If the value of register P1 is 1 or greater, jump to P2 and +** add the literal value P3 to register P1. ** ** If the initial value of register P1 is less than 1, then the ** value is unchanged and control passes through to the next instruction. @@ -86344,55 +79437,38 @@ case OP_IfPos: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); VdbeBranchTaken( pIn1->u.i>0, 2); - if( pIn1->u.i>0 ){ - pIn1->u.i -= pOp->p3; - goto jump_to_p2; - } + if( pIn1->u.i>0 ) goto jump_to_p2; break; } -/* Opcode: OffsetLimit P1 P2 P3 * * -** Synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) -** -** This opcode performs a commonly used computation associated with -** LIMIT and OFFSET process. r[P1] holds the limit counter. r[P3] -** holds the offset counter. The opcode computes the combined value -** of the LIMIT and OFFSET and stores that value in r[P2]. The r[P2] -** value computed is the total number of rows that will need to be -** visited in order to complete the query. -** -** If r[P3] is zero or negative, that means there is no OFFSET -** and r[P2] is set to be the value of the LIMIT, r[P1]. +/* Opcode: IfNeg P1 P2 P3 * * +** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2 ** -** if r[P1] is zero or negative, that means there is no LIMIT -** and r[P2] is set to -1. -** -** Otherwise, r[P2] is set to the sum of r[P1] and r[P3]. +** Register P1 must contain an integer. Add literal P3 to the value in +** register P1 then if the value of register P1 is less than zero, jump to P2. */ -case OP_OffsetLimit: { /* in1, out2, in3 */ +case OP_IfNeg: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; - pIn3 = &aMem[pOp->p3]; - pOut = out2Prerelease(p, pOp); - assert( pIn1->flags & MEM_Int ); - assert( pIn3->flags & MEM_Int ); - pOut->u.i = pIn1->u.i<=0 ? -1 : pIn1->u.i+(pIn3->u.i>0?pIn3->u.i:0); + assert( pIn1->flags&MEM_Int ); + pIn1->u.i += pOp->p3; + VdbeBranchTaken(pIn1->u.i<0, 2); + if( pIn1->u.i<0 ) goto jump_to_p2; break; } /* Opcode: IfNotZero P1 P2 P3 * * -** Synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 +** Synopsis: if r[P1]!=0 then r[P1]+=P3, goto P2 ** ** Register P1 must contain an integer. If the content of register P1 is -** initially nonzero, then subtract P3 from the value in register P1 and -** jump to P2. If register P1 is initially zero, leave it unchanged -** and fall through. +** initially nonzero, then add P3 to P1 and jump to P2. If register P1 is +** initially zero, leave it unchanged and fall through. */ case OP_IfNotZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); VdbeBranchTaken(pIn1->u.i<0, 2); if( pIn1->u.i ){ - pIn1->u.i -= pOp->p3; + pIn1->u.i += pOp->p3; goto jump_to_p2; } break; @@ -86414,102 +79490,72 @@ case OP_DecrJumpZero: { /* jump, in1 */ } -/* Opcode: AggStep0 * P2 P3 P4 P5 -** Synopsis: accum=r[P3] step(r[P2@P5]) +/* Opcode: JumpZeroIncr P1 P2 * * * +** Synopsis: if (r[P1]++)==0 ) goto P2 ** -** Execute the step function for an aggregate. The -** function has P5 arguments. P4 is a pointer to the FuncDef -** structure that specifies the function. Register P3 is the -** accumulator. -** -** The P5 arguments are taken from register P2 and its -** successors. +** The register P1 must contain an integer. If register P1 is initially +** zero, then jump to P2. Increment register P1 regardless of whether or +** not the jump is taken. */ +case OP_JumpZeroIncr: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + VdbeBranchTaken(pIn1->u.i==0, 2); + if( (pIn1->u.i++)==0 ) goto jump_to_p2; + break; +} + /* Opcode: AggStep * P2 P3 P4 P5 ** Synopsis: accum=r[P3] step(r[P2@P5]) ** ** Execute the step function for an aggregate. The -** function has P5 arguments. P4 is a pointer to an sqlite3_context -** object that is used to run the function. Register P3 is -** as the accumulator. +** function has P5 arguments. P4 is a pointer to the FuncDef +** structure that specifies the function. Use register +** P3 as the accumulator. ** ** The P5 arguments are taken from register P2 and its ** successors. -** -** This opcode is initially coded as OP_AggStep0. On first evaluation, -** the FuncDef stored in P4 is converted into an sqlite3_context and -** the opcode is changed. In this way, the initialization of the -** sqlite3_context only happens once, instead of on each call to the -** step function. */ -case OP_AggStep0: { - int n; - sqlite3_context *pCtx; - - assert( pOp->p4type==P4_FUNCDEF ); - n = pOp->p5; - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); - assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); - if( pCtx==0 ) goto no_mem; - pCtx->pMem = 0; - pCtx->pFunc = pOp->p4.pFunc; - pCtx->iOp = (int)(pOp - aOp); - pCtx->pVdbe = p; - pCtx->argc = n; - pOp->p4type = P4_FUNCCTX; - pOp->p4.pCtx = pCtx; - pOp->opcode = OP_AggStep; - /* Fall through into OP_AggStep */ -} case OP_AggStep: { + int n; int i; - sqlite3_context *pCtx; Mem *pMem; + Mem *pRec; Mem t; + sqlite3_context ctx; + sqlite3_value **apVal; - assert( pOp->p4type==P4_FUNCCTX ); - pCtx = pOp->p4.pCtx; - pMem = &aMem[pOp->p3]; - - /* If this function is inside of a trigger, the register array in aMem[] - ** might change from one evaluation to the next. The next block of code - ** checks to see if the register array has changed, and if so it - ** reinitializes the relavant parts of the sqlite3_context object */ - if( pCtx->pMem != pMem ){ - pCtx->pMem = pMem; - for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; - } - -#ifdef SQLITE_DEBUG - for(i=0; iargc; i++){ - assert( memIsValid(pCtx->argv[i]) ); - REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); + n = pOp->p5; + assert( n>=0 ); + pRec = &aMem[pOp->p2]; + apVal = p->apArg; + assert( apVal || n==0 ); + for(i=0; ip4.pFunc; + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + ctx.pMem = pMem = &aMem[pOp->p3]; pMem->n++; sqlite3VdbeMemInit(&t, db, MEM_Null); - pCtx->pOut = &t; - pCtx->fErrorOrAux = 0; - pCtx->skipFlag = 0; - (pCtx->pFunc->xSFunc)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ - if( pCtx->fErrorOrAux ){ - if( pCtx->isError ){ - sqlite3VdbeError(p, "%s", sqlite3_value_text(&t)); - rc = pCtx->isError; - } - sqlite3VdbeMemRelease(&t); - if( rc ) goto abort_due_to_error; - }else{ - assert( t.flags==MEM_Null ); + ctx.pOut = &t; + ctx.isError = 0; + ctx.pVdbe = p; + ctx.iOp = (int)(pOp - aOp); + ctx.skipFlag = 0; + (ctx.pFunc->xStep)(&ctx, n, apVal); /* IMP: R-24505-23230 */ + if( ctx.isError ){ + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&t)); + rc = ctx.isError; } - if( pCtx->skipFlag ){ + if( ctx.skipFlag ){ assert( pOp[-1].opcode==OP_CollSeq ); i = pOp[-1].p1; if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1); } + sqlite3VdbeMemRelease(&t); break; } @@ -86528,13 +79574,12 @@ case OP_AggStep: { */ case OP_AggFinal: { Mem *pMem; - assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); pMem = &aMem[pOp->p1]; assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); if( rc ){ - sqlite3VdbeError(p, "%s", sqlite3_value_text(pMem)); - goto abort_due_to_error; + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(pMem)); } sqlite3VdbeChangeEncoding(pMem, encoding); UPDATE_MAX_BLOBSIZE(pMem); @@ -86570,8 +79615,7 @@ case OP_Checkpoint: { || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); - if( rc ){ - if( rc!=SQLITE_BUSY ) goto abort_due_to_error; + if( rc==SQLITE_BUSY ){ rc = SQLITE_OK; aRes[0] = 1; } @@ -86640,11 +79684,11 @@ case OP_JournalMode: { /* out2 */ ){ if( !db->autoCommit || db->nVdbeRead>1 ){ rc = SQLITE_ERROR; - sqlite3VdbeError(p, + sqlite3SetString(&p->zErrMsg, db, "cannot change %s wal mode from within a transaction", (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of") ); - goto abort_due_to_error; + break; }else{ if( eOld==PAGER_JOURNALMODE_WAL ){ @@ -86674,7 +79718,9 @@ case OP_JournalMode: { /* out2 */ } #endif /* ifndef SQLITE_OMIT_WAL */ - if( rc ) eNew = eOld; + if( rc ){ + eNew = eOld; + } eNew = sqlite3PagerSetJournalMode(pPager, eNew); pOut->flags = MEM_Str|MEM_Static|MEM_Term; @@ -86682,21 +79728,20 @@ case OP_JournalMode: { /* out2 */ pOut->n = sqlite3Strlen30(pOut->z); pOut->enc = SQLITE_UTF8; sqlite3VdbeChangeEncoding(pOut, encoding); - if( rc ) goto abort_due_to_error; break; }; #endif /* SQLITE_OMIT_PRAGMA */ #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) -/* Opcode: Vacuum P1 * * * * +/* Opcode: Vacuum * * * * * ** -** Vacuum the entire database P1. P1 is 0 for "main", and 2 or more -** for an attached database. The "temp" database may not be vacuumed. +** Vacuum the entire database. This opcode will cause other virtual +** machines to be created and run. It may not be called from within +** a transaction. */ case OP_Vacuum: { assert( p->readOnly==0 ); - rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1); - if( rc ) goto abort_due_to_error; + rc = sqlite3RunVacuum(&p->zErrMsg, db); break; } #endif @@ -86717,8 +79762,7 @@ case OP_IncrVacuum: { /* jump */ pBt = db->aDb[pOp->p1].pBt; rc = sqlite3BtreeIncrVacuum(pBt); VdbeBranchTaken(rc==SQLITE_DONE,2); - if( rc ){ - if( rc!=SQLITE_DONE ) goto abort_due_to_error; + if( rc==SQLITE_DONE ){ rc = SQLITE_OK; goto jump_to_p2; } @@ -86769,12 +79813,9 @@ case OP_TableLock: { assert( DbMaskTest(p->btreeMask, p1) ); assert( isWriteLock==0 || isWriteLock==1 ); rc = sqlite3BtreeLockTable(db->aDb[p1].pBt, pOp->p2, isWriteLock); - if( rc ){ - if( (rc&0xFF)==SQLITE_LOCKED ){ - const char *z = pOp->p4.z; - sqlite3VdbeError(p, "database table is locked: %s", z); - } - goto abort_due_to_error; + if( (rc&0xFF)==SQLITE_LOCKED ){ + const char *z = pOp->p4.z; + sqlite3SetString(&p->zErrMsg, db, "database table is locked: %s", z); } } break; @@ -86796,7 +79837,6 @@ case OP_VBegin: { pVTab = pOp->p4.pVtab; rc = sqlite3VtabBegin(db, pVTab); if( pVTab ) sqlite3VtabImportErrmsg(p, pVTab->pVtab); - if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -86825,7 +79865,6 @@ case OP_VCreate: { rc = sqlite3VtabCallCreate(db, pOp->p1, zTab, &p->zErrMsg); } sqlite3VdbeMemRelease(&sMem); - if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -86840,7 +79879,6 @@ case OP_VDestroy: { db->nVDestroy++; rc = sqlite3VtabCallDestroy(db, pOp->p1, pOp->p4.z); db->nVDestroy--; - if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -86854,35 +79892,35 @@ case OP_VDestroy: { */ case OP_VOpen: { VdbeCursor *pCur; - sqlite3_vtab_cursor *pVCur; + sqlite3_vtab_cursor *pVtabCursor; sqlite3_vtab *pVtab; const sqlite3_module *pModule; assert( p->bIsReader ); pCur = 0; - pVCur = 0; + pVtabCursor = 0; pVtab = pOp->p4.pVtab->pVtab; if( pVtab==0 || NEVER(pVtab->pModule==0) ){ rc = SQLITE_LOCKED; - goto abort_due_to_error; + break; } pModule = pVtab->pModule; - rc = pModule->xOpen(pVtab, &pVCur); + rc = pModule->xOpen(pVtab, &pVtabCursor); sqlite3VtabImportErrmsg(p, pVtab); - if( rc ) goto abort_due_to_error; - - /* Initialize sqlite3_vtab_cursor base class */ - pVCur->pVtab = pVtab; + if( SQLITE_OK==rc ){ + /* Initialize sqlite3_vtab_cursor base class */ + pVtabCursor->pVtab = pVtab; - /* Initialize vdbe cursor object */ - pCur = allocateCursor(p, pOp->p1, 0, -1, CURTYPE_VTAB); - if( pCur ){ - pCur->uc.pVCur = pVCur; - pVtab->nRef++; - }else{ - assert( db->mallocFailed ); - pModule->xClose(pVCur); - goto no_mem; + /* Initialize vdbe cursor object */ + pCur = allocateCursor(p, pOp->p1, 0, -1, 0); + if( pCur ){ + pCur->pVtabCursor = pVtabCursor; + pVtab->nRef++; + }else{ + assert( db->mallocFailed ); + pModule->xClose(pVtabCursor); + goto no_mem; + } } break; } @@ -86914,7 +79952,7 @@ case OP_VFilter: { /* jump */ const sqlite3_module *pModule; Mem *pQuery; Mem *pArgc; - sqlite3_vtab_cursor *pVCur; + sqlite3_vtab_cursor *pVtabCursor; sqlite3_vtab *pVtab; VdbeCursor *pCur; int res; @@ -86926,9 +79964,9 @@ case OP_VFilter: { /* jump */ pCur = p->apCsr[pOp->p1]; assert( memIsValid(pQuery) ); REGISTER_TRACE(pOp->p3, pQuery); - assert( pCur->eCurType==CURTYPE_VTAB ); - pVCur = pCur->uc.pVCur; - pVtab = pVCur->pVtab; + assert( pCur->pVtabCursor ); + pVtabCursor = pCur->pVtabCursor; + pVtab = pVtabCursor->pVtab; pModule = pVtab->pModule; /* Grab the index number and argc parameters */ @@ -86942,10 +79980,11 @@ case OP_VFilter: { /* jump */ for(i = 0; ixFilter(pVCur, iQuery, pOp->p4.z, nArg, apArg); + rc = pModule->xFilter(pVtabCursor, iQuery, pOp->p4.z, nArg, apArg); sqlite3VtabImportErrmsg(p, pVtab); - if( rc ) goto abort_due_to_error; - res = pModule->xEof(pVCur); + if( rc==SQLITE_OK ){ + res = pModule->xEof(pVtabCursor); + } pCur->nullRow = 0; VdbeBranchTaken(res!=0,2); if( res ) goto jump_to_p2; @@ -86968,21 +80007,21 @@ case OP_VColumn: { sqlite3_context sContext; VdbeCursor *pCur = p->apCsr[pOp->p1]; - assert( pCur->eCurType==CURTYPE_VTAB ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( pCur->pVtabCursor ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); if( pCur->nullRow ){ sqlite3VdbeMemSetNull(pDest); break; } - pVtab = pCur->uc.pVCur->pVtab; + pVtab = pCur->pVtabCursor->pVtab; pModule = pVtab->pModule; assert( pModule->xColumn ); memset(&sContext, 0, sizeof(sContext)); sContext.pOut = pDest; MemSetTypeFlag(pDest, MEM_Null); - rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2); + rc = pModule->xColumn(pCur->pVtabCursor, &sContext, pOp->p2); sqlite3VtabImportErrmsg(p, pVtab); if( sContext.isError ){ rc = sContext.isError; @@ -86994,7 +80033,6 @@ case OP_VColumn: { if( sqlite3VdbeMemTooBig(pDest) ){ goto too_big; } - if( rc ) goto abort_due_to_error; break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -87014,11 +80052,11 @@ case OP_VNext: { /* jump */ res = 0; pCur = p->apCsr[pOp->p1]; - assert( pCur->eCurType==CURTYPE_VTAB ); + assert( pCur->pVtabCursor ); if( pCur->nullRow ){ break; } - pVtab = pCur->uc.pVCur->pVtab; + pVtab = pCur->pVtabCursor->pVtab; pModule = pVtab->pModule; assert( pModule->xNext ); @@ -87028,10 +80066,11 @@ case OP_VNext: { /* jump */ ** data is available) and the error code returned when xColumn or ** some other method is next invoked on the save virtual table cursor. */ - rc = pModule->xNext(pCur->uc.pVCur); + rc = pModule->xNext(pCur->pVtabCursor); sqlite3VtabImportErrmsg(p, pVtab); - if( rc ) goto abort_due_to_error; - res = pModule->xEof(pCur->uc.pVCur); + if( rc==SQLITE_OK ){ + res = pModule->xEof(pCur->pVtabCursor); + } VdbeBranchTaken(!res,2); if( !res ){ /* If there is data, jump to P2 */ @@ -87063,11 +80102,11 @@ case OP_VRename: { testcase( pName->enc==SQLITE_UTF16BE ); testcase( pName->enc==SQLITE_UTF16LE ); rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8); - if( rc ) goto abort_due_to_error; - rc = pVtab->pModule->xRename(pVtab, pName->z); - sqlite3VtabImportErrmsg(p, pVtab); - p->expired = 0; - if( rc ) goto abort_due_to_error; + if( rc==SQLITE_OK ){ + rc = pVtab->pModule->xRename(pVtab, pName->z); + sqlite3VtabImportErrmsg(p, pVtab); + p->expired = 0; + } break; } #endif @@ -87116,7 +80155,7 @@ case OP_VUpdate: { pVtab = pOp->p4.pVtab->pVtab; if( pVtab==0 || NEVER(pVtab->pModule==0) ){ rc = SQLITE_LOCKED; - goto abort_due_to_error; + break; } pModule = pVtab->pModule; nArg = pOp->p2; @@ -87148,7 +80187,6 @@ case OP_VUpdate: { }else{ p->nChange++; } - if( rc ) goto abort_due_to_error; } break; } @@ -87193,8 +80231,8 @@ case OP_MaxPgcnt: { /* out2 */ #endif -/* Opcode: Init P1 P2 * P4 * -** Synopsis: Start at P2 +/* Opcode: Init * P2 * P4 * +** Synopsis: Start at P2 ** ** Programs contain a single instance of this opcode as the very first ** opcode. @@ -87204,50 +80242,27 @@ case OP_MaxPgcnt: { /* out2 */ ** Or if P4 is blank, use the string returned by sqlite3_sql(). ** ** If P2 is not zero, jump to instruction P2. -** -** Increment the value of P1 so that OP_Once opcodes will jump the -** first time they are evaluated for this run. */ case OP_Init: { /* jump */ char *zTrace; - int i; - - /* If the P4 argument is not NULL, then it must be an SQL comment string. - ** The "--" string is broken up to prevent false-positives with srcck1.c. - ** - ** This assert() provides evidence for: - ** EVIDENCE-OF: R-50676-09860 The callback can compute the same text that - ** would have been returned by the legacy sqlite3_trace() interface by - ** using the X argument when X begins with "--" and invoking - ** sqlite3_expanded_sql(P) otherwise. - */ - assert( pOp->p4.z==0 || strncmp(pOp->p4.z, "-" "- ", 3)==0 ); - assert( pOp==p->aOp ); /* Always instruction 0 */ + char *z; #ifndef SQLITE_OMIT_TRACE - if( (db->mTrace & (SQLITE_TRACE_STMT|SQLITE_TRACE_LEGACY))!=0 + if( db->xTrace && !p->doingRerun && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 ){ -#ifndef SQLITE_OMIT_DEPRECATED - if( db->mTrace & SQLITE_TRACE_LEGACY ){ - void (*x)(void*,const char*) = (void(*)(void*,const char*))db->xTrace; - char *z = sqlite3VdbeExpandSql(p, zTrace); - x(db->pTraceArg, z); - sqlite3_free(z); - }else -#endif - { - (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, p, zTrace); - } + z = sqlite3VdbeExpandSql(p, zTrace); + db->xTrace(db->pTraceArg, z); + sqlite3DbFree(db, z); } #ifdef SQLITE_USE_FCNTL_TRACE zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql); if( zTrace ){ - int j; - for(j=0; jnDb; j++){ - if( DbMaskTest(p->btreeMask, j)==0 ) continue; - sqlite3_file_control(db, db->aDb[j].zDbSName, SQLITE_FCNTL_TRACE, zTrace); + int i; + for(i=0; inDb; i++){ + if( DbMaskTest(p->btreeMask, i)==0 ) continue; + sqlite3_file_control(db, db->aDb[i].zName, SQLITE_FCNTL_TRACE, zTrace); } } #endif /* SQLITE_USE_FCNTL_TRACE */ @@ -87259,39 +80274,10 @@ case OP_Init: { /* jump */ } #endif /* SQLITE_DEBUG */ #endif /* SQLITE_OMIT_TRACE */ - assert( pOp->p2>0 ); - if( pOp->p1>=sqlite3GlobalConfig.iOnceResetThreshold ){ - for(i=1; inOp; i++){ - if( p->aOp[i].opcode==OP_Once ) p->aOp[i].p1 = 0; - } - pOp->p1 = 0; - } - pOp->p1++; - goto jump_to_p2; -} - -#ifdef SQLITE_ENABLE_CURSOR_HINTS -/* Opcode: CursorHint P1 * * P4 * -** -** Provide a hint to cursor P1 that it only needs to return rows that -** satisfy the Expr in P4. TK_REGISTER terms in the P4 expression refer -** to values currently held in registers. TK_COLUMN terms in the P4 -** expression refer to columns in the b-tree to which cursor P1 is pointing. -*/ -case OP_CursorHint: { - VdbeCursor *pC; - - assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( pOp->p4type==P4_EXPR ); - pC = p->apCsr[pOp->p1]; - if( pC ){ - assert( pC->eCurType==CURTYPE_BTREE ); - sqlite3BtreeCursorHint(pC->uc.pCursor, BTREE_HINT_RANGE, - pOp->p4.pExpr, aMem); - } + if( pOp->p2 ) goto jump_to_p2; break; } -#endif /* SQLITE_ENABLE_CURSOR_HINTS */ + /* Opcode: Noop * * * * * ** @@ -87335,12 +80321,11 @@ default: { /* This is really OP_Noop and OP_Explain */ #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeTrace ){ - u8 opProperty = sqlite3OpcodeProperty[pOrigOp->opcode]; if( rc!=0 ) printf("rc=%d\n",rc); - if( opProperty & (OPFLG_OUT2) ){ + if( pOrigOp->opflags & (OPFLG_OUT2) ){ registerTrace(pOrigOp->p2, &aMem[pOrigOp->p2]); } - if( opProperty & OPFLG_OUT3 ){ + if( pOrigOp->opflags & OPFLG_OUT3 ){ registerTrace(pOrigOp->p3, &aMem[pOrigOp->p3]); } } @@ -87351,19 +80336,14 @@ default: { /* This is really OP_Noop and OP_Explain */ /* If we reach this point, it means that execution is finished with ** an error of some kind. */ -abort_due_to_error: - if( db->mallocFailed ) rc = SQLITE_NOMEM_BKPT; +vdbe_error_halt: assert( rc ); - if( p->zErrMsg==0 && rc!=SQLITE_IOERR_NOMEM ){ - sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); - } p->rc = rc; - sqlite3SystemError(db, rc); testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(rc, "statement aborts at %d: [%s] %s", (int)(pOp - aOp), p->zSql, p->zErrMsg); sqlite3VdbeHalt(p); - if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); + if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1; rc = SQLITE_ERROR; if( resetSchemaOnFault>0 ){ sqlite3ResetOneSchema(db, resetSchemaOnFault-1); @@ -87377,36 +80357,44 @@ default: { /* This is really OP_Noop and OP_Explain */ testcase( nVmStep>0 ); p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; sqlite3VdbeLeave(p); - assert( rc!=SQLITE_OK || nExtraDelete==0 - || sqlite3_strlike("DELETE%",p->zSql,0)!=0 - ); return rc; /* Jump to here if a string or blob larger than SQLITE_MAX_LENGTH ** is encountered. */ too_big: - sqlite3VdbeError(p, "string or blob too big"); + sqlite3SetString(&p->zErrMsg, db, "string or blob too big"); rc = SQLITE_TOOBIG; - goto abort_due_to_error; + goto vdbe_error_halt; /* Jump to here if a malloc() fails. */ no_mem: - sqlite3OomFault(db); - sqlite3VdbeError(p, "out of memory"); - rc = SQLITE_NOMEM_BKPT; - goto abort_due_to_error; + db->mallocFailed = 1; + sqlite3SetString(&p->zErrMsg, db, "out of memory"); + rc = SQLITE_NOMEM; + goto vdbe_error_halt; + + /* Jump to here for any other kind of fatal error. The "rc" variable + ** should hold the error number. + */ +abort_due_to_error: + assert( p->zErrMsg==0 ); + if( db->mallocFailed ) rc = SQLITE_NOMEM; + if( rc!=SQLITE_IOERR_NOMEM ){ + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); + } + goto vdbe_error_halt; /* Jump to here if the sqlite3_interrupt() API sets the interrupt ** flag. */ abort_due_to_interrupt: assert( db->u1.isInterrupted ); - rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; + rc = SQLITE_INTERRUPT; p->rc = rc; - sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); - goto abort_due_to_error; + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); + goto vdbe_error_halt; } @@ -87427,8 +80415,6 @@ default: { /* This is really OP_Noop and OP_Explain */ ** This file contains code used to implement incremental BLOB I/O. */ -/* #include "sqliteInt.h" */ -/* #include "vdbeInt.h" */ #ifndef SQLITE_OMIT_INCRBLOB @@ -87444,8 +80430,6 @@ struct Incrblob { BtCursor *pCsr; /* Cursor pointing at blob row */ sqlite3_stmt *pStmt; /* Statement holding cursor open */ sqlite3 *db; /* The associated database */ - char *zDb; /* Database name */ - Table *pTab; /* Table object */ }; @@ -87492,7 +80476,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ }else{ p->iOffset = pC->aType[p->iCol + pC->nField]; p->nByte = sqlite3VdbeSerialTypeLen(type); - p->pCsr = pC->uc.pCursor; + p->pCsr = pC->pCursor; sqlite3BtreeIncrblobCursor(p->pCsr); } } @@ -87520,7 +80504,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ /* ** Open a blob handle. */ -SQLITE_API int sqlite3_blob_open( +SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( sqlite3* db, /* The database connection */ const char *zDb, /* The attached database containing the blob */ const char *zTable, /* The table containing the blob */ @@ -87531,6 +80515,38 @@ SQLITE_API int sqlite3_blob_open( ){ int nAttempt = 0; int iCol; /* Index of zColumn in row-record */ + + /* This VDBE program seeks a btree cursor to the identified + ** db/table/row entry. The reason for using a vdbe program instead + ** of writing code to use the b-tree layer directly is that the + ** vdbe program will take advantage of the various transaction, + ** locking and error handling infrastructure built into the vdbe. + ** + ** After seeking the cursor, the vdbe executes an OP_ResultRow. + ** Code external to the Vdbe then "borrows" the b-tree cursor and + ** uses it to implement the blob_read(), blob_write() and + ** blob_bytes() functions. + ** + ** The sqlite3_blob_close() function finalizes the vdbe program, + ** which closes the b-tree cursor and (possibly) commits the + ** transaction. + */ + static const int iLn = VDBE_OFFSET_LINENO(4); + static const VdbeOpList openBlob[] = { + /* {OP_Transaction, 0, 0, 0}, // 0: Inserted separately */ + {OP_TableLock, 0, 0, 0}, /* 1: Acquire a read or write lock */ + /* One of the following two instructions is replaced by an OP_Noop. */ + {OP_OpenRead, 0, 0, 0}, /* 2: Open cursor 0 for reading */ + {OP_OpenWrite, 0, 0, 0}, /* 3: Open cursor 0 for read/write */ + {OP_Variable, 1, 1, 1}, /* 4: Push the rowid to the stack */ + {OP_NotExists, 0, 10, 1}, /* 5: Seek the cursor */ + {OP_Column, 0, 0, 1}, /* 6 */ + {OP_ResultRow, 1, 0, 0}, /* 7 */ + {OP_Goto, 0, 4, 0}, /* 8 */ + {OP_Close, 0, 0, 0}, /* 9 */ + {OP_Halt, 0, 0, 0}, /* 10 */ + }; + int rc = SQLITE_OK; char *zErr = 0; Table *pTab; @@ -87589,8 +80605,6 @@ SQLITE_API int sqlite3_blob_open( sqlite3BtreeLeaveAll(db); goto blob_open_out; } - pBlob->pTab = pTab; - pBlob->zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName; /* Now search pTab for the exact column. */ for(iCol=0; iColnCol; iCol++) { @@ -87633,8 +80647,7 @@ SQLITE_API int sqlite3_blob_open( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int j; for(j=0; jnKeyCol; j++){ - /* FIXME: Be smarter about indexes that use expressions */ - if( pIdx->aiColumn[j]==iCol || pIdx->aiColumn[j]==XN_EXPR ){ + if( pIdx->aiColumn[j]==iCol ){ zFault = "indexed"; } } @@ -87651,78 +80664,45 @@ SQLITE_API int sqlite3_blob_open( pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(pParse); assert( pBlob->pStmt || db->mallocFailed ); if( pBlob->pStmt ){ - - /* This VDBE program seeks a btree cursor to the identified - ** db/table/row entry. The reason for using a vdbe program instead - ** of writing code to use the b-tree layer directly is that the - ** vdbe program will take advantage of the various transaction, - ** locking and error handling infrastructure built into the vdbe. - ** - ** After seeking the cursor, the vdbe executes an OP_ResultRow. - ** Code external to the Vdbe then "borrows" the b-tree cursor and - ** uses it to implement the blob_read(), blob_write() and - ** blob_bytes() functions. - ** - ** The sqlite3_blob_close() function finalizes the vdbe program, - ** which closes the b-tree cursor and (possibly) commits the - ** transaction. - */ - static const int iLn = VDBE_OFFSET_LINENO(2); - static const VdbeOpList openBlob[] = { - {OP_TableLock, 0, 0, 0}, /* 0: Acquire a read or write lock */ - {OP_OpenRead, 0, 0, 0}, /* 1: Open a cursor */ - {OP_Variable, 1, 1, 0}, /* 2: Move ?1 into reg[1] */ - {OP_NotExists, 0, 7, 1}, /* 3: Seek the cursor */ - {OP_Column, 0, 0, 1}, /* 4 */ - {OP_ResultRow, 1, 0, 0}, /* 5 */ - {OP_Goto, 0, 2, 0}, /* 6 */ - {OP_Close, 0, 0, 0}, /* 7 */ - {OP_Halt, 0, 0, 0}, /* 8 */ - }; Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - VdbeOp *aOp; + sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, pTab->pSchema->schema_cookie, pTab->pSchema->iGeneration); sqlite3VdbeChangeP5(v, 1); - aOp = sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); + sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); /* Make sure a mutex is held on the table to be accessed */ sqlite3VdbeUsesBtree(v, iDb); - if( db->mallocFailed==0 ){ - assert( aOp!=0 ); - /* Configure the OP_TableLock instruction */ + /* Configure the OP_TableLock instruction */ #ifdef SQLITE_OMIT_SHARED_CACHE - aOp[0].opcode = OP_Noop; + sqlite3VdbeChangeToNoop(v, 1); #else - aOp[0].p1 = iDb; - aOp[0].p2 = pTab->tnum; - aOp[0].p3 = flags; - sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); - } - if( db->mallocFailed==0 ){ -#endif - - /* Remove either the OP_OpenWrite or OpenRead. Set the P2 - ** parameter of the other to pTab->tnum. */ - if( flags ) aOp[1].opcode = OP_OpenWrite; - aOp[1].p2 = pTab->tnum; - aOp[1].p3 = iDb; - - /* Configure the number of columns. Configure the cursor to - ** think that the table has one more column than it really - ** does. An OP_Column to retrieve this imaginary column will - ** always return an SQL NULL. This is useful because it means - ** we can invoke OP_Column to fill in the vdbe cursors type - ** and offset cache without causing any IO. - */ - aOp[1].p4type = P4_INT32; - aOp[1].p4.i = pTab->nCol+1; - aOp[4].p2 = pTab->nCol; - + sqlite3VdbeChangeP1(v, 1, iDb); + sqlite3VdbeChangeP2(v, 1, pTab->tnum); + sqlite3VdbeChangeP3(v, 1, flags); + sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); +#endif + + /* Remove either the OP_OpenWrite or OpenRead. Set the P2 + ** parameter of the other to pTab->tnum. */ + sqlite3VdbeChangeToNoop(v, 3 - flags); + sqlite3VdbeChangeP2(v, 2 + flags, pTab->tnum); + sqlite3VdbeChangeP3(v, 2 + flags, iDb); + + /* Configure the number of columns. Configure the cursor to + ** think that the table has one more column than it really + ** does. An OP_Column to retrieve this imaginary column will + ** always return an SQL NULL. This is useful because it means + ** we can invoke OP_Column to fill in the vdbe cursors type + ** and offset cache without causing any IO. + */ + sqlite3VdbeChangeP4(v, 2+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); + sqlite3VdbeChangeP2(v, 6, pTab->nCol); + if( !db->mallocFailed ){ pParse->nVar = 1; pParse->nMem = 1; pParse->nTab = 1; @@ -87761,7 +80741,7 @@ SQLITE_API int sqlite3_blob_open( ** Close a blob handle that was previously created using ** sqlite3_blob_open(). */ -SQLITE_API int sqlite3_blob_close(sqlite3_blob *pBlob){ +SQLITE_API int SQLITE_STDCALL sqlite3_blob_close(sqlite3_blob *pBlob){ Incrblob *p = (Incrblob *)pBlob; int rc; sqlite3 *db; @@ -87812,30 +80792,6 @@ static int blobReadWrite( */ assert( db == v->db ); sqlite3BtreeEnterCursor(p->pCsr); - -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK - if( xCall==sqlite3BtreePutData && db->xPreUpdateCallback ){ - /* If a pre-update hook is registered and this is a write cursor, - ** invoke it here. - ** - ** TODO: The preupdate-hook is passed SQLITE_DELETE, even though this - ** operation should really be an SQLITE_UPDATE. This is probably - ** incorrect, but is convenient because at this point the new.* values - ** are not easily obtainable. And for the sessions module, an - ** SQLITE_UPDATE where the PK columns do not change is handled in the - ** same way as an SQLITE_DELETE (the SQLITE_DELETE code is actually - ** slightly more efficient). Since you cannot write to a PK column - ** using the incremental-blob API, this works. For the sessions module - ** anyhow. - */ - sqlite3_int64 iKey; - iKey = sqlite3BtreeIntegerKey(p->pCsr); - sqlite3VdbePreUpdateHook( - v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1 - ); - } -#endif - rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); sqlite3BtreeLeaveCursor(p->pCsr); if( rc==SQLITE_ABORT ){ @@ -87854,14 +80810,14 @@ static int blobReadWrite( /* ** Read data from a blob handle. */ -SQLITE_API int sqlite3_blob_read(sqlite3_blob *pBlob, void *z, int n, int iOffset){ +SQLITE_API int SQLITE_STDCALL sqlite3_blob_read(sqlite3_blob *pBlob, void *z, int n, int iOffset){ return blobReadWrite(pBlob, z, n, iOffset, sqlite3BtreeData); } /* ** Write data to a blob handle. */ -SQLITE_API int sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int iOffset){ +SQLITE_API int SQLITE_STDCALL sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int iOffset){ return blobReadWrite(pBlob, (void *)z, n, iOffset, sqlite3BtreePutData); } @@ -87871,7 +80827,7 @@ SQLITE_API int sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int ** The Incrblob.nByte field is fixed for the lifetime of the Incrblob ** so no mutex is required for access. */ -SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){ +SQLITE_API int SQLITE_STDCALL sqlite3_blob_bytes(sqlite3_blob *pBlob){ Incrblob *p = (Incrblob *)pBlob; return (p && p->pStmt) ? p->nByte : 0; } @@ -87886,7 +80842,7 @@ SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){ ** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) ** immediately return SQLITE_ABORT. */ -SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ +SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ int rc; Incrblob *p = (Incrblob *)pBlob; sqlite3 *db; @@ -88057,8 +81013,6 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ ** thread to merge the output of each of the others to a single PMA for ** the main thread to read from. */ -/* #include "sqliteInt.h" */ -/* #include "vdbeInt.h" */ /* ** If SQLITE_DEBUG_SORTER_THREADS is defined, this module outputs various @@ -88462,7 +81416,7 @@ static int vdbePmaReadBlob( int nNew = MAX(128, p->nAlloc*2); while( nByte>nNew ) nNew = nNew*2; aNew = sqlite3Realloc(p->aAlloc, nNew); - if( !aNew ) return SQLITE_NOMEM_BKPT; + if( !aNew ) return SQLITE_NOMEM; p->nAlloc = nNew; p->aAlloc = aNew; } @@ -88574,7 +81528,7 @@ static int vdbePmaReaderSeek( int iBuf = pReadr->iReadOff % pgsz; if( pReadr->aBuffer==0 ){ pReadr->aBuffer = (u8*)sqlite3Malloc(pgsz); - if( pReadr->aBuffer==0 ) rc = SQLITE_NOMEM_BKPT; + if( pReadr->aBuffer==0 ) rc = SQLITE_NOMEM; pReadr->nBuffer = pgsz; } if( rc==SQLITE_OK && iBuf ){ @@ -88659,7 +81613,7 @@ static int vdbePmaReaderInit( rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); if( rc==SQLITE_OK ){ - u64 nByte = 0; /* Size of PMA in bytes */ + u64 nByte; /* Size of PMA in bytes */ rc = vdbePmaReadVarint(pReadr, &nByte); pReadr->iEof = pReadr->iReadOff + nByte; *pnByte += nByte; @@ -88853,6 +81807,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( ){ int pgsz; /* Page size of main database */ int i; /* Used to iterate through aTask[] */ + int mxCache; /* Cache size */ VdbeSorter *pSorter; /* The new sorter */ KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ @@ -88882,14 +81837,13 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( #endif assert( pCsr->pKeyInfo && pCsr->pBt==0 ); - assert( pCsr->eCurType==CURTYPE_SORTER ); szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); - pCsr->uc.pSorter = pSorter; + pCsr->pSorter = pSorter; if( pSorter==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; }else{ pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); @@ -88900,7 +81854,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( } pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nTask = nWorker + 1; - pSorter->iPrev = (u8)(nWorker - 1); + pSorter->iPrev = nWorker-1; pSorter->bUseThreads = (pSorter->nTask>1); pSorter->db = db; for(i=0; inTask; i++){ @@ -88909,20 +81863,11 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( } if( !sqlite3TempInMemory(db) ){ - i64 mxCache; /* Cache size in bytes*/ u32 szPma = sqlite3GlobalConfig.szPma; pSorter->mnPmaSize = szPma * pgsz; - mxCache = db->aDb[0].pSchema->cache_size; - if( mxCache<0 ){ - /* A negative cache-size value C indicates that the cache is abs(C) - ** KiB in size. */ - mxCache = mxCache * -1024; - }else{ - mxCache = mxCache * pgsz; - } - mxCache = MIN(mxCache, SQLITE_MAX_PMASZ); - pSorter->mxPmaSize = MAX(pSorter->mnPmaSize, (int)mxCache); + if( mxCache<(int)szPma ) mxCache = (int)szPma; + pSorter->mxPmaSize = MIN((i64)mxCache*pgsz, SQLITE_MAX_PMASZ); /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary @@ -88932,7 +81877,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( assert( pSorter->iMemory==0 ); pSorter->nMemory = pgsz; pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); - if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM_BKPT; + if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM; } } @@ -89180,14 +82125,12 @@ SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ ** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. */ SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ - VdbeSorter *pSorter; - assert( pCsr->eCurType==CURTYPE_SORTER ); - pSorter = pCsr->uc.pSorter; + VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); - pCsr->uc.pSorter = 0; + pCsr->pSorter = 0; } } @@ -89254,7 +82197,7 @@ static int vdbeSortAllocUnpacked(SortSubtask *pTask){ pTask->pSorter->pKeyInfo, 0, 0, &pFree ); assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); - if( pFree==0 ) return SQLITE_NOMEM_BKPT; + if( pFree==0 ) return SQLITE_NOMEM; pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField; pTask->pUnpacked->errCode = 0; } @@ -89264,18 +82207,19 @@ static int vdbeSortAllocUnpacked(SortSubtask *pTask){ /* ** Merge the two sorted lists p1 and p2 into a single list. +** Set *ppOut to the head of the new list. */ -static SorterRecord *vdbeSorterMerge( +static void vdbeSorterMerge( SortSubtask *pTask, /* Calling thread context */ SorterRecord *p1, /* First list to merge */ - SorterRecord *p2 /* Second list to merge */ + SorterRecord *p2, /* Second list to merge */ + SorterRecord **ppOut /* OUT: Head of merged list */ ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; int bCached = 0; - assert( p1!=0 && p2!=0 ); - for(;;){ + while( p1 && p2 ){ int res; res = pTask->xCompare( pTask, &bCached, SRVAL(p1), p1->nVal, SRVAL(p2), p2->nVal @@ -89285,22 +82229,15 @@ static SorterRecord *vdbeSorterMerge( *pp = p1; pp = &p1->u.pNext; p1 = p1->u.pNext; - if( p1==0 ){ - *pp = p2; - break; - } }else{ *pp = p2; pp = &p2->u.pNext; p2 = p2->u.pNext; bCached = 0; - if( p2==0 ){ - *pp = p1; - break; - } } } - return pFinal; + *pp = p1 ? p1 : p2; + *ppOut = pFinal; } /* @@ -89335,7 +82272,7 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } while( p ){ @@ -89353,7 +82290,7 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ p->u.pNext = 0; for(i=0; aSlot[i]; i++){ - p = vdbeSorterMerge(pTask, p, aSlot[i]); + vdbeSorterMerge(pTask, p, aSlot[i], &p); aSlot[i] = 0; } aSlot[i] = p; @@ -89362,8 +82299,7 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ p = 0; for(i=0; i<64; i++){ - if( aSlot[i]==0 ) continue; - p = p ? vdbeSorterMerge(pTask, p, aSlot[i]) : aSlot[i]; + vdbeSorterMerge(pTask, p, aSlot[i], &p); } pList->pList = p; @@ -89386,7 +82322,7 @@ static void vdbePmaWriterInit( memset(p, 0, sizeof(PmaWriter)); p->aBuffer = (u8*)sqlite3Malloc(nBuf); if( !p->aBuffer ){ - p->eFWErr = SQLITE_NOMEM_BKPT; + p->eFWErr = SQLITE_NOMEM; }else{ p->iBufEnd = p->iBufStart = (iStart % nBuf); p->iWriteOff = iStart - p->iBufStart; @@ -89674,7 +82610,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ pSorter->nMemory = sqlite3MallocSize(aMem); }else if( pSorter->list.aMemory ){ pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory); - if( !pSorter->list.aMemory ) return SQLITE_NOMEM_BKPT; + if( !pSorter->list.aMemory ) return SQLITE_NOMEM; } rc = vdbeSorterCreateThread(pTask, vdbeSorterFlushThread, pCtx); @@ -89692,16 +82628,15 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ - VdbeSorter *pSorter; + VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ + int bFlush; /* True to flush contents of memory to PMA */ int nReq; /* Bytes of memory required */ int nPMA; /* Bytes of PMA space required */ int t; /* serial type of first record field */ - assert( pCsr->eCurType==CURTYPE_SORTER ); - pSorter = pCsr->uc.pSorter; getVarint32((const u8*)&pVal->z[1], t); if( t>0 && t<10 && t!=7 ){ pSorter->typeMask &= SORTER_TYPE_INTEGER; @@ -89758,28 +82693,27 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( if( nMin>pSorter->nMemory ){ u8 *aNew; - int iListOff = (u8*)pSorter->list.pList - pSorter->list.aMemory; int nNew = pSorter->nMemory * 2; while( nNew < nMin ) nNew = nNew*2; if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; if( nNew < nMin ) nNew = nMin; aNew = sqlite3Realloc(pSorter->list.aMemory, nNew); - if( !aNew ) return SQLITE_NOMEM_BKPT; - pSorter->list.pList = (SorterRecord*)&aNew[iListOff]; + if( !aNew ) return SQLITE_NOMEM; + pSorter->list.pList = (SorterRecord*)( + aNew + ((u8*)pSorter->list.pList - pSorter->list.aMemory) + ); pSorter->list.aMemory = aNew; pSorter->nMemory = nNew; } pNew = (SorterRecord*)&pSorter->list.aMemory[pSorter->iMemory]; pSorter->iMemory += ROUND8(nReq); - if( pSorter->list.pList ){ - pNew->u.iNext = (int)((u8*)(pSorter->list.pList) - pSorter->list.aMemory); - } + pNew->u.iNext = (int)((u8*)(pSorter->list.pList) - pSorter->list.aMemory); }else{ pNew = (SorterRecord *)sqlite3Malloc(nReq); if( pNew==0 ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } pNew->u.pNext = pSorter->list.pList; } @@ -89926,7 +82860,7 @@ static int vdbeIncrMergerNew( pTask->file2.iEof += pIncr->mxSz; }else{ vdbeMergeEngineFree(pMerger); - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; } return rc; } @@ -90231,10 +83165,10 @@ static int vdbeMergeEngineLevel0( int rc = SQLITE_OK; *ppOut = pNew = vdbeMergeEngineNew(nPMA); - if( pNew==0 ) rc = SQLITE_NOMEM_BKPT; + if( pNew==0 ) rc = SQLITE_NOMEM; for(i=0; iaReadr[i]; rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pReadr, &nDummy); iOff = pReadr->iEof; @@ -90302,7 +83236,7 @@ static int vdbeSorterAddToTree( if( pReadr->pIncr==0 ){ MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); if( pNew==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; }else{ rc = vdbeIncrMergerNew(pTask, pNew, &pReadr->pIncr); } @@ -90347,7 +83281,7 @@ static int vdbeSorterMergeTreeBuild( assert( pSorter->bUseThreads || pSorter->nTask==1 ); if( pSorter->nTask>1 ){ pMain = vdbeMergeEngineNew(pSorter->nTask); - if( pMain==0 ) rc = SQLITE_NOMEM_BKPT; + if( pMain==0 ) rc = SQLITE_NOMEM; } #endif @@ -90365,7 +83299,7 @@ static int vdbeSorterMergeTreeBuild( int i; int iSeq = 0; pRoot = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); - if( pRoot==0 ) rc = SQLITE_NOMEM_BKPT; + if( pRoot==0 ) rc = SQLITE_NOMEM; for(i=0; inPMA && rc==SQLITE_OK; i += SORTER_MAX_MERGE_COUNT){ MergeEngine *pMerger = 0; /* New level-0 PMA merger */ int nReader; /* Number of level-0 PMAs to merge */ @@ -90436,7 +83370,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ if( rc==SQLITE_OK ){ pReadr = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); pSorter->pReader = pReadr; - if( pReadr==0 ) rc = SQLITE_NOMEM_BKPT; + if( pReadr==0 ) rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ rc = vdbeIncrMergerNew(pLast, pMain, &pReadr->pIncr); @@ -90494,11 +83428,9 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ ** in sorted order. */ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ - VdbeSorter *pSorter; + VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; /* Return code */ - assert( pCsr->eCurType==CURTYPE_SORTER ); - pSorter = pCsr->uc.pSorter; assert( pSorter ); /* If no data has been written to disk, then do not do so now. Instead, @@ -90542,11 +83474,9 @@ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ ** Advance to the next element in the sorter. */ SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ - VdbeSorter *pSorter; + VdbeSorter *pSorter = pCsr->pSorter; int rc; /* Return code */ - assert( pCsr->eCurType==CURTYPE_SORTER ); - pSorter = pCsr->uc.pSorter; assert( pSorter->bUsePMA || (pSorter->pReader==0 && pSorter->pMerger==0) ); if( pSorter->bUsePMA ){ assert( pSorter->pReader==0 || pSorter->pMerger==0 ); @@ -90606,14 +83536,12 @@ static void *vdbeSorterRowkey( ** Copy the current sorter key into the memory cell pOut. */ SQLITE_PRIVATE int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ - VdbeSorter *pSorter; + VdbeSorter *pSorter = pCsr->pSorter; void *pKey; int nKey; /* Sorter key to copy into pOut */ - assert( pCsr->eCurType==CURTYPE_SORTER ); - pSorter = pCsr->uc.pSorter; pKey = vdbeSorterRowkey(pSorter, &nKey); if( sqlite3VdbeMemClearAndResize(pOut, nKey) ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } pOut->n = nKey; MemSetTypeFlag(pOut, MEM_Blob); @@ -90644,21 +83572,17 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( int nKeyCol, /* Compare this many columns */ int *pRes /* OUT: Result of comparison */ ){ - VdbeSorter *pSorter; - UnpackedRecord *r2; - KeyInfo *pKeyInfo; + VdbeSorter *pSorter = pCsr->pSorter; + UnpackedRecord *r2 = pSorter->pUnpacked; + KeyInfo *pKeyInfo = pCsr->pKeyInfo; int i; void *pKey; int nKey; /* Sorter key to compare pVal with */ - assert( pCsr->eCurType==CURTYPE_SORTER ); - pSorter = pCsr->uc.pSorter; - r2 = pSorter->pUnpacked; - pKeyInfo = pCsr->pKeyInfo; if( r2==0 ){ char *p; r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo,0,0,&p); assert( pSorter->pUnpacked==(UnpackedRecord*)p ); - if( r2==0 ) return SQLITE_NOMEM_BKPT; + if( r2==0 ) return SQLITE_NOMEM; r2->nField = nKeyCol; } assert( r2->nField==nKeyCol ); @@ -90677,6 +83601,264 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( } /************** End of vdbesort.c ********************************************/ +/************** Begin file journal.c *****************************************/ +/* +** 2007 August 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a special kind of sqlite3_file object used +** by SQLite to create journal files if the atomic-write optimization +** is enabled. +** +** The distinctive characteristic of this sqlite3_file is that the +** actual on disk file is created lazily. When the file is created, +** the caller specifies a buffer size for an in-memory buffer to +** be used to service read() and write() requests. The actual file +** on disk is not created or populated until either: +** +** 1) The in-memory representation grows too large for the allocated +** buffer, or +** 2) The sqlite3JournalCreate() function is called. +*/ +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + + +/* +** A JournalFile object is a subclass of sqlite3_file used by +** as an open file handle for journal files. +*/ +struct JournalFile { + sqlite3_io_methods *pMethod; /* I/O methods on journal files */ + int nBuf; /* Size of zBuf[] in bytes */ + char *zBuf; /* Space to buffer journal writes */ + int iSize; /* Amount of zBuf[] currently used */ + int flags; /* xOpen flags */ + sqlite3_vfs *pVfs; /* The "real" underlying VFS */ + sqlite3_file *pReal; /* The "real" underlying file descriptor */ + const char *zJournal; /* Name of the journal file */ +}; +typedef struct JournalFile JournalFile; + +/* +** If it does not already exists, create and populate the on-disk file +** for JournalFile p. +*/ +static int createFile(JournalFile *p){ + int rc = SQLITE_OK; + if( !p->pReal ){ + sqlite3_file *pReal = (sqlite3_file *)&p[1]; + rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0); + if( rc==SQLITE_OK ){ + p->pReal = pReal; + if( p->iSize>0 ){ + assert(p->iSize<=p->nBuf); + rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); + } + if( rc!=SQLITE_OK ){ + /* If an error occurred while writing to the file, close it before + ** returning. This way, SQLite uses the in-memory journal data to + ** roll back changes made to the internal page-cache before this + ** function was called. */ + sqlite3OsClose(pReal); + p->pReal = 0; + } + } + } + return rc; +} + +/* +** Close the file. +*/ +static int jrnlClose(sqlite3_file *pJfd){ + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + sqlite3OsClose(p->pReal); + } + sqlite3_free(p->zBuf); + return SQLITE_OK; +} + +/* +** Read data from the file. +*/ +static int jrnlRead( + sqlite3_file *pJfd, /* The journal file from which to read */ + void *zBuf, /* Put the results here */ + int iAmt, /* Number of bytes to read */ + sqlite_int64 iOfst /* Begin reading at this offset */ +){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst); + }else if( (iAmt+iOfst)>p->iSize ){ + rc = SQLITE_IOERR_SHORT_READ; + }else{ + memcpy(zBuf, &p->zBuf[iOfst], iAmt); + } + return rc; +} + +/* +** Write data to the file. +*/ +static int jrnlWrite( + sqlite3_file *pJfd, /* The journal file into which to write */ + const void *zBuf, /* Take data to be written from here */ + int iAmt, /* Number of bytes to write */ + sqlite_int64 iOfst /* Begin writing at this offset into the file */ +){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( !p->pReal && (iOfst+iAmt)>p->nBuf ){ + rc = createFile(p); + } + if( rc==SQLITE_OK ){ + if( p->pReal ){ + rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); + }else{ + memcpy(&p->zBuf[iOfst], zBuf, iAmt); + if( p->iSize<(iOfst+iAmt) ){ + p->iSize = (iOfst+iAmt); + } + } + } + return rc; +} + +/* +** Truncate the file. +*/ +static int jrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsTruncate(p->pReal, size); + }else if( sizeiSize ){ + p->iSize = size; + } + return rc; +} + +/* +** Sync the file. +*/ +static int jrnlSync(sqlite3_file *pJfd, int flags){ + int rc; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsSync(p->pReal, flags); + }else{ + rc = SQLITE_OK; + } + return rc; +} + +/* +** Query the size of the file in bytes. +*/ +static int jrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsFileSize(p->pReal, pSize); + }else{ + *pSize = (sqlite_int64) p->iSize; + } + return rc; +} + +/* +** Table of methods for JournalFile sqlite3_file object. +*/ +static struct sqlite3_io_methods JournalFileMethods = { + 1, /* iVersion */ + jrnlClose, /* xClose */ + jrnlRead, /* xRead */ + jrnlWrite, /* xWrite */ + jrnlTruncate, /* xTruncate */ + jrnlSync, /* xSync */ + jrnlFileSize, /* xFileSize */ + 0, /* xLock */ + 0, /* xUnlock */ + 0, /* xCheckReservedLock */ + 0, /* xFileControl */ + 0, /* xSectorSize */ + 0, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0 /* xShmUnmap */ +}; + +/* +** Open a journal file. +*/ +SQLITE_PRIVATE int sqlite3JournalOpen( + sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ + const char *zName, /* Name of the journal file */ + sqlite3_file *pJfd, /* Preallocated, blank file handle */ + int flags, /* Opening flags */ + int nBuf /* Bytes buffered before opening the file */ +){ + JournalFile *p = (JournalFile *)pJfd; + memset(p, 0, sqlite3JournalSize(pVfs)); + if( nBuf>0 ){ + p->zBuf = sqlite3MallocZero(nBuf); + if( !p->zBuf ){ + return SQLITE_NOMEM; + } + }else{ + return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); + } + p->pMethod = &JournalFileMethods; + p->nBuf = nBuf; + p->flags = flags; + p->zJournal = zName; + p->pVfs = pVfs; + return SQLITE_OK; +} + +/* +** If the argument p points to a JournalFile structure, and the underlying +** file has not yet been created, create it now. +*/ +SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *p){ + if( p->pMethods!=&JournalFileMethods ){ + return SQLITE_OK; + } + return createFile((JournalFile *)p); +} + +/* +** The file-handle passed as the only argument is guaranteed to be an open +** file. It may or may not be of class JournalFile. If the file is a +** JournalFile, and the underlying file on disk has not yet been opened, +** return 0. Otherwise, return 1. +*/ +SQLITE_PRIVATE int sqlite3JournalExists(sqlite3_file *p){ + return (p->pMethods!=&JournalFileMethods || ((JournalFile *)p)->pReal!=0); +} + +/* +** Return the number of bytes required to store a JournalFile that uses vfs +** pVfs to create the underlying on-disk files. +*/ +SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ + return (pVfs->szOsFile+sizeof(JournalFile)); +} +#endif + +/************** End of journal.c *********************************************/ /************** Begin file memjournal.c **************************************/ /* ** 2008 October 7 @@ -90693,45 +83875,31 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( ** This file contains code use to implement an in-memory rollback journal. ** The in-memory rollback journal is used to journal transactions for ** ":memory:" databases and when the journal_mode=MEMORY pragma is used. -** -** Update: The in-memory journal is also used to temporarily cache -** smaller journals that are not critical for power-loss recovery. -** For example, statement journals that are not too big will be held -** entirely in memory, thus reducing the number of file I/O calls, and -** more importantly, reducing temporary file creation events. If these -** journals become too large for memory, they are spilled to disk. But -** in the common case, they are usually small and no file I/O needs to -** occur. */ -/* #include "sqliteInt.h" */ /* Forward references to internal structures */ typedef struct MemJournal MemJournal; typedef struct FilePoint FilePoint; typedef struct FileChunk FileChunk; +/* Space to hold the rollback journal is allocated in increments of +** this many bytes. +** +** The size chosen is a little less than a power of two. That way, +** the FileChunk object will have a size that almost exactly fills +** a power-of-two allocation. This minimizes wasted space in power-of-two +** memory allocators. +*/ +#define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) + /* ** The rollback journal is composed of a linked list of these structures. -** -** The zChunk array is always at least 8 bytes in size - usually much more. -** Its actual size is stored in the MemJournal.nChunkSize variable. */ struct FileChunk { FileChunk *pNext; /* Next chunk in the journal */ - u8 zChunk[8]; /* Content of this chunk */ + u8 zChunk[JOURNAL_CHUNKSIZE]; /* Content of this chunk */ }; -/* -** By default, allocate this many bytes of memory for each FileChunk object. -*/ -#define MEMJOURNAL_DFLT_FILECHUNKSIZE 1024 - -/* -** For chunk size nChunkSize, return the number of bytes that should -** be allocated for each FileChunk structure. -*/ -#define fileChunkSize(nChunkSize) (sizeof(FileChunk) + ((nChunkSize)-8)) - /* ** An instance of this object serves as a cursor into the rollback journal. ** The cursor can be either for reading or writing. @@ -90742,22 +83910,14 @@ struct FilePoint { }; /* -** This structure is a subclass of sqlite3_file. Each open memory-journal +** This subclass is a subclass of sqlite3_file. Each open memory-journal ** is an instance of this class. */ struct MemJournal { - const sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ - int nChunkSize; /* In-memory chunk-size */ - - int nSpill; /* Bytes of data before flushing */ - int nSize; /* Bytes of data currently in memory */ + sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ FileChunk *pFirst; /* Head of in-memory chunk-list */ FilePoint endpoint; /* Pointer to the end of the file */ FilePoint readpoint; /* Pointer to the end of the last xRead() */ - - int flags; /* xOpen flags */ - sqlite3_vfs *pVfs; /* The "real" underlying VFS */ - const char *zJournal; /* Name of the journal file */ }; /* @@ -90776,94 +83936,36 @@ static int memjrnlRead( int iChunkOffset; FileChunk *pChunk; -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - if( (iAmt+iOfst)>p->endpoint.iOffset ){ - return SQLITE_IOERR_SHORT_READ; - } -#endif + /* SQLite never tries to read past the end of a rollback journal file */ + assert( iOfst+iAmt<=p->endpoint.iOffset ); - assert( (iAmt+iOfst)<=p->endpoint.iOffset ); - assert( p->readpoint.iOffset==0 || p->readpoint.pChunk!=0 ); if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ sqlite3_int64 iOff = 0; for(pChunk=p->pFirst; - ALWAYS(pChunk) && (iOff+p->nChunkSize)<=iOfst; + ALWAYS(pChunk) && (iOff+JOURNAL_CHUNKSIZE)<=iOfst; pChunk=pChunk->pNext ){ - iOff += p->nChunkSize; + iOff += JOURNAL_CHUNKSIZE; } }else{ pChunk = p->readpoint.pChunk; - assert( pChunk!=0 ); } - iChunkOffset = (int)(iOfst%p->nChunkSize); + iChunkOffset = (int)(iOfst%JOURNAL_CHUNKSIZE); do { - int iSpace = p->nChunkSize - iChunkOffset; - int nCopy = MIN(nRead, (p->nChunkSize - iChunkOffset)); - memcpy(zOut, (u8*)pChunk->zChunk + iChunkOffset, nCopy); + int iSpace = JOURNAL_CHUNKSIZE - iChunkOffset; + int nCopy = MIN(nRead, (JOURNAL_CHUNKSIZE - iChunkOffset)); + memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy); zOut += nCopy; nRead -= iSpace; iChunkOffset = 0; } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); - p->readpoint.iOffset = pChunk ? iOfst+iAmt : 0; + p->readpoint.iOffset = iOfst+iAmt; p->readpoint.pChunk = pChunk; return SQLITE_OK; } -/* -** Free the list of FileChunk structures headed at MemJournal.pFirst. -*/ -static void memjrnlFreeChunks(MemJournal *p){ - FileChunk *pIter; - FileChunk *pNext; - for(pIter=p->pFirst; pIter; pIter=pNext){ - pNext = pIter->pNext; - sqlite3_free(pIter); - } - p->pFirst = 0; -} - -/* -** Flush the contents of memory to a real file on disk. -*/ -static int memjrnlCreateFile(MemJournal *p){ - int rc; - sqlite3_file *pReal = (sqlite3_file*)p; - MemJournal copy = *p; - - memset(p, 0, sizeof(MemJournal)); - rc = sqlite3OsOpen(copy.pVfs, copy.zJournal, pReal, copy.flags, 0); - if( rc==SQLITE_OK ){ - int nChunk = copy.nChunkSize; - i64 iOff = 0; - FileChunk *pIter; - for(pIter=copy.pFirst; pIter; pIter=pIter->pNext){ - if( iOff + nChunk > copy.endpoint.iOffset ){ - nChunk = copy.endpoint.iOffset - iOff; - } - rc = sqlite3OsWrite(pReal, (u8*)pIter->zChunk, nChunk, iOff); - if( rc ) break; - iOff += nChunk; - } - if( rc==SQLITE_OK ){ - /* No error has occurred. Free the in-memory buffers. */ - memjrnlFreeChunks(©); - } - } - if( rc!=SQLITE_OK ){ - /* If an error occurred while creating or writing to the file, restore - ** the original before returning. This way, SQLite uses the in-memory - ** journal data to roll back changes made to the internal page-cache - ** before this function was called. */ - sqlite3OsClose(pReal); - *p = copy; - } - return rc; -} - - /* ** Write data to the file. */ @@ -90877,62 +83979,38 @@ static int memjrnlWrite( int nWrite = iAmt; u8 *zWrite = (u8 *)zBuf; - /* If the file should be created now, create it and write the new data - ** into the file on disk. */ - if( p->nSpill>0 && (iAmt+iOfst)>p->nSpill ){ - int rc = memjrnlCreateFile(p); - if( rc==SQLITE_OK ){ - rc = sqlite3OsWrite(pJfd, zBuf, iAmt, iOfst); - } - return rc; - } + /* An in-memory journal file should only ever be appended to. Random + ** access writes are not required by sqlite. + */ + assert( iOfst==p->endpoint.iOffset ); + UNUSED_PARAMETER(iOfst); - /* If the contents of this write should be stored in memory */ - else{ - /* An in-memory journal file should only ever be appended to. Random - ** access writes are not required. The only exception to this is when - ** the in-memory journal is being used by a connection using the - ** atomic-write optimization. In this case the first 28 bytes of the - ** journal file may be written as part of committing the transaction. */ - assert( iOfst==p->endpoint.iOffset || iOfst==0 ); -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - if( iOfst==0 && p->pFirst ){ - assert( p->nChunkSize>iAmt ); - memcpy((u8*)p->pFirst->zChunk, zBuf, iAmt); - }else -#else - assert( iOfst>0 || p->pFirst==0 ); -#endif - { - while( nWrite>0 ){ - FileChunk *pChunk = p->endpoint.pChunk; - int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize); - int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset); - - if( iChunkOffset==0 ){ - /* New chunk is required to extend the file. */ - FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize)); - if( !pNew ){ - return SQLITE_IOERR_NOMEM_BKPT; - } - pNew->pNext = 0; - if( pChunk ){ - assert( p->pFirst ); - pChunk->pNext = pNew; - }else{ - assert( !p->pFirst ); - p->pFirst = pNew; - } - p->endpoint.pChunk = pNew; - } + while( nWrite>0 ){ + FileChunk *pChunk = p->endpoint.pChunk; + int iChunkOffset = (int)(p->endpoint.iOffset%JOURNAL_CHUNKSIZE); + int iSpace = MIN(nWrite, JOURNAL_CHUNKSIZE - iChunkOffset); - memcpy((u8*)p->endpoint.pChunk->zChunk + iChunkOffset, zWrite, iSpace); - zWrite += iSpace; - nWrite -= iSpace; - p->endpoint.iOffset += iSpace; + if( iChunkOffset==0 ){ + /* New chunk is required to extend the file. */ + FileChunk *pNew = sqlite3_malloc(sizeof(FileChunk)); + if( !pNew ){ + return SQLITE_IOERR_NOMEM; } - p->nSize = iAmt + iOfst; + pNew->pNext = 0; + if( pChunk ){ + assert( p->pFirst ); + pChunk->pNext = pNew; + }else{ + assert( !p->pFirst ); + p->pFirst = pNew; + } + p->endpoint.pChunk = pNew; } + + memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace); + zWrite += iSpace; + nWrite -= iSpace; + p->endpoint.iOffset += iSpace; } return SQLITE_OK; @@ -90940,21 +84018,19 @@ static int memjrnlWrite( /* ** Truncate the file. -** -** If the journal file is already on disk, truncate it there. Or, if it -** is still in main memory but is being truncated to zero bytes in size, -** ignore */ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ MemJournal *p = (MemJournal *)pJfd; - if( ALWAYS(size==0) ){ - memjrnlFreeChunks(p); - p->nSize = 0; - p->endpoint.pChunk = 0; - p->endpoint.iOffset = 0; - p->readpoint.pChunk = 0; - p->readpoint.iOffset = 0; - } + FileChunk *pChunk; + assert(size==0); + UNUSED_PARAMETER(size); + pChunk = p->pFirst; + while( pChunk ){ + FileChunk *pTmp = pChunk; + pChunk = pChunk->pNext; + sqlite3_free(pTmp); + } + sqlite3MemJournalOpen(pJfd); return SQLITE_OK; } @@ -90962,19 +84038,21 @@ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ ** Close the file. */ static int memjrnlClose(sqlite3_file *pJfd){ - MemJournal *p = (MemJournal *)pJfd; - memjrnlFreeChunks(p); + memjrnlTruncate(pJfd, 0); return SQLITE_OK; } + /* ** Sync the file. ** -** If the real file has been created, call its xSync method. Otherwise, -** syncing an in-memory journal is a no-op. +** Syncing an in-memory journal is a no-op. And, in fact, this routine +** is never called in a working implementation. This implementation +** exists purely as a contingency, in case some malfunction in some other +** part of SQLite causes Sync to be called by mistake. */ -static int memjrnlSync(sqlite3_file *pJfd, int flags){ - UNUSED_PARAMETER2(pJfd, flags); +static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); return SQLITE_OK; } @@ -91013,88 +84091,28 @@ static const struct sqlite3_io_methods MemJournalMethods = { }; /* -** Open a journal file. -** -** The behaviour of the journal file depends on the value of parameter -** nSpill. If nSpill is 0, then the journal file is always create and -** accessed using the underlying VFS. If nSpill is less than zero, then -** all content is always stored in main-memory. Finally, if nSpill is a -** positive value, then the journal file is initially created in-memory -** but may be flushed to disk later on. In this case the journal file is -** flushed to disk either when it grows larger than nSpill bytes in size, -** or when sqlite3JournalCreate() is called. -*/ -SQLITE_PRIVATE int sqlite3JournalOpen( - sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ - const char *zName, /* Name of the journal file */ - sqlite3_file *pJfd, /* Preallocated, blank file handle */ - int flags, /* Opening flags */ - int nSpill /* Bytes buffered before opening the file */ -){ - MemJournal *p = (MemJournal*)pJfd; - - /* Zero the file-handle object. If nSpill was passed zero, initialize - ** it using the sqlite3OsOpen() function of the underlying VFS. In this - ** case none of the code in this module is executed as a result of calls - ** made on the journal file-handle. */ - memset(p, 0, sizeof(MemJournal)); - if( nSpill==0 ){ - return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); - } - - if( nSpill>0 ){ - p->nChunkSize = nSpill; - }else{ - p->nChunkSize = 8 + MEMJOURNAL_DFLT_FILECHUNKSIZE - sizeof(FileChunk); - assert( MEMJOURNAL_DFLT_FILECHUNKSIZE==fileChunkSize(p->nChunkSize) ); - } - - p->pMethod = (const sqlite3_io_methods*)&MemJournalMethods; - p->nSpill = nSpill; - p->flags = flags; - p->zJournal = zName; - p->pVfs = pVfs; - return SQLITE_OK; -} - -/* -** Open an in-memory journal file. +** Open a journal file. */ SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *pJfd){ - sqlite3JournalOpen(0, 0, pJfd, 0, -1); -} - -#ifdef SQLITE_ENABLE_ATOMIC_WRITE -/* -** If the argument p points to a MemJournal structure that is not an -** in-memory-only journal file (i.e. is one that was opened with a +ve -** nSpill parameter), and the underlying file has not yet been created, -** create it now. -*/ -SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *p){ - int rc = SQLITE_OK; - if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nSpill>0 ){ - rc = memjrnlCreateFile((MemJournal*)p); - } - return rc; + MemJournal *p = (MemJournal *)pJfd; + assert( EIGHT_BYTE_ALIGNMENT(p) ); + memset(p, 0, sqlite3MemJournalSize()); + p->pMethod = (sqlite3_io_methods*)&MemJournalMethods; } -#endif /* -** The file-handle passed as the only argument is open on a journal file. -** Return true if this "journal file" is currently stored in heap memory, -** or false otherwise. +** Return true if the file-handle passed as an argument is +** an in-memory journal */ -SQLITE_PRIVATE int sqlite3JournalIsInMemory(sqlite3_file *p){ - return p->pMethods==&MemJournalMethods; +SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *pJfd){ + return pJfd->pMethods==&MemJournalMethods; } /* -** Return the number of bytes required to store a JournalFile that uses vfs -** pVfs to create the underlying on-disk files. +** Return the number of bytes required to store a MemJournal file descriptor. */ -SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ - return MAX(pVfs->szOsFile, (int)sizeof(MemJournal)); +SQLITE_PRIVATE int sqlite3MemJournalSize(void){ + return sizeof(MemJournal); } /************** End of memjournal.c ******************************************/ @@ -91113,7 +84131,6 @@ SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ ** This file contains routines used for walking the parser tree for ** an SQL statement. */ -/* #include "sqliteInt.h" */ /* #include */ /* #include */ @@ -91137,25 +84154,23 @@ SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. */ -static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ +SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ int rc; + if( pExpr==0 ) return WRC_Continue; testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); testcase( ExprHasProperty(pExpr, EP_Reduced) ); rc = pWalker->xExprCallback(pWalker, pExpr); - if( rc || ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ - return rc & WRC_Abort; - } - if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; - if( pExpr->pRight && walkExpr(pWalker, pExpr->pRight) ) return WRC_Abort; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; - }else if( pExpr->x.pList ){ - if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; + if( rc==WRC_Continue + && !ExprHasProperty(pExpr,EP_TokenOnly) ){ + if( sqlite3WalkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, pExpr->pRight) ) return WRC_Abort; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; + }else{ + if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; + } } - return WRC_Continue; -} -SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ - return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; + return rc & WRC_Abort; } /* @@ -91208,11 +84223,6 @@ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ if( sqlite3WalkSelect(pWalker, pItem->pSelect) ){ return WRC_Abort; } - if( pItem->fg.isTabFunc - && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg) - ){ - return WRC_Abort; - } } } return WRC_Continue; @@ -91279,7 +84289,6 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ ** resolve all identifiers by associating them with a particular ** table and column. */ -/* #include "sqliteInt.h" */ /* #include */ /* #include */ @@ -91310,6 +84319,30 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ ** Turn the pExpr expression into an alias for the iCol-th column of the ** result set in pEList. ** +** If the result set column is a simple column reference, then this routine +** makes an exact copy. But for any other kind of expression, this +** routine make a copy of the result set column as the argument to the +** TK_AS operator. The TK_AS operator causes the expression to be +** evaluated just once and then reused for each alias. +** +** The reason for suppressing the TK_AS term when the expression is a simple +** column reference is so that the column reference will be recognized as +** usable by indices within the WHERE clause processing logic. +** +** The TK_AS operator is inhibited if zType[0]=='G'. This means +** that in a GROUP BY clause, the expression is evaluated twice. Hence: +** +** SELECT random()%5 AS x, count(*) FROM tab GROUP BY x +** +** Is equivalent to: +** +** SELECT random()%5 AS x, count(*) FROM tab GROUP BY random()%5 +** +** The result of random()%5 in the GROUP BY clause is probably different +** from the result in the result-set. On the other hand Standard SQL does +** not allow the GROUP BY clause to contain references to result-set columns. +** So this should never come up in well-formed queries. +** ** If the reference is followed by a COLLATE operator, then make sure ** the COLLATE operator is preserved. For example: ** @@ -91343,11 +84376,19 @@ static void resolveAlias( db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( pDup==0 ) return; - if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery); + if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ + incrAggFunctionDepth(pDup, nSubquery); + pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); + if( pDup==0 ) return; + ExprSetProperty(pDup, EP_Skip); + if( pEList->a[iCol].u.x.iAlias==0 ){ + pEList->a[iCol].u.x.iAlias = (u16)(++pParse->nAlias); + } + pDup->iTable = pEList->a[iCol].u.x.iAlias; + } if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } - ExprSetProperty(pDup, EP_Alias); /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, @@ -91486,8 +84527,8 @@ static int lookupName( zDb = 0; }else{ for(i=0; inDb; i++){ - assert( db->aDb[i].zDbSName ); - if( sqlite3StrICmp(db->aDb[i].zDbSName,zDb)==0 ){ + assert( db->aDb[i].zName ); + if( sqlite3StrICmp(db->aDb[i].zName,zDb)==0 ){ pSchema = db->aDb[i].pSchema; break; } @@ -91539,7 +84580,7 @@ static int lookupName( ** USING clause, then skip this match. */ if( cnt==1 ){ - if( pItem->fg.jointype & JT_NATURAL ) continue; + if( pItem->jointype & JT_NATURAL ) continue; if( nameInUsingClause(pItem->pUsing, zCol) ) continue; } cnt++; @@ -91554,8 +84595,8 @@ static int lookupName( pExpr->iTable = pMatch->iCursor; pExpr->pTab = pMatch->pTab; /* RIGHT JOIN not (yet) supported */ - assert( (pMatch->fg.jointype & JT_RIGHT)==0 ); - if( (pMatch->fg.jointype & JT_LEFT)!=0 ){ + assert( (pMatch->jointype & JT_RIGHT)==0 ); + if( (pMatch->jointype & JT_LEFT)!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } pSchema = pExpr->pTab->pSchema; @@ -91591,8 +84632,9 @@ static int lookupName( break; } } - if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ + if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && HasRowid(pTab) ){ /* IMP: R-51414-32910 */ + /* IMP: R-44911-55124 */ iCol = -1; } if( iColnCol ){ @@ -91619,15 +84661,10 @@ static int lookupName( /* ** Perhaps the name is a reference to the ROWID */ - if( cnt==0 - && cntTab==1 - && pMatch - && (pNC->ncFlags & NC_IdxExpr)==0 - && sqlite3IsRowid(zCol) - && VisibleRowid(pMatch->pTab) - ){ + if( cnt==0 && cntTab==1 && pMatch && sqlite3IsRowid(zCol) + && HasRowid(pMatch->pTab) ){ cnt = 1; - pExpr->iColumn = -1; + pExpr->iColumn = -1; /* IMP: R-44911-55124 */ pExpr->affinity = SQLITE_AFF_INTEGER; } @@ -91644,9 +84681,9 @@ static int lookupName( ** resolved by the time the WHERE clause is resolved. ** ** The ability to use an output result-set column in the WHERE, GROUP BY, - ** or HAVING clauses, or as part of a larger expression in the ORDER BY + ** or HAVING clauses, or as part of a larger expression in the ORDRE BY ** clause is not standard SQL. This is a (goofy) SQLite extension, that - ** is supported for backwards compatibility only. Hence, we issue a warning + ** is supported for backwards compatibility only. TO DO: Issue a warning ** on sqlite3_log() whenever the capability is used. */ if( (pEList = pNC->pEList)!=0 @@ -91665,10 +84702,6 @@ static int lookupName( sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); return WRC_Abort; } - if( sqlite3ExprVectorSize(pOrig)!=1 ){ - sqlite3ErrorMsg(pParse, "row value misused"); - return WRC_Abort; - } resolveAlias(pParse, pEList, j, pExpr, "", nSubquery); cnt = 1; pMatch = 0; @@ -91747,7 +84780,7 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - if( !ExprHasProperty(pExpr, EP_Alias) ){ + if( pExpr->op!=TK_AS ){ sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); } /* Increment the nRef value on all name contexts from TopNC up to @@ -91788,25 +84821,36 @@ SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSr } /* -** Report an error that an expression is not valid for some set of -** pNC->ncFlags values determined by validMask. +** Report an error that an expression is not valid for a partial index WHERE +** clause. */ -static void notValid( +static void notValidPartIdxWhere( Parse *pParse, /* Leave error message here */ NameContext *pNC, /* The name context */ - const char *zMsg, /* Type of error */ - int validMask /* Set of contexts for which prohibited */ + const char *zMsg /* Type of error */ ){ - assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 ); - if( (pNC->ncFlags & validMask)!=0 ){ - const char *zIn = "partial index WHERE clauses"; - if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; + if( (pNC->ncFlags & NC_PartIdx)!=0 ){ + sqlite3ErrorMsg(pParse, "%s prohibited in partial index WHERE clauses", + zMsg); + } +} + #ifndef SQLITE_OMIT_CHECK - else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; -#endif - sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); +/* +** Report an error that an expression is not valid for a CHECK constraint. +*/ +static void notValidCheckConstraint( + Parse *pParse, /* Leave error message here */ + NameContext *pNC, /* The name context */ + const char *zMsg /* Type of error */ +){ + if( (pNC->ncFlags & NC_IsCheck)!=0 ){ + sqlite3ErrorMsg(pParse,"%s prohibited in CHECK constraints", zMsg); } } +#else +# define notValidCheckConstraint(P,N,M) +#endif /* ** Expression p should encode a floating point value between 1.0 and 0.0. @@ -91891,7 +84935,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ Expr *pRight; /* if( pSrcList==0 ) break; */ - notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; @@ -91914,24 +84957,26 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ int no_such_func = 0; /* True if no such function exists */ int wrong_num_args = 0; /* True if wrong number of arguments */ int is_agg = 0; /* True if is an aggregate function */ + int auth; /* Authorization to use the function */ int nId; /* Number of characters in function name */ const char *zId; /* The function name. */ FuncDef *pDef; /* Information about the function */ u8 enc = ENC(pParse->db); /* The database encoding */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + notValidPartIdxWhere(pParse, pNC, "functions"); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); - pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); + pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); if( pDef==0 ){ - pDef = sqlite3FindFunction(pParse->db, zId, -2, enc, 0); + pDef = sqlite3FindFunction(pParse->db, zId, nId, -2, enc, 0); if( pDef==0 ){ no_such_func = 1; }else{ wrong_num_args = 1; } }else{ - is_agg = pDef->xFinalize!=0; + is_agg = pDef->xFunc==0; if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ ExprSetProperty(pExpr, EP_Unlikely|EP_Skip); if( n==2 ){ @@ -91956,42 +85001,26 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } } #ifndef SQLITE_OMIT_AUTHORIZATION - { - int auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0,pDef->zName,0); - if( auth!=SQLITE_OK ){ - if( auth==SQLITE_DENY ){ - sqlite3ErrorMsg(pParse, "not authorized to use function: %s", - pDef->zName); - pNC->nErr++; - } - pExpr->op = TK_NULL; - return WRC_Prune; + auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); + if( auth!=SQLITE_OK ){ + if( auth==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized to use function: %s", + pDef->zName); + pNC->nErr++; } + pExpr->op = TK_NULL; + return WRC_Prune; } #endif - if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ - /* For the purposes of the EP_ConstFunc flag, date and time - ** functions and other functions that change slowly are considered - ** constant because they are constant for the duration of one query */ + if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ){ ExprSetProperty(pExpr,EP_ConstFunc); } - if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ - /* Date/time functions that use 'now', and other functions like - ** sqlite_version() that might change over time cannot be used - ** in an index. */ - notValid(pParse, pNC, "non-deterministic functions", - NC_IdxExpr|NC_PartIdx); - } } if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); pNC->nErr++; is_agg = 0; - }else if( no_such_func && pParse->db->init.busy==0 -#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION - && pParse->explain==0 -#endif - ){ + }else if( no_such_func && pParse->db->init.busy==0 ){ sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); pNC->nErr++; }else if( wrong_num_args ){ @@ -92031,56 +85060,21 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_IN ); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ int nRef = pNC->nRef; - notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr); + notValidCheckConstraint(pParse, pNC, "subqueries"); + notValidPartIdxWhere(pParse, pNC, "subqueries"); sqlite3WalkSelect(pWalker, pExpr->x.pSelect); assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); - pNC->ncFlags |= NC_VarSelect; } } break; } case TK_VARIABLE: { - notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); + notValidCheckConstraint(pParse, pNC, "parameters"); + notValidPartIdxWhere(pParse, pNC, "parameters"); break; } - case TK_BETWEEN: - case TK_EQ: - case TK_NE: - case TK_LT: - case TK_LE: - case TK_GT: - case TK_GE: - case TK_IS: - case TK_ISNOT: { - int nLeft, nRight; - if( pParse->db->mallocFailed ) break; - assert( pExpr->pLeft!=0 ); - nLeft = sqlite3ExprVectorSize(pExpr->pLeft); - if( pExpr->op==TK_BETWEEN ){ - nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[0].pExpr); - if( nRight==nLeft ){ - nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[1].pExpr); - } - }else{ - assert( pExpr->pRight!=0 ); - nRight = sqlite3ExprVectorSize(pExpr->pRight); - } - if( nLeft!=nRight ){ - testcase( pExpr->op==TK_EQ ); - testcase( pExpr->op==TK_NE ); - testcase( pExpr->op==TK_LT ); - testcase( pExpr->op==TK_LE ); - testcase( pExpr->op==TK_GT ); - testcase( pExpr->op==TK_GE ); - testcase( pExpr->op==TK_IS ); - testcase( pExpr->op==TK_ISNOT ); - testcase( pExpr->op==TK_BETWEEN ); - sqlite3ErrorMsg(pParse, "row value misused"); - } - break; - } } return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; } @@ -92422,6 +85416,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ int isCompound; /* True if p is a compound select */ int nCompound; /* Number of compound terms processed so far */ Parse *pParse; /* Parsing context */ + ExprList *pEList; /* Result set expression list */ int i; /* Loop counter */ ExprList *pGroupBy; /* The GROUP BY clause */ Select *pLeftmost; /* Left-most of SELECT of a compound */ @@ -92494,7 +85489,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** parent contexts. After resolving references to expressions in ** pItem->pSelect, check if this value has changed. If so, then ** SELECT statement pItem->pSelect must be correlated. Set the - ** pItem->fg.isCorrelated flag if this is the case. */ + ** pItem->isCorrelated flag if this is the case. */ for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef; if( pItem->zName ) pParse->zAuthContext = pItem->zName; @@ -92503,8 +85498,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pParse->nErr || db->mallocFailed ) return WRC_Abort; for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef; - assert( pItem->fg.isCorrelated==0 && nRef<=0 ); - pItem->fg.isCorrelated = (nRef!=0); + assert( pItem->isCorrelated==0 && nRef<=0 ); + pItem->isCorrelated = (nRef!=0); } } @@ -92516,7 +85511,14 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ sNC.pNext = pOuterNC; /* Resolve names in the result set. */ - if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; + pEList = p->pEList; + assert( pEList!=0 ); + for(i=0; inExpr; i++){ + Expr *pX = pEList->a[i].pExpr; + if( sqlite3ResolveExprNames(&sNC, pX) ){ + return WRC_Abort; + } + } /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. @@ -92549,16 +85551,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; - /* Resolve names in table-valued-function arguments */ - for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - if( pItem->fg.isTabFunc - && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) - ){ - return WRC_Abort; - } - } - /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries */ @@ -92613,13 +85605,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } - /* If this is part of a compound SELECT, check that it has the right - ** number of expressions in the select list. */ - if( p->pNext && p->pEList->nExpr!=p->pNext->pEList->nExpr ){ - sqlite3SelectWrongNumTermsError(pParse, p->pNext); - return WRC_Abort; - } - /* Advance to the next term of the compound */ p = p->pPrior; @@ -92703,12 +85688,10 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( #endif savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg); pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg); - w.pParse = pNC->pParse; + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; - w.xSelectCallback2 = 0; - w.walkerDepth = 0; - w.eCode = 0; + w.pParse = pNC->pParse; w.u.pNC = pNC; sqlite3WalkExpr(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 @@ -92724,23 +85707,6 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( return ExprHasProperty(pExpr, EP_Error); } -/* -** Resolve all names for all expression in an expression list. This is -** just like sqlite3ResolveExprNames() except that it works for an expression -** list rather than a single expression. -*/ -SQLITE_PRIVATE int sqlite3ResolveExprListNames( - NameContext *pNC, /* Namespace to resolve expressions in. */ - ExprList *pList /* The expression list to be analyzed. */ -){ - int i; - if( pList ){ - for(i=0; inExpr; i++){ - if( sqlite3ResolveExprNames(pNC, pList->a[i].pExpr) ) return WRC_Abort; - } - } - return WRC_Continue; -} /* ** Resolve all names in all expressions of a SELECT and in all @@ -92784,14 +85750,15 @@ SQLITE_PRIVATE void sqlite3ResolveSelectNames( SQLITE_PRIVATE void sqlite3ResolveSelfReference( Parse *pParse, /* Parsing context */ Table *pTab, /* The table being referenced */ - int type, /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */ + int type, /* NC_IsCheck or NC_PartIdx */ Expr *pExpr, /* Expression to resolve. May be NULL. */ ExprList *pList /* Expression list to resolve. May be NUL. */ ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ + int i; /* Loop counter */ - assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr ); + assert( type==NC_IsCheck || type==NC_PartIdx ); memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); sSrc.nSrc = 1; @@ -92802,7 +85769,13 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( sNC.pSrcList = &sSrc; sNC.ncFlags = type; if( sqlite3ResolveExprNames(&sNC, pExpr) ) return; - if( pList ) sqlite3ResolveExprListNames(&sNC, pList); + if( pList ){ + for(i=0; inExpr; i++){ + if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ + return; + } + } + } } /************** End of resolve.c *********************************************/ @@ -92821,19 +85794,6 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( ** This file contains routines used for analyzing expressions and ** for generating VDBE code that evaluates expressions in SQLite. */ -/* #include "sqliteInt.h" */ - -/* Forward declarations */ -static void exprCodeBetween(Parse*,Expr*,int,void(*)(Parse*,Expr*,int,int),int); -static int exprCodeVector(Parse *pParse, Expr *p, int *piToFree); - -/* -** Return the affinity character for a single column of a table. -*/ -SQLITE_PRIVATE char sqlite3TableColumnAffinity(Table *pTab, int iCol){ - assert( iColnCol ); - return iCol>=0 ? pTab->aCol[iCol].affinity : SQLITE_AFF_INTEGER; -} /* ** Return the 'affinity' of the expression pExpr if any. @@ -92860,21 +85820,21 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ assert( pExpr->flags&EP_xIsSelect ); return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); } - if( op==TK_REGISTER ) op = pExpr->op2; #ifndef SQLITE_OMIT_CAST if( op==TK_CAST ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); return sqlite3AffinityType(pExpr->u.zToken, 0); } #endif - if( op==TK_AGG_COLUMN || op==TK_COLUMN ){ - return sqlite3TableColumnAffinity(pExpr->pTab, pExpr->iColumn); - } - if( op==TK_SELECT_COLUMN ){ - assert( pExpr->pLeft->flags&EP_xIsSelect ); - return sqlite3ExprAffinity( - pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr - ); + if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) + && pExpr->pTab!=0 + ){ + /* op==TK_REGISTER && pExpr->pTab!=0 happens when pExpr was originally + ** a TK_COLUMN but was previously evaluated and cached in a register */ + int j = pExpr->iColumn; + if( j<0 ) return SQLITE_AFF_INTEGER; + assert( pExpr->pTab && jpTab->nCol ); + return pExpr->pTab->aCol[j].affinity; } return pExpr->affinity; } @@ -92906,12 +85866,13 @@ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken( SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ Token s; assert( zC!=0 ); - sqlite3TokenInit(&s, (char*)zC); + s.z = zC; + s.n = sqlite3Strlen30(s.z); return sqlite3ExprAddCollateToken(pParse, pExpr, &s, 0); } /* -** Skip over any TK_COLLATE operators and any unlikely() +** Skip over any TK_COLLATE or TK_AS operators and any unlikely() ** or likelihood() function at the root of an expression. */ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ @@ -92922,7 +85883,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; }else{ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr->op==TK_COLLATE || pExpr->op==TK_AS ); pExpr = pExpr->pLeft; } } @@ -93011,13 +85972,13 @@ SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ if( sqlite3IsNumericAffinity(aff1) || sqlite3IsNumericAffinity(aff2) ){ return SQLITE_AFF_NUMERIC; }else{ - return SQLITE_AFF_BLOB; + return SQLITE_AFF_NONE; } }else if( !aff1 && !aff2 ){ /* Neither side of the comparison is a column. Compare the ** results directly. */ - return SQLITE_AFF_BLOB; + return SQLITE_AFF_NONE; }else{ /* One side is a column, the other is not. Use the columns affinity. */ assert( aff1==0 || aff2==0 ); @@ -93040,8 +86001,8 @@ static char comparisonAffinity(Expr *pExpr){ aff = sqlite3CompareAffinity(pExpr->pRight, aff); }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); - }else if( NEVER(aff==0) ){ - aff = SQLITE_AFF_BLOB; + }else if( !aff ){ + aff = SQLITE_AFF_NONE; } return aff; } @@ -93055,7 +86016,7 @@ static char comparisonAffinity(Expr *pExpr){ SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity){ char aff = comparisonAffinity(pExpr); switch( aff ){ - case SQLITE_AFF_BLOB: + case SQLITE_AFF_NONE: return 1; case SQLITE_AFF_TEXT: return idx_affinity==SQLITE_AFF_TEXT; @@ -93130,270 +86091,6 @@ static int codeCompare( return addr; } -/* -** Return true if expression pExpr is a vector, or false otherwise. -** -** A vector is defined as any expression that results in two or more -** columns of result. Every TK_VECTOR node is an vector because the -** parser will not generate a TK_VECTOR with fewer than two entries. -** But a TK_SELECT might be either a vector or a scalar. It is only -** considered a vector if it has two or more result columns. -*/ -SQLITE_PRIVATE int sqlite3ExprIsVector(Expr *pExpr){ - return sqlite3ExprVectorSize(pExpr)>1; -} - -/* -** If the expression passed as the only argument is of type TK_VECTOR -** return the number of expressions in the vector. Or, if the expression -** is a sub-select, return the number of columns in the sub-select. For -** any other type of expression, return 1. -*/ -SQLITE_PRIVATE int sqlite3ExprVectorSize(Expr *pExpr){ - u8 op = pExpr->op; - if( op==TK_REGISTER ) op = pExpr->op2; - if( op==TK_VECTOR ){ - return pExpr->x.pList->nExpr; - }else if( op==TK_SELECT ){ - return pExpr->x.pSelect->pEList->nExpr; - }else{ - return 1; - } -} - -#ifndef SQLITE_OMIT_SUBQUERY -/* -** Return a pointer to a subexpression of pVector that is the i-th -** column of the vector (numbered starting with 0). The caller must -** ensure that i is within range. -** -** If pVector is really a scalar (and "scalar" here includes subqueries -** that return a single column!) then return pVector unmodified. -** -** pVector retains ownership of the returned subexpression. -** -** If the vector is a (SELECT ...) then the expression returned is -** just the expression for the i-th term of the result set, and may -** not be ready for evaluation because the table cursor has not yet -** been positioned. -*/ -SQLITE_PRIVATE Expr *sqlite3VectorFieldSubexpr(Expr *pVector, int i){ - assert( iop2==0 || pVector->op==TK_REGISTER ); - if( pVector->op==TK_SELECT || pVector->op2==TK_SELECT ){ - return pVector->x.pSelect->pEList->a[i].pExpr; - }else{ - return pVector->x.pList->a[i].pExpr; - } - } - return pVector; -} -#endif /* !defined(SQLITE_OMIT_SUBQUERY) */ - -#ifndef SQLITE_OMIT_SUBQUERY -/* -** Compute and return a new Expr object which when passed to -** sqlite3ExprCode() will generate all necessary code to compute -** the iField-th column of the vector expression pVector. -** -** It is ok for pVector to be a scalar (as long as iField==0). -** In that case, this routine works like sqlite3ExprDup(). -** -** The caller owns the returned Expr object and is responsible for -** ensuring that the returned value eventually gets freed. -** -** The caller retains ownership of pVector. If pVector is a TK_SELECT, -** then the returned object will reference pVector and so pVector must remain -** valid for the life of the returned object. If pVector is a TK_VECTOR -** or a scalar expression, then it can be deleted as soon as this routine -** returns. -** -** A trick to cause a TK_SELECT pVector to be deleted together with -** the returned Expr object is to attach the pVector to the pRight field -** of the returned TK_SELECT_COLUMN Expr object. -*/ -SQLITE_PRIVATE Expr *sqlite3ExprForVectorField( - Parse *pParse, /* Parsing context */ - Expr *pVector, /* The vector. List of expressions or a sub-SELECT */ - int iField /* Which column of the vector to return */ -){ - Expr *pRet; - if( pVector->op==TK_SELECT ){ - assert( pVector->flags & EP_xIsSelect ); - /* The TK_SELECT_COLUMN Expr node: - ** - ** pLeft: pVector containing TK_SELECT - ** pRight: not used. But recursively deleted. - ** iColumn: Index of a column in pVector - ** pLeft->iTable: First in an array of register holding result, or 0 - ** if the result is not yet computed. - ** - ** sqlite3ExprDelete() specifically skips the recursive delete of - ** pLeft on TK_SELECT_COLUMN nodes. But pRight is followed, so pVector - ** can be attached to pRight to cause this node to take ownership of - ** pVector. Typically there will be multiple TK_SELECT_COLUMN nodes - ** with the same pLeft pointer to the pVector, but only one of them - ** will own the pVector. - */ - pRet = sqlite3PExpr(pParse, TK_SELECT_COLUMN, 0, 0, 0); - if( pRet ){ - pRet->iColumn = iField; - pRet->pLeft = pVector; - } - assert( pRet==0 || pRet->iTable==0 ); - }else{ - if( pVector->op==TK_VECTOR ) pVector = pVector->x.pList->a[iField].pExpr; - pRet = sqlite3ExprDup(pParse->db, pVector, 0); - } - return pRet; -} -#endif /* !define(SQLITE_OMIT_SUBQUERY) */ - -/* -** If expression pExpr is of type TK_SELECT, generate code to evaluate -** it. Return the register in which the result is stored (or, if the -** sub-select returns more than one column, the first in an array -** of registers in which the result is stored). -** -** If pExpr is not a TK_SELECT expression, return 0. -*/ -static int exprCodeSubselect(Parse *pParse, Expr *pExpr){ - int reg = 0; -#ifndef SQLITE_OMIT_SUBQUERY - if( pExpr->op==TK_SELECT ){ - reg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); - } -#endif - return reg; -} - -/* -** Argument pVector points to a vector expression - either a TK_VECTOR -** or TK_SELECT that returns more than one column. This function returns -** the register number of a register that contains the value of -** element iField of the vector. -** -** If pVector is a TK_SELECT expression, then code for it must have -** already been generated using the exprCodeSubselect() routine. In this -** case parameter regSelect should be the first in an array of registers -** containing the results of the sub-select. -** -** If pVector is of type TK_VECTOR, then code for the requested field -** is generated. In this case (*pRegFree) may be set to the number of -** a temporary register to be freed by the caller before returning. -** -** Before returning, output parameter (*ppExpr) is set to point to the -** Expr object corresponding to element iElem of the vector. -*/ -static int exprVectorRegister( - Parse *pParse, /* Parse context */ - Expr *pVector, /* Vector to extract element from */ - int iField, /* Field to extract from pVector */ - int regSelect, /* First in array of registers */ - Expr **ppExpr, /* OUT: Expression element */ - int *pRegFree /* OUT: Temp register to free */ -){ - u8 op = pVector->op; - assert( op==TK_VECTOR || op==TK_REGISTER || op==TK_SELECT ); - if( op==TK_REGISTER ){ - *ppExpr = sqlite3VectorFieldSubexpr(pVector, iField); - return pVector->iTable+iField; - } - if( op==TK_SELECT ){ - *ppExpr = pVector->x.pSelect->pEList->a[iField].pExpr; - return regSelect+iField; - } - *ppExpr = pVector->x.pList->a[iField].pExpr; - return sqlite3ExprCodeTemp(pParse, *ppExpr, pRegFree); -} - -/* -** Expression pExpr is a comparison between two vector values. Compute -** the result of the comparison (1, 0, or NULL) and write that -** result into register dest. -** -** The caller must satisfy the following preconditions: -** -** if pExpr->op==TK_IS: op==TK_EQ and p5==SQLITE_NULLEQ -** if pExpr->op==TK_ISNOT: op==TK_NE and p5==SQLITE_NULLEQ -** otherwise: op==pExpr->op and p5==0 -*/ -static void codeVectorCompare( - Parse *pParse, /* Code generator context */ - Expr *pExpr, /* The comparison operation */ - int dest, /* Write results into this register */ - u8 op, /* Comparison operator */ - u8 p5 /* SQLITE_NULLEQ or zero */ -){ - Vdbe *v = pParse->pVdbe; - Expr *pLeft = pExpr->pLeft; - Expr *pRight = pExpr->pRight; - int nLeft = sqlite3ExprVectorSize(pLeft); - int i; - int regLeft = 0; - int regRight = 0; - u8 opx = op; - int addrDone = sqlite3VdbeMakeLabel(v); - - assert( nLeft==sqlite3ExprVectorSize(pRight) ); - assert( pExpr->op==TK_EQ || pExpr->op==TK_NE - || pExpr->op==TK_IS || pExpr->op==TK_ISNOT - || pExpr->op==TK_LT || pExpr->op==TK_GT - || pExpr->op==TK_LE || pExpr->op==TK_GE - ); - assert( pExpr->op==op || (pExpr->op==TK_IS && op==TK_EQ) - || (pExpr->op==TK_ISNOT && op==TK_NE) ); - assert( p5==0 || pExpr->op!=op ); - assert( p5==SQLITE_NULLEQ || pExpr->op==op ); - - p5 |= SQLITE_STOREP2; - if( opx==TK_LE ) opx = TK_LT; - if( opx==TK_GE ) opx = TK_GT; - - regLeft = exprCodeSubselect(pParse, pLeft); - regRight = exprCodeSubselect(pParse, pRight); - - for(i=0; 1 /*Loop exits by "break"*/; i++){ - int regFree1 = 0, regFree2 = 0; - Expr *pL, *pR; - int r1, r2; - assert( i>=0 && i0 ) sqlite3ExprCachePush(pParse); - r1 = exprVectorRegister(pParse, pLeft, i, regLeft, &pL, ®Free1); - r2 = exprVectorRegister(pParse, pRight, i, regRight, &pR, ®Free2); - codeCompare(pParse, pL, pR, opx, r1, r2, dest, p5); - testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); - testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); - testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); - testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - sqlite3ReleaseTempReg(pParse, regFree1); - sqlite3ReleaseTempReg(pParse, regFree2); - if( i>0 ) sqlite3ExprCachePop(pParse); - if( i==nLeft-1 ){ - break; - } - if( opx==TK_EQ ){ - sqlite3VdbeAddOp2(v, OP_IfNot, dest, addrDone); VdbeCoverage(v); - p5 |= SQLITE_KEEPNULL; - }else if( opx==TK_NE ){ - sqlite3VdbeAddOp2(v, OP_If, dest, addrDone); VdbeCoverage(v); - p5 |= SQLITE_KEEPNULL; - }else{ - assert( op==TK_LT || op==TK_GT || op==TK_LE || op==TK_GE ); - sqlite3VdbeAddOp2(v, OP_ElseNotEq, 0, addrDone); - VdbeCoverageIf(v, op==TK_LT); - VdbeCoverageIf(v, op==TK_GT); - VdbeCoverageIf(v, op==TK_LE); - VdbeCoverageIf(v, op==TK_GE); - if( i==nLeft-2 ) opx = op; - } - } - sqlite3VdbeResolveLabel(v, addrDone); -} - #if SQLITE_MAX_EXPR_DEPTH>0 /* ** Check that argument nHeight is less than or equal to the maximum @@ -93517,7 +86214,7 @@ SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ ** is responsible for making sure the node eventually gets freed. ** ** If dequote is true, then the token (if it exists) is dequoted. -** If dequote is false, no dequoting is performed. The deQuote +** If dequote is false, no dequoting is performance. The deQuote ** parameter is ignored if pToken is NULL or if the token does not ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. @@ -93529,7 +86226,7 @@ SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ ** is allocated to hold the integer text and the dequote flag is ignored. */ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( - sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ + sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ int op, /* Expression opcode */ const Token *pToken, /* Token argument. Might be NULL */ int dequote /* True to dequote */ @@ -93538,7 +86235,6 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( int nExtra = 0; int iValue = 0; - assert( db!=0 ); if( pToken ){ if( op!=TK_INTEGER || pToken->z==0 || sqlite3GetInt32(pToken->z, &iValue)==0 ){ @@ -93546,9 +86242,8 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( assert( iValue>=0 ); } } - pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)+nExtra); + pNew = sqlite3DbMallocZero(db, sizeof(Expr)+nExtra); if( pNew ){ - memset(pNew, 0, sizeof(Expr)); pNew->op = (u8)op; pNew->iAgg = -1; if( pToken ){ @@ -93556,13 +86251,15 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( pNew->flags |= EP_IntValue; pNew->u.iValue = iValue; }else{ + int c; pNew->u.zToken = (char*)&pNew[1]; assert( pToken->z!=0 || pToken->n==0 ); if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); pNew->u.zToken[pToken->n] = 0; - if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ - if( pNew->u.zToken[0]=='"' ) pNew->flags |= EP_DblQuoted; + if( dequote && nExtra>=3 + && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){ sqlite3Dequote(pNew->u.zToken); + if( c=='"' ) pNew->flags |= EP_DblQuoted; } } } @@ -93632,11 +86329,11 @@ SQLITE_PRIVATE Expr *sqlite3PExpr( const Token *pToken /* Argument token */ ){ Expr *p; - if( op==TK_AND && pParse->nErr==0 ){ + if( op==TK_AND && pLeft && pRight && pParse->nErr==0 ){ /* Take advantage of short-circuit false optimization for AND */ p = sqlite3ExprAnd(pParse->db, pLeft, pRight); }else{ - p = sqlite3ExprAlloc(pParse->db, op & TKFLG_MASK, pToken, 1); + p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); } if( p ) { @@ -93645,22 +86342,6 @@ SQLITE_PRIVATE Expr *sqlite3PExpr( return p; } -/* -** Add pSelect to the Expr.x.pSelect field. Or, if pExpr is NULL (due -** do a memory allocation failure) then delete the pSelect object. -*/ -SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pSelect){ - if( pExpr ){ - pExpr->x.pSelect = pSelect; - ExprSetProperty(pExpr, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, pExpr); - }else{ - assert( pParse->db->mallocFailed ); - sqlite3SelectDelete(pParse->db, pSelect); - } -} - - /* ** If the expression is always either TRUE or FALSE (respectively), ** then return 1. If one cannot determine the truth value of the @@ -93747,7 +86428,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token * ** instance of the wildcard, the next sequential variable number is ** assigned. */ -SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ +SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ sqlite3 *db = pParse->db; const char *z; @@ -93756,19 +86437,19 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n z = pExpr->u.zToken; assert( z!=0 ); assert( z[0]!=0 ); - assert( n==sqlite3Strlen30(z) ); if( z[1]==0 ){ /* Wildcard of the form "?". Assign the next variable number */ assert( z[0]=='?' ); pExpr->iColumn = (ynVar)(++pParse->nVar); }else{ - ynVar x; + ynVar x = 0; + u32 n = sqlite3Strlen30(z); if( z[0]=='?' ){ /* Wildcard of the form "?nnn". Convert "nnn" to an integer and ** use it as the variable number */ i64 i; int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); - x = (ynVar)i; + pExpr->iColumn = x = (ynVar)i; testcase( i==0 ); testcase( i==1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); @@ -93776,7 +86457,7 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); - return; + x = 0; } if( i>pParse->nVar ){ pParse->nVar = (int)i; @@ -93787,31 +86468,30 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n ** has never appeared before, reuse the same variable number */ ynVar i; - for(i=x=0; inzVar; i++){ + for(i=0; inzVar; i++){ if( pParse->azVar[i] && strcmp(pParse->azVar[i],z)==0 ){ - x = (ynVar)i+1; + pExpr->iColumn = x = (ynVar)i+1; break; } } - if( x==0 ) x = (ynVar)(++pParse->nVar); + if( x==0 ) x = pExpr->iColumn = (ynVar)(++pParse->nVar); } - pExpr->iColumn = x; - if( x>pParse->nzVar ){ - char **a; - a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); - if( a==0 ){ - assert( db->mallocFailed ); /* Error reported through mallocFailed */ - return; + if( x>0 ){ + if( x>pParse->nzVar ){ + char **a; + a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); + if( a==0 ) return; /* Error reported through db->mallocFailed */ + pParse->azVar = a; + memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); + pParse->nzVar = x; + } + if( z[0]!='?' || pParse->azVar[x-1]==0 ){ + sqlite3DbFree(db, pParse->azVar[x-1]); + pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); } - pParse->azVar = a; - memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); - pParse->nzVar = x; - } - if( pParse->azVar[x-1]==0 ){ - pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); } } - if( pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + if( !pParse->nErr && pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "too many SQL variables"); } } @@ -93819,36 +86499,26 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n /* ** Recursively delete an expression tree. */ -static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ - assert( p!=0 ); +SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ + if( p==0 ) return; /* Sanity check: Assert that the IntValue is non-negative if it exists */ assert( !ExprHasProperty(p, EP_IntValue) || p->u.iValue>=0 ); -#ifdef SQLITE_DEBUG - if( ExprHasProperty(p, EP_Leaf) && !ExprHasProperty(p, EP_TokenOnly) ){ - assert( p->pLeft==0 ); - assert( p->pRight==0 ); - assert( p->x.pSelect==0 ); - } -#endif - if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ + if( !ExprHasProperty(p, EP_TokenOnly) ){ /* The Expr.x union is never used at the same time as Expr.pRight */ assert( p->x.pList==0 || p->pRight==0 ); - if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); + sqlite3ExprDelete(db, p->pLeft); sqlite3ExprDelete(db, p->pRight); + if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); if( ExprHasProperty(p, EP_xIsSelect) ){ sqlite3SelectDelete(db, p->x.pSelect); }else{ sqlite3ExprListDelete(db, p->x.pList); } } - if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); if( !ExprHasProperty(p, EP_Static) ){ sqlite3DbFree(db, p); } } -SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ - if( p ) sqlite3ExprDeleteNN(db, p); -} /* ** Return the number of bytes allocated for the expression structure @@ -93900,7 +86570,7 @@ static int dupedExprStructSize(Expr *p, int flags){ assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==flags ){ + if( 0==(flags&EXPRDUP_REDUCE) ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); @@ -93962,92 +86632,84 @@ static int dupedExprSize(Expr *p, int flags){ ** if any. Before returning, *pzBuffer is set to the first byte past the ** portion of the buffer copied into by this function. */ -static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ - Expr *pNew; /* Value to return */ - u8 *zAlloc; /* Memory space from which to build Expr object */ - u32 staticFlag; /* EP_Static if space not obtained from malloc */ - - assert( db!=0 ); - assert( p ); - assert( dupFlags==0 || dupFlags==EXPRDUP_REDUCE ); - assert( pzBuffer==0 || dupFlags==EXPRDUP_REDUCE ); +static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ + Expr *pNew = 0; /* Value to return */ + if( p ){ + const int isReduced = (flags&EXPRDUP_REDUCE); + u8 *zAlloc; + u32 staticFlag = 0; - /* Figure out where to write the new Expr structure. */ - if( pzBuffer ){ - zAlloc = *pzBuffer; - staticFlag = EP_Static; - }else{ - zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, dupFlags)); - staticFlag = 0; - } - pNew = (Expr *)zAlloc; + assert( pzBuffer==0 || isReduced ); - if( pNew ){ - /* Set nNewSize to the size allocated for the structure pointed to - ** by pNew. This is either EXPR_FULLSIZE, EXPR_REDUCEDSIZE or - ** EXPR_TOKENONLYSIZE. nToken is set to the number of bytes consumed - ** by the copy of the p->u.zToken string (if any). - */ - const unsigned nStructSize = dupedExprStructSize(p, dupFlags); - const int nNewSize = nStructSize & 0xfff; - int nToken; - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nToken = sqlite3Strlen30(p->u.zToken) + 1; + /* Figure out where to write the new Expr structure. */ + if( pzBuffer ){ + zAlloc = *pzBuffer; + staticFlag = EP_Static; }else{ - nToken = 0; + zAlloc = sqlite3DbMallocRaw(db, dupedExprSize(p, flags)); } - if( dupFlags ){ - assert( ExprHasProperty(p, EP_Reduced)==0 ); - memcpy(zAlloc, p, nNewSize); - }else{ - u32 nSize = (u32)exprStructSize(p); - memcpy(zAlloc, p, nSize); - if( nSizeu.zToken string (if any). + */ + const unsigned nStructSize = dupedExprStructSize(p, flags); + const int nNewSize = nStructSize & 0xfff; + int nToken; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; + }else{ + nToken = 0; + } + if( isReduced ){ + assert( ExprHasProperty(p, EP_Reduced)==0 ); + memcpy(zAlloc, p, nNewSize); + }else{ + int nSize = exprStructSize(p); + memcpy(zAlloc, p, nSize); memset(&zAlloc[nSize], 0, EXPR_FULLSIZE-nSize); } - } - /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ - pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); - pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); - pNew->flags |= staticFlag; + /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ + pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); + pNew->flags |= staticFlag; - /* Copy the p->u.zToken string, if any. */ - if( nToken ){ - char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; - memcpy(zToken, p->u.zToken, nToken); - } - - if( 0==((p->flags|pNew->flags) & (EP_TokenOnly|EP_Leaf)) ){ - /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ - if( ExprHasProperty(p, EP_xIsSelect) ){ - pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, dupFlags); - }else{ - pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, dupFlags); + /* Copy the p->u.zToken string, if any. */ + if( nToken ){ + char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + memcpy(zToken, p->u.zToken, nToken); } - } - /* Fill in pNew->pLeft and pNew->pRight. */ - if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ - zAlloc += dupedExprNodeSize(p, dupFlags); - if( !ExprHasProperty(pNew, EP_TokenOnly|EP_Leaf) ){ - pNew->pLeft = p->pLeft ? - exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc) : 0; - pNew->pRight = p->pRight ? - exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; - } - if( pzBuffer ){ - *pzBuffer = zAlloc; - } - }else{ - if( !ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ - if( pNew->op==TK_SELECT_COLUMN ){ - pNew->pLeft = p->pLeft; + if( 0==((p->flags|pNew->flags) & EP_TokenOnly) ){ + /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ + if( ExprHasProperty(p, EP_xIsSelect) ){ + pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, isReduced); }else{ + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, isReduced); + } + } + + /* Fill in pNew->pLeft and pNew->pRight. */ + if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ + zAlloc += dupedExprNodeSize(p, flags); + if( ExprHasProperty(pNew, EP_Reduced) ){ + pNew->pLeft = exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc); + pNew->pRight = exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc); + } + if( pzBuffer ){ + *pzBuffer = zAlloc; + } + }else{ + if( !ExprHasProperty(p, EP_TokenOnly) ){ pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); + pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); } - pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); } + } } return pNew; @@ -94098,20 +86760,18 @@ static With *withDup(sqlite3 *db, With *p){ ** part of the in-memory representation of the database schema. */ SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3 *db, Expr *p, int flags){ - assert( flags==0 || flags==EXPRDUP_REDUCE ); - return p ? exprDup(db, p, flags, 0) : 0; + return exprDup(db, p, flags, 0); } SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ ExprList *pNew; struct ExprList_item *pItem, *pOldItem; int i; - assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); if( pNew==0 ) return 0; pNew->nExpr = i = p->nExpr; if( (flags & EXPRDUP_REDUCE)==0 ) for(i=1; inExpr; i+=i){} - pNew->a = pItem = sqlite3DbMallocRawNN(db, i*sizeof(p->a[0]) ); + pNew->a = pItem = sqlite3DbMallocRaw(db, i*sizeof(p->a[0]) ); if( pItem==0 ){ sqlite3DbFree(db, pNew); return 0; @@ -94142,10 +86802,9 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ SrcList *pNew; int i; int nByte; - assert( db!=0 ); if( p==0 ) return 0; nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); - pNew = sqlite3DbMallocRawNN(db, nByte ); + pNew = sqlite3DbMallocRaw(db, nByte ); if( pNew==0 ) return 0; pNew->nSrc = pNew->nAlloc = p->nSrc; for(i=0; inSrc; i++){ @@ -94156,18 +86815,16 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->fg = pOldItem->fg; + pNewItem->jointype = pOldItem->jointype; pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; - if( pNewItem->fg.isIndexedBy ){ - pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); - } - pNewItem->pIBIndex = pOldItem->pIBIndex; - if( pNewItem->fg.isTabFunc ){ - pNewItem->u1.pFuncArg = - sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); - } + pNewItem->isCorrelated = pOldItem->isCorrelated; + pNewItem->viaCoroutine = pOldItem->viaCoroutine; + pNewItem->isRecursive = pOldItem->isRecursive; + pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); + pNewItem->notIndexed = pOldItem->notIndexed; + pNewItem->pIndex = pOldItem->pIndex; pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ pTab->nRef++; @@ -94182,12 +86839,11 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ IdList *pNew; int i; - assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); if( pNew==0 ) return 0; pNew->nId = p->nId; - pNew->a = sqlite3DbMallocRawNN(db, p->nId*sizeof(p->a[0]) ); + pNew->a = sqlite3DbMallocRaw(db, p->nId*sizeof(p->a[0]) ); if( pNew->a==0 ){ sqlite3DbFree(db, pNew); return 0; @@ -94205,9 +86861,8 @@ SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ } SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ Select *pNew, *pPrior; - assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); + pNew = sqlite3DbMallocRaw(db, sizeof(*p) ); if( pNew==0 ) return 0; pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); @@ -94253,14 +86908,12 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( Expr *pExpr /* Expression to be appended. Might be NULL */ ){ sqlite3 *db = pParse->db; - assert( db!=0 ); if( pList==0 ){ - pList = sqlite3DbMallocRawNN(db, sizeof(ExprList) ); + pList = sqlite3DbMallocZero(db, sizeof(ExprList) ); if( pList==0 ){ goto no_mem; } - pList->nExpr = 0; - pList->a = sqlite3DbMallocRawNN(db, sizeof(pList->a[0])); + pList->a = sqlite3DbMallocRaw(db, sizeof(pList->a[0])); if( pList->a==0 ) goto no_mem; }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ struct ExprList_item *a; @@ -94286,74 +86939,6 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( return 0; } -/* -** pColumns and pExpr form a vector assignment which is part of the SET -** clause of an UPDATE statement. Like this: -** -** (a,b,c) = (expr1,expr2,expr3) -** Or: (a,b,c) = (SELECT x,y,z FROM ....) -** -** For each term of the vector assignment, append new entries to the -** expression list pList. In the case of a subquery on the LHS, append -** TK_SELECT_COLUMN expressions. -*/ -SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector( - Parse *pParse, /* Parsing context */ - ExprList *pList, /* List to which to append. Might be NULL */ - IdList *pColumns, /* List of names of LHS of the assignment */ - Expr *pExpr /* Vector expression to be appended. Might be NULL */ -){ - sqlite3 *db = pParse->db; - int n; - int i; - int iFirst = pList ? pList->nExpr : 0; - /* pColumns can only be NULL due to an OOM but an OOM will cause an - ** exit prior to this routine being invoked */ - if( NEVER(pColumns==0) ) goto vector_append_error; - if( pExpr==0 ) goto vector_append_error; - n = sqlite3ExprVectorSize(pExpr); - if( pColumns->nId!=n ){ - sqlite3ErrorMsg(pParse, "%d columns assigned %d values", - pColumns->nId, n); - goto vector_append_error; - } - for(i=0; inExpr==iFirst+i+1 ); - pList->a[pList->nExpr-1].zName = pColumns->a[i].zName; - pColumns->a[i].zName = 0; - } - } - if( pExpr->op==TK_SELECT ){ - if( pList && pList->a[iFirst].pExpr ){ - assert( pList->a[iFirst].pExpr->op==TK_SELECT_COLUMN ); - pList->a[iFirst].pExpr->pRight = pExpr; - pExpr = 0; - } - } - -vector_append_error: - sqlite3ExprDelete(db, pExpr); - sqlite3IdListDelete(db, pColumns); - return pList; -} - -/* -** Set the sort order for the last element on the given ExprList. -*/ -SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){ - if( p==0 ) return; - assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC>=0 && SQLITE_SO_DESC>0 ); - assert( p->nExpr>0 ); - if( iSortOrder<0 ){ - assert( p->a[p->nExpr-1].sortOrder==SQLITE_SO_ASC ); - return; - } - p->a[p->nExpr-1].sortOrder = (u8)iSortOrder; -} - /* ** Set the ExprList.a[].zName element of the most recently added item ** on the expression list. @@ -94375,7 +86960,7 @@ SQLITE_PRIVATE void sqlite3ExprListSetName( pItem = &pList->a[pList->nExpr-1]; assert( pItem->zName==0 ); pItem->zName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); - if( dequote ) sqlite3Dequote(pItem->zName); + if( dequote && pItem->zName ) sqlite3Dequote(pItem->zName); } } @@ -94424,9 +87009,10 @@ SQLITE_PRIVATE void sqlite3ExprListCheckLength( /* ** Delete an entire expression list. */ -static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ +SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ int i; struct ExprList_item *pItem; + if( pList==0 ) return; assert( pList->a!=0 || pList->nExpr==0 ); for(pItem=pList->a, i=0; inExpr; i++, pItem++){ sqlite3ExprDelete(db, pItem->pExpr); @@ -94436,9 +87022,6 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ sqlite3DbFree(db, pList->a); sqlite3DbFree(db, pList); } -SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ - if( pList ) exprListDeleteNN(db, pList); -} /* ** Return the bitwise-OR of all Expr.flags fields in the given @@ -94450,8 +87033,7 @@ SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList *pList){ if( pList ){ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - assert( pExpr!=0 ); - m |= pExpr->flags; + if( ALWAYS(pExpr) ) m |= pExpr->flags; } } return m; @@ -94467,7 +87049,7 @@ SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList *pList){ ** ** sqlite3ExprIsConstant() pWalker->eCode==1 ** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 -** sqlite3ExprIsTableConstant() pWalker->eCode==3 +** sqlite3ExprRefOneTableOnly() pWalker->eCode==3 ** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 ** ** In all cases, the callbacks set Walker.eCode=0 and abort if the expression @@ -94575,7 +87157,7 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ } /* -** Walk an expression tree. Return non-zero if the expression is constant +** Walk an expression tree. Return non-zero if the expression constant ** for any single row of the table with cursor iCur. In other words, the ** expression must not refer to any non-deterministic function nor any ** table other than iCur. @@ -94598,22 +87180,6 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ return exprIsConst(p, 4+isInit, 0); } -#ifdef SQLITE_ENABLE_CURSOR_HINTS -/* -** Walk an expression tree. Return 1 if the expression contains a -** subquery of some kind. Return 0 if there are no subqueries. -*/ -SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr *p){ - Walker w; - memset(&w, 0, sizeof(w)); - w.eCode = 1; - w.xExprCallback = sqlite3ExprWalkNoop; - w.xSelectCallback = selectNodeIsConstant; - sqlite3WalkExpr(&w, p); - return w.eCode==0; -} -#endif - /* ** If the expression p codes a constant integer that is small enough ** to fit in a 32-bit integer, return 1 and put the value of the integer @@ -94697,7 +87263,7 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ */ SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ u8 op; - if( aff==SQLITE_AFF_BLOB ) return 1; + if( aff==SQLITE_AFF_NONE ) return 1; while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } op = p->op; if( op==TK_REGISTER ) op = p->op2; @@ -94736,22 +87302,23 @@ SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ } /* -** pX is the RHS of an IN operator. If pX is a SELECT statement -** that can be simplified to a direct table access, then return -** a pointer to the SELECT statement. If pX is not a SELECT statement, -** or if the SELECT statement needs to be manifested into a transient -** table, then return NULL. +** Return true if we are able to the IN operator optimization on a +** query of the form +** +** x IN (SELECT ...) +** +** Where the SELECT... clause is as specified by the parameter to this +** routine. +** +** The Select object passed in has already been preprocessed and no +** errors have been found. */ #ifndef SQLITE_OMIT_SUBQUERY -static Select *isCandidateForInOpt(Expr *pX){ - Select *p; +static int isCandidateForInOpt(Select *p){ SrcList *pSrc; ExprList *pEList; Table *pTab; - int i; - if( !ExprHasProperty(pX, EP_xIsSelect) ) return 0; /* Not a subquery */ - if( ExprHasProperty(pX, EP_VarSelect) ) return 0; /* Correlated subq */ - p = pX->x.pSelect; + if( p==0 ) return 0; /* right-hand side of IN is SELECT */ if( p->pPrior ) return 0; /* Not a compound SELECT */ if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); @@ -94767,22 +87334,25 @@ static Select *isCandidateForInOpt(Expr *pX){ if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ pTab = pSrc->a[0].pTab; - assert( pTab!=0 ); + if( NEVER(pTab==0) ) return 0; assert( pTab->pSelect==0 ); /* FROM clause is not a view */ if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ pEList = p->pEList; - assert( pEList!=0 ); - /* All SELECT results must be columns. */ - for(i=0; inExpr; i++){ - Expr *pRes = pEList->a[i].pExpr; - if( pRes->op!=TK_COLUMN ) return 0; - assert( pRes->iTable==pSrc->a[0].iCursor ); /* Not a correlated subquery */ - } - return p; + if( pEList->nExpr!=1 ) return 0; /* One column in the result set */ + if( pEList->a[0].pExpr->op!=TK_COLUMN ) return 0; /* Result is a column */ + return 1; } #endif /* SQLITE_OMIT_SUBQUERY */ -#ifndef SQLITE_OMIT_SUBQUERY +/* +** Code an OP_Once instruction and allocate space for its flag. Return the +** address of the new instruction. +*/ +SQLITE_PRIVATE int sqlite3CodeOnce(Parse *pParse){ + Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ + return sqlite3VdbeAddOp1(v, OP_Once, pParse->nOnce++); +} + /* ** Generate code that checks the left-most column of index table iCur to see if ** it contains any NULL entries. Cause the register at regHasNull to be set @@ -94790,15 +87360,14 @@ static Select *isCandidateForInOpt(Expr *pX){ ** to be set to NULL if iCur contains one or more NULL values. */ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ - int addr1; + int j1; sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); - addr1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + j1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); VdbeComment((v, "first_entry_in(%d)", iCur)); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeJumpHere(v, j1); } -#endif #ifndef SQLITE_OMIT_SUBQUERY @@ -94843,7 +87412,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** An existing b-tree might be used if the RHS expression pX is a simple ** subquery such as: ** -** SELECT , ... FROM +** SELECT FROM
    ** ** If the RHS of the IN operator is a list or a more complex subquery, then ** an ephemeral table might need to be generated from the RHS and then @@ -94859,14 +87428,14 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** ** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate ** through the set members) then the b-tree must not contain duplicates. -** An epheremal table must be used unless the selected columns are guaranteed -** to be unique - either because it is an INTEGER PRIMARY KEY or due to -** a UNIQUE constraint or index. +** An epheremal table must be used unless the selected is guaranteed +** to be unique - either because it is an INTEGER PRIMARY KEY or it +** has a UNIQUE constraint or UNIQUE index. ** ** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used ** for fast set membership tests) then an epheremal table must -** be used unless is a single INTEGER PRIMARY KEY column or an -** index can be found with the specified as its left-most. +** be used unless is an INTEGER PRIMARY KEY or an index can +** be found with as its left-most column. ** ** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and ** if the RHS of the IN operator is a list (not a subquery) then this @@ -94887,26 +87456,9 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** the value in that register will be NULL if the b-tree contains one or more ** NULL values, and it will be some non-NULL value if the b-tree contains no ** NULL values. -** -** If the aiMap parameter is not NULL, it must point to an array containing -** one element for each column returned by the SELECT statement on the RHS -** of the IN(...) operator. The i'th entry of the array is populated with the -** offset of the index column that matches the i'th column returned by the -** SELECT. For example, if the expression and selected index are: -** -** (?,?,?) IN (SELECT a, b, c FROM t1) -** CREATE INDEX i1 ON t1(b, c, a); -** -** then aiMap[] is populated with {2, 0, 1}. */ #ifndef SQLITE_OMIT_SUBQUERY -SQLITE_PRIVATE int sqlite3FindInIndex( - Parse *pParse, /* Parsing context */ - Expr *pX, /* The right-hand side (RHS) of the IN operator */ - u32 inFlags, /* IN_INDEX_LOOP, _MEMBERSHIP, and/or _NOOP_OK */ - int *prRhsHasNull, /* Register holding NULL status. See notes */ - int *aiMap /* Mapping from Index fields to RHS fields */ -){ +SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ Select *p; /* SELECT to the right of IN operator */ int eType = 0; /* Type of RHS table. IN_INDEX_* */ int iTab = pParse->nTab++; /* Cursor of the RHS table */ @@ -94916,46 +87468,38 @@ SQLITE_PRIVATE int sqlite3FindInIndex( assert( pX->op==TK_IN ); mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; - /* If the RHS of this IN(...) operator is a SELECT, and if it matters - ** whether or not the SELECT result contains NULL values, check whether - ** or not NULL is actually possible (it may not be, for example, due - ** to NOT NULL constraints in the schema). If no NULL values are possible, - ** set prRhsHasNull to 0 before continuing. */ - if( prRhsHasNull && (pX->flags & EP_xIsSelect) ){ - int i; - ExprList *pEList = pX->x.pSelect->pEList; - for(i=0; inExpr; i++){ - if( sqlite3ExprCanBeNull(pEList->a[i].pExpr) ) break; - } - if( i==pEList->nExpr ){ - prRhsHasNull = 0; - } - } - /* Check to see if an existing table or index can be used to ** satisfy the query. This is preferable to generating a new - ** ephemeral table. */ - if( pParse->nErr==0 && (p = isCandidateForInOpt(pX))!=0 ){ + ** ephemeral table. + */ + p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); + if( pParse->nErr==0 && isCandidateForInOpt(p) ){ sqlite3 *db = pParse->db; /* Database connection */ Table *pTab; /* Table
    . */ + Expr *pExpr; /* Expression */ + i16 iCol; /* Index of column */ i16 iDb; /* Database idx for pTab */ - ExprList *pEList = p->pEList; - int nExpr = pEList->nExpr; + assert( p ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ pTab = p->pSrc->a[0].pTab; - + pExpr = p->pEList->a[0].pExpr; + iCol = (i16)pExpr->iColumn; + /* Code an OP_Transaction and OP_TableLock for
    . */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - assert(v); /* sqlite3GetVdbe() has always been previously called */ - if( nExpr==1 && pEList->a[0].pExpr->iColumn<0 ){ - /* The "x IN (SELECT rowid FROM table)" case */ - int iAddr = sqlite3VdbeAddOp0(v, OP_Once); + /* This function is only called from two places. In both cases the vdbe + ** has already been allocated. So assume sqlite3GetVdbe() is always + ** successful here. + */ + assert(v); + if( iCol<0 ){ + int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); @@ -94964,114 +87508,44 @@ SQLITE_PRIVATE int sqlite3FindInIndex( sqlite3VdbeJumpHere(v, iAddr); }else{ Index *pIdx; /* Iterator variable */ - int affinity_ok = 1; - int i; - /* Check that the affinity that will be used to perform each - ** comparison is the same as the affinity of each column in table - ** on the RHS of the IN operator. If it not, it is not possible to - ** use any index of the RHS table. */ - for(i=0; ipLeft, i); - int iCol = pEList->a[i].pExpr->iColumn; - char idxaff = sqlite3TableColumnAffinity(pTab,iCol); /* RHS table */ - char cmpaff = sqlite3CompareAffinity(pLhs, idxaff); - testcase( cmpaff==SQLITE_AFF_BLOB ); - testcase( cmpaff==SQLITE_AFF_TEXT ); - switch( cmpaff ){ - case SQLITE_AFF_BLOB: - break; - case SQLITE_AFF_TEXT: - /* sqlite3CompareAffinity() only returns TEXT if one side or the - ** other has no affinity and the other side is TEXT. Hence, - ** the only way for cmpaff to be TEXT is for idxaff to be TEXT - ** and for the term on the LHS of the IN to have no affinity. */ - assert( idxaff==SQLITE_AFF_TEXT ); - break; - default: - affinity_ok = sqlite3IsNumericAffinity(idxaff); - } - } - - if( affinity_ok ){ - /* Search for an existing index that will work for this IN operator */ - for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){ - Bitmask colUsed; /* Columns of the index used */ - Bitmask mCol; /* Mask for the current column */ - if( pIdx->nColumnnColumn==BMS-2 ); - testcase( pIdx->nColumn==BMS-1 ); - if( pIdx->nColumn>=BMS-1 ) continue; - if( mustBeUnique ){ - if( pIdx->nKeyCol>nExpr - ||(pIdx->nColumn>nExpr && !IsUniqueIndex(pIdx)) - ){ - continue; /* This index is not unique over the IN RHS columns */ - } - } - - colUsed = 0; /* Columns of index used so far */ - for(i=0; ipLeft, i); - Expr *pRhs = pEList->a[i].pExpr; - CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); - int j; - - assert( pReq!=0 || pRhs->iColumn==XN_ROWID || pParse->nErr ); - for(j=0; jaiColumn[j]!=pRhs->iColumn ) continue; - assert( pIdx->azColl[j] ); - if( pReq!=0 && sqlite3StrICmp(pReq->zName, pIdx->azColl[j])!=0 ){ - continue; - } - break; - } - if( j==nExpr ) break; - mCol = MASKBIT(j); - if( mCol & colUsed ) break; /* Each column used only once */ - colUsed |= mCol; - if( aiMap ) aiMap[i] = j; - } - - assert( i==nExpr || colUsed!=(MASKBIT(nExpr)-1) ); - if( colUsed==(MASKBIT(nExpr)-1) ){ - /* If we reach this point, that means the index pIdx is usable */ - int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); -#ifndef SQLITE_OMIT_EXPLAIN - sqlite3VdbeAddOp4(v, OP_Explain, 0, 0, 0, - sqlite3MPrintf(db, "USING INDEX %s FOR IN-OPERATOR",pIdx->zName), - P4_DYNAMIC); -#endif - sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); - sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - VdbeComment((v, "%s", pIdx->zName)); - assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); - eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; - - if( prRhsHasNull ){ -#ifdef SQLITE_ENABLE_COLUMN_USED_MASK - i64 mask = (1<nMem; - if( nExpr==1 ){ - sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); - } - } - sqlite3VdbeJumpHere(v, iAddr); + /* The collation sequence used by the comparison. If an index is to + ** be used in place of a temp-table, it must be ordered according + ** to this collation sequence. */ + CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pExpr); + + /* Check that the affinity that will be used to perform the + ** comparison is the same as the affinity of the column. If + ** it is not, it is not possible to use any index. + */ + int affinity_ok = sqlite3IndexAffinityOk(pX, pTab->aCol[iCol].affinity); + + for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ + if( (pIdx->aiColumn[0]==iCol) + && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq + && (!mustBeUnique || (pIdx->nKeyCol==1 && IsUniqueIndex(pIdx))) + ){ + int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; + + if( prRhsHasNull && !pTab->aCol[iCol].notNull ){ + *prRhsHasNull = ++pParse->nMem; + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); } - } /* End loop over indexes */ - } /* End if( affinity_ok ) */ - } /* End if not an rowid index */ - } /* End attempt to optimize using an index */ + sqlite3VdbeJumpHere(v, iAddr); + } + } + } + } /* If no preexisting index is available for the IN clause ** and IN_INDEX_NOOP is an allowed reply ** and the RHS of the IN operator is a list, not a subquery - ** and the RHS is not constant or has two or fewer terms, + ** and the RHS is not contant or has two or fewer terms, ** then it is not worth creating an ephemeral table to evaluate ** the IN operator so return IN_INDEX_NOOP. */ @@ -95082,6 +87556,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex( ){ eType = IN_INDEX_NOOP; } + if( eType==0 ){ /* Could not find an existing table or index to use as the RHS b-tree. @@ -95103,63 +87578,10 @@ SQLITE_PRIVATE int sqlite3FindInIndex( }else{ pX->iTable = iTab; } - - if( aiMap && eType!=IN_INDEX_INDEX_ASC && eType!=IN_INDEX_INDEX_DESC ){ - int i, n; - n = sqlite3ExprVectorSize(pX->pLeft); - for(i=0; ipLeft; - int nVal = sqlite3ExprVectorSize(pLeft); - Select *pSelect = (pExpr->flags & EP_xIsSelect) ? pExpr->x.pSelect : 0; - char *zRet; - - assert( pExpr->op==TK_IN ); - zRet = sqlite3DbMallocZero(pParse->db, nVal+1); - if( zRet ){ - int i; - for(i=0; ipEList->a[i].pExpr, a); - }else{ - zRet[i] = a; - } - } - zRet[nVal] = '\0'; - } - return zRet; -} -#endif - -#ifndef SQLITE_OMIT_SUBQUERY -/* -** Load the Parse object passed as the first argument with an error -** message of the form: -** -** "sub-select returns N columns - expected M" -*/ -SQLITE_PRIVATE void sqlite3SubselectError(Parse *pParse, int nActual, int nExpect){ - const char *zFmt = "sub-select returns %d columns - expected %d"; - sqlite3ErrorMsg(pParse, zFmt, nActual, nExpect); -} -#endif - /* ** Generate code for scalar subqueries used as a subquery expression, EXISTS, ** or IN operators. Examples: @@ -95185,9 +87607,7 @@ SQLITE_PRIVATE void sqlite3SubselectError(Parse *pParse, int nActual, int nExpec ** value to non-NULL if the RHS is NULL-free. ** ** For a SELECT or EXISTS operator, return the register that holds the -** result. For a multi-column SELECT, the result is stored in a contiguous -** array of registers and the return value is the register of the left-most -** result column. Return 0 for IN operators or if an error occurs. +** result. For IN operators or if an error occurs, the return value is 0. */ #ifndef SQLITE_OMIT_SUBQUERY SQLITE_PRIVATE int sqlite3CodeSubselect( @@ -95202,8 +87622,8 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( if( NEVER(v==0) ) return 0; sqlite3ExprCachePush(pParse); - /* The evaluation of the IN/EXISTS/SELECT must be repeated every time it - ** is encountered if any of the following is true: + /* This code must be run in its entirety every time it is encountered + ** if any of the following is true: ** ** * The right-hand side is a correlated subquery ** * The right-hand side is an expression list containing variables @@ -95213,15 +87633,14 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** save the results, and reuse the same result on subsequent invocations. */ if( !ExprHasProperty(pExpr, EP_VarSelect) ){ - jmpIfDynamic = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + jmpIfDynamic = sqlite3CodeOnce(pParse); VdbeCoverage(v); } #ifndef SQLITE_OMIT_EXPLAIN if( pParse->explain==2 ){ - char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %s%s SUBQUERY %d", - jmpIfDynamic>=0?"":"CORRELATED ", - pExpr->op==TK_IN?"LIST":"SCALAR", - pParse->iNextSelectId + char *zMsg = sqlite3MPrintf( + pParse->db, "EXECUTE %s%s SUBQUERY %d", jmpIfDynamic>=0?"":"CORRELATED ", + pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId ); sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); } @@ -95229,18 +87648,17 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( switch( pExpr->op ){ case TK_IN: { + char affinity; /* Affinity of the LHS of the IN */ int addr; /* Address of OP_OpenEphemeral instruction */ Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ KeyInfo *pKeyInfo = 0; /* Key information */ - int nVal; /* Size of vector pLeft */ - - nVal = sqlite3ExprVectorSize(pLeft); - assert( !isRowid || nVal==1 ); + + affinity = sqlite3ExprAffinity(pLeft); /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' ** expression it is handled the same way. An ephemeral table is - ** filled with index keys representing the results from the - ** SELECT or the . + ** filled with single-field index keys representing the results + ** from the SELECT or the . ** ** If the 'x' expression is a column value, or the SELECT... ** statement returns a column value, then the affinity of that @@ -95251,9 +87669,8 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** is used. */ pExpr->iTable = pParse->nTab++; - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, - pExpr->iTable, (isRowid?0:nVal)); - pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, nVal, 1); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); + pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, 1, 1); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* Case 1: expr IN (SELECT ...) @@ -95262,37 +87679,27 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** table allocated and opened above. */ Select *pSelect = pExpr->x.pSelect; - ExprList *pEList = pSelect->pEList; + SelectDest dest; + ExprList *pEList; assert( !isRowid ); - /* If the LHS and RHS of the IN operator do not match, that - ** error will have been caught long before we reach this point. */ - if( ALWAYS(pEList->nExpr==nVal) ){ - SelectDest dest; - int i; - sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); - dest.zAffSdst = exprINAffinity(pParse, pExpr); - assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); - pSelect->iLimit = 0; - testcase( pSelect->selFlags & SF_Distinct ); - testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ - if( sqlite3Select(pParse, pSelect, &dest) ){ - sqlite3DbFree(pParse->db, dest.zAffSdst); - sqlite3KeyInfoUnref(pKeyInfo); - return 0; - } - sqlite3DbFree(pParse->db, dest.zAffSdst); - assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ - assert( pEList!=0 ); - assert( pEList->nExpr>0 ); - assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); - for(i=0; iaColl[i] = sqlite3BinaryCompareCollSeq( - pParse, p, pEList->a[i].pExpr - ); - } + sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); + dest.affSdst = (u8)affinity; + assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); + pSelect->iLimit = 0; + testcase( pSelect->selFlags & SF_Distinct ); + testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ + if( sqlite3Select(pParse, pSelect, &dest) ){ + sqlite3KeyInfoUnref(pKeyInfo); + return 0; } + pEList = pSelect->pEList; + assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ + assert( pEList!=0 ); + assert( pEList->nExpr>0 ); + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + pKeyInfo->aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, + pEList->a[0].pExpr); }else if( ALWAYS(pExpr->x.pList!=0) ){ /* Case 2: expr IN (exprlist) ** @@ -95301,15 +87708,13 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** that columns affinity when building index keys. If is not ** a column, use numeric affinity. */ - char affinity; /* Affinity of the LHS of the IN */ int i; ExprList *pList = pExpr->x.pList; struct ExprList_item *pItem; int r1, r2, r3; - affinity = sqlite3ExprAffinity(pLeft); if( !affinity ){ - affinity = SQLITE_AFF_BLOB; + affinity = SQLITE_AFF_NONE; } if( pKeyInfo ){ assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); @@ -95363,37 +87768,26 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( case TK_EXISTS: case TK_SELECT: default: { - /* Case 3: (SELECT ... FROM ...) - ** or: EXISTS(SELECT ... FROM ...) - ** - ** For a SELECT, generate code to put the values for all columns of - ** the first row into an array of registers and return the index of - ** the first register. - ** - ** If this is an EXISTS, write an integer 0 (not exists) or 1 (exists) - ** into a register and return that register number. - ** - ** In both cases, the query is augmented with "LIMIT 1". Any - ** preexisting limit is discarded in place of the new LIMIT 1. + /* If this has to be a scalar SELECT. Generate code to put the + ** value of this select in a memory cell and record the number + ** of the memory cell in iColumn. If this is an EXISTS, write + ** an integer 0 (not exists) or 1 (exists) into a memory cell + ** and record that memory cell in iColumn. */ Select *pSel; /* SELECT statement to encode */ - SelectDest dest; /* How to deal with SELECT result */ - int nReg; /* Registers to allocate */ + SelectDest dest; /* How to deal with SELECt result */ testcase( pExpr->op==TK_EXISTS ); testcase( pExpr->op==TK_SELECT ); assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); - assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + assert( ExprHasProperty(pExpr, EP_xIsSelect) ); pSel = pExpr->x.pSelect; - nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; - sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); - pParse->nMem += nReg; + sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; dest.iSdst = dest.iSDParm; - dest.nSdst = nReg; - sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); + sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iSDParm); VdbeComment((v, "Init subquery result")); }else{ dest.eDest = SRT_Exists; @@ -95401,8 +87795,8 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( VdbeComment((v, "Init EXISTS result")); } sqlite3ExprDelete(pParse->db, pSel->pLimit); - pSel->pLimit = sqlite3ExprAlloc(pParse->db, TK_INTEGER, - &sqlite3IntTokens[1], 0); + pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, + &sqlite3IntTokens[1]); pSel->iLimit = 0; pSel->selFlags &= ~SF_MultiValue; if( sqlite3Select(pParse, pSel, &dest) ){ @@ -95427,32 +87821,6 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( } #endif /* SQLITE_OMIT_SUBQUERY */ -#ifndef SQLITE_OMIT_SUBQUERY -/* -** Expr pIn is an IN(...) expression. This function checks that the -** sub-select on the RHS of the IN() operator has the same number of -** columns as the vector on the LHS. Or, if the RHS of the IN() is not -** a sub-query, that the LHS is a vector of size 1. -*/ -SQLITE_PRIVATE int sqlite3ExprCheckIN(Parse *pParse, Expr *pIn){ - int nVector = sqlite3ExprVectorSize(pIn->pLeft); - if( (pIn->flags & EP_xIsSelect) ){ - if( nVector!=pIn->x.pSelect->pEList->nExpr ){ - sqlite3SubselectError(pParse, pIn->x.pSelect->pEList->nExpr, nVector); - return 1; - } - }else if( nVector!=1 ){ - if( (pIn->pLeft->flags & EP_xIsSelect) ){ - sqlite3SubselectError(pParse, nVector, 1); - }else{ - sqlite3ErrorMsg(pParse, "row value misused"); - } - return 1; - } - return 0; -} -#endif - #ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code for an IN expression. @@ -95460,24 +87828,16 @@ SQLITE_PRIVATE int sqlite3ExprCheckIN(Parse *pParse, Expr *pIn){ ** x IN (SELECT ...) ** x IN (value, value, ...) ** -** The left-hand side (LHS) is a scalar or vector expression. The -** right-hand side (RHS) is an array of zero or more scalar values, or a -** subquery. If the RHS is a subquery, the number of result columns must -** match the number of columns in the vector on the LHS. If the RHS is -** a list of values, the LHS must be a scalar. -** -** The IN operator is true if the LHS value is contained within the RHS. -** The result is false if the LHS is definitely not in the RHS. The -** result is NULL if the presence of the LHS in the RHS cannot be -** determined due to NULLs. +** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS) +** is an array of zero or more values. The expression is true if the LHS is +** contained within the RHS. The value of the expression is unknown (NULL) +** if the LHS is NULL or if the LHS is not contained within the RHS and the +** RHS contains one or more NULL values. ** ** This routine generates code that jumps to destIfFalse if the LHS is not ** contained within the RHS. If due to NULLs we cannot determine if the LHS ** is contained in the RHS then jump to destIfNull. If the LHS is contained ** within the RHS then fall through. -** -** See the separate in-operator.md documentation file in the canonical -** SQLite source tree for additional information. */ static void sqlite3ExprCodeIN( Parse *pParse, /* Parsing and code generating context */ @@ -95486,83 +87846,36 @@ static void sqlite3ExprCodeIN( int destIfNull /* Jump here if the results are unknown due to NULLs */ ){ int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ + char affinity; /* Comparison affinity to use */ int eType; /* Type of the RHS */ - int rLhs; /* Register(s) holding the LHS values */ - int rLhsOrig; /* LHS values prior to reordering by aiMap[] */ + int r1; /* Temporary use register */ Vdbe *v; /* Statement under construction */ - int *aiMap = 0; /* Map from vector field to index column */ - char *zAff = 0; /* Affinity string for comparisons */ - int nVector; /* Size of vectors for this IN operator */ - int iDummy; /* Dummy parameter to exprCodeVector() */ - Expr *pLeft; /* The LHS of the IN operator */ - int i; /* loop counter */ - int destStep2; /* Where to jump when NULLs seen in step 2 */ - int destStep6 = 0; /* Start of code for Step 6 */ - int addrTruthOp; /* Address of opcode that determines the IN is true */ - int destNotNull; /* Jump here if a comparison is not true in step 6 */ - int addrTop; /* Top of the step-6 loop */ - - pLeft = pExpr->pLeft; - if( sqlite3ExprCheckIN(pParse, pExpr) ) return; - zAff = exprINAffinity(pParse, pExpr); - nVector = sqlite3ExprVectorSize(pExpr->pLeft); - aiMap = (int*)sqlite3DbMallocZero( - pParse->db, nVector*(sizeof(int) + sizeof(char)) + 1 - ); - if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; - /* Attempt to compute the RHS. After this step, if anything other than - ** IN_INDEX_NOOP is returned, the table opened ith cursor pExpr->iTable - ** contains the values that make up the RHS. If IN_INDEX_NOOP is returned, - ** the RHS has not yet been coded. */ + /* Compute the RHS. After this step, the table with cursor + ** pExpr->iTable will contains the values that make up the RHS. + */ v = pParse->pVdbe; assert( v!=0 ); /* OOM detected prior to this routine */ VdbeNoopComment((v, "begin IN expr")); eType = sqlite3FindInIndex(pParse, pExpr, IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, - destIfFalse==destIfNull ? 0 : &rRhsHasNull, aiMap); + destIfFalse==destIfNull ? 0 : &rRhsHasNull); - assert( pParse->nErr || nVector==1 || eType==IN_INDEX_EPH - || eType==IN_INDEX_INDEX_ASC || eType==IN_INDEX_INDEX_DESC - ); -#ifdef SQLITE_DEBUG - /* Confirm that aiMap[] contains nVector integer values between 0 and - ** nVector-1. */ - for(i=0; i from " IN (...)". If the LHS is a - ** vector, then it is stored in an array of nVector registers starting - ** at r1. - ** - ** sqlite3FindInIndex() might have reordered the fields of the LHS vector - ** so that the fields are in the same order as an existing index. The - ** aiMap[] array contains a mapping from the original LHS field order to - ** the field order that matches the RHS index. + /* Code the LHS, the from " IN (...)". */ sqlite3ExprCachePush(pParse); - rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); - for(i=0; ipLeft, r1); /* If sqlite3FindInIndex() did not find or create an index that is ** suitable for evaluating the IN operator, then evaluate using a ** sequence of comparisons. - ** - ** This is step (1) in the in-operator.md optimized algorithm. */ if( eType==IN_INDEX_NOOP ){ ExprList *pList = pExpr->x.pList; @@ -95574,7 +87887,7 @@ static void sqlite3ExprCodeIN( assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); if( destIfNull!=destIfFalse ){ regCkNull = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_BitAnd, rLhs, rLhs, regCkNull); + sqlite3VdbeAddOp3(v, OP_BitAnd, r1, r1, regCkNull); } for(ii=0; iinExpr; ii++){ r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); @@ -95582,135 +87895,111 @@ static void sqlite3ExprCodeIN( sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); } if( iinExpr-1 || destIfNull!=destIfFalse ){ - sqlite3VdbeAddOp4(v, OP_Eq, rLhs, labelOk, r2, + sqlite3VdbeAddOp4(v, OP_Eq, r1, labelOk, r2, (void*)pColl, P4_COLLSEQ); VdbeCoverageIf(v, iinExpr-1); VdbeCoverageIf(v, ii==pList->nExpr-1); - sqlite3VdbeChangeP5(v, zAff[0]); + sqlite3VdbeChangeP5(v, affinity); }else{ assert( destIfNull==destIfFalse ); - sqlite3VdbeAddOp4(v, OP_Ne, rLhs, destIfFalse, r2, + sqlite3VdbeAddOp4(v, OP_Ne, r1, destIfFalse, r2, (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, zAff[0] | SQLITE_JUMPIFNULL); + sqlite3VdbeChangeP5(v, affinity | SQLITE_JUMPIFNULL); } sqlite3ReleaseTempReg(pParse, regToFree); } if( regCkNull ){ sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); - sqlite3VdbeGoto(v, destIfFalse); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); } sqlite3VdbeResolveLabel(v, labelOk); sqlite3ReleaseTempReg(pParse, regCkNull); - goto sqlite3ExprCodeIN_finished; - } - - /* Step 2: Check to see if the LHS contains any NULL columns. If the - ** LHS does contain NULLs then the result must be either FALSE or NULL. - ** We will then skip the binary search of the RHS. - */ - if( destIfNull==destIfFalse ){ - destStep2 = destIfFalse; }else{ - destStep2 = destStep6 = sqlite3VdbeMakeLabel(v); - } - for(i=0; ipLeft, i); - if( sqlite3ExprCanBeNull(p) ){ - sqlite3VdbeAddOp2(v, OP_IsNull, rLhs+i, destStep2); - VdbeCoverage(v); + + /* If the LHS is NULL, then the result is either false or NULL depending + ** on whether the RHS is empty or not, respectively. + */ + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + if( destIfNull==destIfFalse ){ + /* Shortcut for the common case where the false and NULL outcomes are + ** the same. */ + sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); + }else{ + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + sqlite3VdbeJumpHere(v, addr1); + } } - } - - /* Step 3. The LHS is now known to be non-NULL. Do the binary search - ** of the RHS using the LHS as a probe. If found, the result is - ** true. - */ - if( eType==IN_INDEX_ROWID ){ - /* In this case, the RHS is the ROWID of table b-tree and so we also - ** know that the RHS is non-NULL. Hence, we combine steps 3 and 4 - ** into a single opcode. */ - sqlite3VdbeAddOp3(v, OP_SeekRowid, pExpr->iTable, destIfFalse, rLhs); - VdbeCoverage(v); - addrTruthOp = sqlite3VdbeAddOp0(v, OP_Goto); /* Return True */ - }else{ - sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); - if( destIfFalse==destIfNull ){ - /* Combine Step 3 and Step 5 into a single opcode */ - sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, - rLhs, nVector); VdbeCoverage(v); - goto sqlite3ExprCodeIN_finished; + + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree + */ + sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); + VdbeCoverage(v); + }else{ + /* In this case, the RHS is an index b-tree. + */ + sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); + + /* If the set membership test fails, then the result of the + ** "x IN (...)" expression must be either 0 or NULL. If the set + ** contains no NULL values, then the result is 0. If the set + ** contains one or more NULL values, then the result of the + ** expression is also NULL. + */ + assert( destIfFalse!=destIfNull || rRhsHasNull==0 ); + if( rRhsHasNull==0 ){ + /* This branch runs if it is known at compile time that the RHS + ** cannot contain NULL values. This happens as the result + ** of a "NOT NULL" constraint in the database schema. + ** + ** Also run this branch if NULL is equivalent to FALSE + ** for this particular IN operator. + */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); + VdbeCoverage(v); + }else{ + /* In this branch, the RHS of the IN might contain a NULL and + ** the presence of a NULL on the RHS makes a difference in the + ** outcome. + */ + int j1; + + /* First check to see if the LHS is contained in the RHS. If so, + ** then the answer is TRUE the presence of NULLs in the RHS does + ** not matter. If the LHS is not contained in the RHS, then the + ** answer is NULL if the RHS contains NULLs and the answer is + ** FALSE if the RHS is NULL-free. + */ + j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + sqlite3VdbeJumpHere(v, j1); + } } - /* Ordinary Step 3, for the case where FALSE and NULL are distinct */ - addrTruthOp = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, - rLhs, nVector); VdbeCoverage(v); - } - - /* Step 4. If the RHS is known to be non-NULL and we did not find - ** an match on the search above, then the result must be FALSE. - */ - if( rRhsHasNull && nVector==1 ){ - sqlite3VdbeAddOp2(v, OP_NotNull, rRhsHasNull, destIfFalse); - VdbeCoverage(v); - } - - /* Step 5. If we do not care about the difference between NULL and - ** FALSE, then just return false. - */ - if( destIfFalse==destIfNull ) sqlite3VdbeGoto(v, destIfFalse); - - /* Step 6: Loop through rows of the RHS. Compare each row to the LHS. - ** If any comparison is NULL, then the result is NULL. If all - ** comparisons are FALSE then the final result is FALSE. - ** - ** For a scalar LHS, it is sufficient to check just the first row - ** of the RHS. - */ - if( destStep6 ) sqlite3VdbeResolveLabel(v, destStep6); - addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); - VdbeCoverage(v); - if( nVector>1 ){ - destNotNull = sqlite3VdbeMakeLabel(v); - }else{ - /* For nVector==1, combine steps 6 and 7 by immediately returning - ** FALSE if the first comparison is not NULL */ - destNotNull = destIfFalse; - } - for(i=0; iiTable, i, r3); - sqlite3VdbeAddOp4(v, OP_Ne, rLhs+i, destNotNull, r3, - (void*)pColl, P4_COLLSEQ); - VdbeCoverage(v); - sqlite3ReleaseTempReg(pParse, r3); - } - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); - if( nVector>1 ){ - sqlite3VdbeResolveLabel(v, destNotNull); - sqlite3VdbeAddOp2(v, OP_Next, pExpr->iTable, addrTop+1); - VdbeCoverage(v); - - /* Step 7: If we reach this point, we know that the result must - ** be false. */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); } - - /* Jumps here in order to return true. */ - sqlite3VdbeJumpHere(v, addrTruthOp); - -sqlite3ExprCodeIN_finished: - if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); + sqlite3ReleaseTempReg(pParse, r1); sqlite3ExprCachePop(pParse); VdbeComment((v, "end IN expr")); -sqlite3ExprCodeIN_oom_error: - sqlite3DbFree(pParse->db, aiMap); - sqlite3DbFree(pParse->db, zAff); } #endif /* SQLITE_OMIT_SUBQUERY */ +/* +** Duplicate an 8-byte value +*/ +static char *dup8bytes(Vdbe *v, const char *in){ + char *out = sqlite3DbMallocRaw(sqlite3VdbeDb(v), 8); + if( out ){ + memcpy(out, in, 8); + } + return out; +} + #ifndef SQLITE_OMIT_FLOATING_POINT /* ** Generate an instruction that will put the floating point @@ -95723,10 +88012,12 @@ static void sqlite3ExprCodeIN( static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ if( ALWAYS(z!=0) ){ double value; + char *zV; sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; - sqlite3VdbeAddOp4Dup8(v, OP_Real, 0, iMem, 0, (u8*)&value, P4_REAL); + zV = dup8bytes(v, (char*)&value); + sqlite3VdbeAddOp4(v, OP_Real, 0, iMem, 0, zV, P4_REAL); } } #endif @@ -95752,8 +88043,10 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ assert( z!=0 ); c = sqlite3DecOrHexToI64(z, &value); if( c==0 || (c==2 && negFlag) ){ + char *zV; if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } - sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64); + zV = dup8bytes(v, (char*)&value); + sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); }else{ #ifdef SQLITE_OMIT_FLOATING_POINT sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); @@ -95772,17 +88065,14 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ } /* -** Erase column-cache entry number i +** Clear a cache entry. */ -static void cacheEntryClear(Parse *pParse, int i){ - if( pParse->aColCache[i].tempReg ){ +static void cacheEntryClear(Parse *pParse, struct yColCache *p){ + if( p->tempReg ){ if( pParse->nTempRegaTempReg) ){ - pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; + pParse->aTempReg[pParse->nTempReg++] = p->iReg; } - } - pParse->nColCache--; - if( inColCache ){ - pParse->aColCache[i] = pParse->aColCache[pParse->nColCache]; + p->tempReg = 0; } } @@ -95813,33 +88103,43 @@ SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int ** that the object will never already be in cache. Verify this guarantee. */ #ifndef NDEBUG - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - assert( p->iTable!=iTab || p->iColumn!=iCol ); + for(i=0, p=pParse->aColCache; iiReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); } #endif - /* If the cache is already full, delete the least recently used entry */ - if( pParse->nColCache>=SQLITE_N_COLCACHE ){ - minLru = 0x7fffffff; - idxLru = -1; - for(i=0, p=pParse->aColCache; ilrulru; - } + /* Find an empty slot and replace it */ + for(i=0, p=pParse->aColCache; iiReg==0 ){ + p->iLevel = pParse->iCacheLevel; + p->iTable = iTab; + p->iColumn = iCol; + p->iReg = iReg; + p->tempReg = 0; + p->lru = pParse->iCacheCnt++; + return; } - p = &pParse->aColCache[idxLru]; - }else{ - p = &pParse->aColCache[pParse->nColCache++]; } - /* Add the new entry to the end of the cache */ - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; + /* Replace the last recently used */ + minLru = 0x7fffffff; + idxLru = -1; + for(i=0, p=pParse->aColCache; ilrulru; + } + } + if( ALWAYS(idxLru>=0) ){ + p = &pParse->aColCache[idxLru]; + p->iLevel = pParse->iCacheLevel; + p->iTable = iTab; + p->iColumn = iCol; + p->iReg = iReg; + p->tempReg = 0; + p->lru = pParse->iCacheCnt++; + return; + } } /* @@ -95847,13 +88147,14 @@ SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int ** Purge the range of registers from the column cache. */ SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ - int i = 0; - while( inColCache ){ - struct yColCache *p = &pParse->aColCache[i]; - if( p->iReg >= iReg && p->iReg < iReg+nReg ){ - cacheEntryClear(pParse, i); - }else{ - i++; + int i; + int iLast = iReg + nReg - 1; + struct yColCache *p; + for(i=0, p=pParse->aColCache; iiReg; + if( r>=iReg && r<=iLast ){ + cacheEntryClear(pParse, p); + p->iReg = 0; } } } @@ -95878,7 +88179,8 @@ SQLITE_PRIVATE void sqlite3ExprCachePush(Parse *pParse){ ** the cache to the state it was in prior the most recent Push. */ SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ - int i = 0; + int i; + struct yColCache *p; assert( pParse->iCacheLevel>=1 ); pParse->iCacheLevel--; #ifdef SQLITE_DEBUG @@ -95886,11 +88188,10 @@ SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ printf("POP to %d\n", pParse->iCacheLevel); } #endif - while( inColCache ){ - if( pParse->aColCache[i].iLevel>pParse->iCacheLevel ){ - cacheEntryClear(pParse, i); - }else{ - i++; + for(i=0, p=pParse->aColCache; iiReg && p->iLevel>pParse->iCacheLevel ){ + cacheEntryClear(pParse, p); + p->iReg = 0; } } } @@ -95904,35 +88205,13 @@ SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ + for(i=0, p=pParse->aColCache; iiReg==iReg ){ p->tempReg = 0; } } } -/* Generate code that will load into register regOut a value that is -** appropriate for the iIdxCol-th column of index pIdx. -*/ -SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn( - Parse *pParse, /* The parsing context */ - Index *pIdx, /* The index whose column is to be loaded */ - int iTabCur, /* Cursor pointing to a table row */ - int iIdxCol, /* The column of the index to be loaded */ - int regOut /* Store the index column value in this register */ -){ - i16 iTabCol = pIdx->aiColumn[iIdxCol]; - if( iTabCol==XN_EXPR ){ - assert( pIdx->aColExpr ); - assert( pIdx->aColExpr->nExpr>iIdxCol ); - pParse->iSelfTab = iTabCur; - sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); - }else{ - sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, - iTabCol, regOut); - } -} - /* ** Generate code to extract the value of the iCol-th column of a table. */ @@ -95948,7 +88227,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( }else{ int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; int x = iCol; - if( !HasRowid(pTab) && !IsVirtual(pTab) ){ + if( !HasRowid(pTab) ){ x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol); } sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); @@ -95960,12 +88239,9 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( /* ** Generate code that will extract the iColumn-th column from -** table pTab and store the column value in a register. -** -** An effort is made to store the column value in register iReg. This -** is not garanteeed for GetColumn() - the result can be stored in -** any register. But the result is guaranteed to land in register iReg -** for GetColumnToReg(). +** table pTab and store the column value in a register. An effort +** is made to store the column value in register iReg, but this is +** not guaranteed. The location of the column value is returned. ** ** There must be an open cursor to pTab in iTable when this routine ** is called. If iColumn<0 then code is generated that extracts the rowid. @@ -95976,14 +88252,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ int iReg, /* Store results here */ - u8 p5 /* P5 value for OP_Column + FLAGS */ + u8 p5 /* P5 value for OP_Column */ ){ Vdbe *v = pParse->pVdbe; int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - if( p->iTable==iTable && p->iColumn==iColumn ){ + for(i=0, p=pParse->aColCache; iiReg>0 && p->iTable==iTable && p->iColumn==iColumn ){ p->lru = pParse->iCacheCnt++; sqlite3ExprCachePinRegister(pParse, p->iReg); return p->iReg; @@ -95998,37 +88274,25 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( } return iReg; } -SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg( - Parse *pParse, /* Parsing and code generating context */ - Table *pTab, /* Description of the table we are reading from */ - int iColumn, /* Index of the table column */ - int iTable, /* The cursor pointing to the table */ - int iReg /* Store results here */ -){ - int r1 = sqlite3ExprCodeGetColumn(pParse, pTab, iColumn, iTable, iReg, 0); - if( r1!=iReg ) sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, r1, iReg); -} - /* ** Clear all column cache entries. */ SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse *pParse){ int i; + struct yColCache *p; #if SQLITE_DEBUG if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ printf("CLEAR\n"); } #endif - for(i=0; inColCache; i++){ - if( pParse->aColCache[i].tempReg - && pParse->nTempRegaTempReg) - ){ - pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; + for(i=0, p=pParse->aColCache; iiReg ){ + cacheEntryClear(pParse, p); + p->iReg = 0; } } - pParse->nColCache = 0; } /* @@ -96060,7 +88324,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int n static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ + for(i=0, p=pParse->aColCache; iiReg; if( r>=iFrom && r<=iTo ) return 1; /*NO_TEST*/ } @@ -96068,11 +88332,8 @@ static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ } #endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ - /* -** Convert a scalar expression node to a TK_REGISTER referencing -** register iReg. The caller must ensure that iReg already contains -** the correct value for the expression. +** Convert an expression node to a TK_REGISTER */ static void exprToRegister(Expr *p, int iReg){ p->op2 = p->op; @@ -96081,38 +88342,6 @@ static void exprToRegister(Expr *p, int iReg){ ExprClearProperty(p, EP_Skip); } -/* -** Evaluate an expression (either a vector or a scalar expression) and store -** the result in continguous temporary registers. Return the index of -** the first register used to store the result. -** -** If the returned result register is a temporary scalar, then also write -** that register number into *piFreeable. If the returned result register -** is not a temporary or if the expression is a vector set *piFreeable -** to 0. -*/ -static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ - int iResult; - int nResult = sqlite3ExprVectorSize(p); - if( nResult==1 ){ - iResult = sqlite3ExprCodeTemp(pParse, p, piFreeable); - }else{ - *piFreeable = 0; - if( p->op==TK_SELECT ){ - iResult = sqlite3CodeSubselect(pParse, p, 0, 0); - }else{ - int i; - iResult = pParse->nMem+1; - pParse->nMem += nResult; - for(i=0; ix.pList->a[i].pExpr, i+iResult); - } - } - } - return iResult; -} - - /* ** Generate code into the current Vdbe to evaluate the given ** expression. Attempt to store the results in register "target". @@ -96130,9 +88359,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) int inReg = target; /* Results stored in register inReg */ int regFree1 = 0; /* If non-zero free this temporary register */ int regFree2 = 0; /* If non-zero free this temporary register */ - int r1, r2; /* Various register numbers */ + int r1, r2, r3, r4; /* Various register numbers */ + sqlite3 *db = pParse->db; /* The database connection */ Expr tempX; /* Temporary expression node */ - int p5 = 0; assert( target>0 && target<=pParse->nMem ); if( v==0 ){ @@ -96151,11 +88380,12 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) struct AggInfo_col *pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ assert( pCol->iMem>0 ); - return pCol->iMem; + inReg = pCol->iMem; + break; }else if( pAggInfo->useSortingIdx ){ sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); - return target; + break; } /* Otherwise, fall thru into the TK_COLUMN case */ } @@ -96164,36 +88394,37 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( iTab<0 ){ if( pParse->ckBase>0 ){ /* Generating CHECK constraints or inserting into partial index */ - return pExpr->iColumn + pParse->ckBase; + inReg = pExpr->iColumn + pParse->ckBase; + break; }else{ - /* Coding an expression that is part of an index where column names - ** in the index refer to the table to which the index belongs */ - iTab = pParse->iSelfTab; + /* Deleting from a partial index */ + iTab = pParse->iPartIdxTab; } } - return sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, + inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, pExpr->iColumn, iTab, target, pExpr->op2); + break; } case TK_INTEGER: { codeInteger(pParse, pExpr, 0, target); - return target; + break; } #ifndef SQLITE_OMIT_FLOATING_POINT case TK_FLOAT: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pExpr->u.zToken, 0, target); - return target; + break; } #endif case TK_STRING: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); - sqlite3VdbeLoadString(v, target, pExpr->u.zToken); - return target; + sqlite3VdbeAddOp4(v, OP_String8, 0, target, 0, pExpr->u.zToken, 0); + break; } case TK_NULL: { sqlite3VdbeAddOp2(v, OP_Null, 0, target); - return target; + break; } #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { @@ -96208,7 +88439,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( z[n]=='\'' ); zBlob = sqlite3HexToBlob(sqlite3VdbeDb(v), z, n); sqlite3VdbeAddOp4(v, OP_Blob, n/2, target, 0, zBlob, P4_DYNAMIC); - return target; + break; } #endif case TK_VARIABLE: { @@ -96221,10 +88452,15 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) || strcmp(pExpr->u.zToken, pParse->azVar[pExpr->iColumn-1])==0 ); sqlite3VdbeChangeP4(v, -1, pParse->azVar[pExpr->iColumn-1], P4_STATIC); } - return target; + break; } case TK_REGISTER: { - return pExpr->iTable; + inReg = pExpr->iTable; + break; + } + case TK_AS: { + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + break; } #ifndef SQLITE_OMIT_CAST case TK_CAST: { @@ -96238,37 +88474,42 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3AffinityType(pExpr->u.zToken, 0)); testcase( usedAsColumnCache(pParse, inReg, inReg) ); sqlite3ExprCacheAffinityChange(pParse, inReg, 1); - return inReg; + break; } #endif /* SQLITE_OMIT_CAST */ - case TK_IS: - case TK_ISNOT: - op = (op==TK_IS) ? TK_EQ : TK_NE; - p5 = SQLITE_NULLEQ; - /* fall-through */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { - Expr *pLeft = pExpr->pLeft; - if( sqlite3ExprIsVector(pLeft) ){ - codeVectorCompare(pParse, pExpr, target, op, p5); - }else{ - r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - codeCompare(pParse, pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2 | p5); - assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); - assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); - assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); - assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - } + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, inReg, SQLITE_STOREP2); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_IS: + case TK_ISNOT: { + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + op = (op==TK_IS) ? TK_EQ : TK_NE; + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, inReg, SQLITE_STOREP2 | SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); + testcase( regFree1==0 ); + testcase( regFree2==0 ); break; } case TK_AND: @@ -96306,12 +88547,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( pLeft ); if( pLeft->op==TK_INTEGER ){ codeInteger(pParse, pLeft, 1, target); - return target; #ifndef SQLITE_OMIT_FLOATING_POINT }else if( pLeft->op==TK_FLOAT ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pLeft->u.zToken, 1, target); - return target; #endif }else{ tempX.op = TK_INTEGER; @@ -96322,6 +88561,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); testcase( regFree2==0 ); } + inReg = target; break; } case TK_BITNOT: @@ -96330,6 +88570,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( TK_NOT==OP_Not ); testcase( op==TK_NOT ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); + inReg = target; sqlite3VdbeAddOp2(v, op, r1, inReg); break; } @@ -96354,7 +88595,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); }else{ - return pInfo->aFunc[pExpr->iAgg].iMem; + inReg = pInfo->aFunc[pExpr->iAgg].iMem; } break; } @@ -96362,10 +88603,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ExprList *pFarg; /* List of function arguments */ int nFarg; /* Number of function arguments */ FuncDef *pDef; /* The function definition object */ + int nId; /* Length of the function name in bytes */ const char *zId; /* The function name */ u32 constMask = 0; /* Mask of function arguments that are constant */ int i; /* Loop counter */ - sqlite3 *db = pParse->db; /* The database connection */ u8 enc = ENC(db); /* The text encoding used by this database */ CollSeq *pColl = 0; /* A collating sequence */ @@ -96378,14 +88619,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) nFarg = pFarg ? pFarg->nExpr : 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); zId = pExpr->u.zToken; - pDef = sqlite3FindFunction(db, zId, nFarg, enc, 0); -#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION - if( pDef==0 && pParse->explain ){ - pDef = sqlite3FindFunction(db, "unknown", nFarg, enc, 0); - } -#endif - if( pDef==0 || pDef->xFinalize!=0 ){ - sqlite3ErrorMsg(pParse, "unknown function: %s()", zId); + nId = sqlite3Strlen30(zId); + pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0); + if( pDef==0 || pDef->xFunc==0 ){ + sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); break; } @@ -96414,7 +88651,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) */ if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ assert( nFarg>=1 ); - return sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); + sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + break; } for(i=0; ipDfltColl; sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_Function0, constMask, r1, target, + sqlite3VdbeAddOp4(v, OP_Function, constMask, r1, target, (char*)pDef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nFarg); if( nFarg && constMask==0 ){ sqlite3ReleaseTempRange(pParse, r1, nFarg); } - return target; + break; } #ifndef SQLITE_OMIT_SUBQUERY case TK_EXISTS: case TK_SELECT: { - int nCol; testcase( op==TK_EXISTS ); testcase( op==TK_SELECT ); - if( op==TK_SELECT && (nCol = pExpr->x.pSelect->pEList->nExpr)!=1 ){ - sqlite3SubselectError(pParse, nCol, 1); - }else{ - return sqlite3CodeSubselect(pParse, pExpr, 0, 0); - } + inReg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); break; } - case TK_SELECT_COLUMN: { - if( pExpr->pLeft->iTable==0 ){ - pExpr->pLeft->iTable = sqlite3CodeSubselect(pParse, pExpr->pLeft, 0, 0); - } - return pExpr->pLeft->iTable + pExpr->iColumn; - } case TK_IN: { int destIfFalse = sqlite3VdbeMakeLabel(v); int destIfNull = sqlite3VdbeMakeLabel(v); @@ -96519,7 +88746,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3VdbeResolveLabel(v, destIfFalse); sqlite3VdbeAddOp2(v, OP_AddImm, target, 0); sqlite3VdbeResolveLabel(v, destIfNull); - return target; + break; } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -96536,13 +88763,34 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** Z is stored in pExpr->pList->a[1].pExpr. */ case TK_BETWEEN: { - exprCodeBetween(pParse, pExpr, target, 0, 0); - return target; + Expr *pLeft = pExpr->pLeft; + struct ExprList_item *pLItem = pExpr->x.pList->a; + Expr *pRight = pLItem->pExpr; + + r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + r3 = sqlite3GetTempReg(pParse); + r4 = sqlite3GetTempReg(pParse); + codeCompare(pParse, pLeft, pRight, OP_Ge, + r1, r2, r3, SQLITE_STOREP2); VdbeCoverage(v); + pLItem++; + pRight = pLItem->pExpr; + sqlite3ReleaseTempReg(pParse, regFree2); + r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); + testcase( regFree2==0 ); + codeCompare(pParse, pLeft, pRight, OP_Le, r1, r2, r4, SQLITE_STOREP2); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_And, r3, r4, target); + sqlite3ReleaseTempReg(pParse, r3); + sqlite3ReleaseTempReg(pParse, r4); + break; } - case TK_SPAN: case TK_COLLATE: case TK_UPLUS: { - return sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + break; } case TK_TRIGGER: { @@ -96601,10 +88849,6 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) break; } - case TK_VECTOR: { - sqlite3ErrorMsg(pParse, "row value misused"); - break; - } /* ** Form A: @@ -96648,9 +88892,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( (pX = pExpr->pLeft)!=0 ){ tempX = *pX; testcase( pX->op==TK_COLUMN ); - exprToRegister(&tempX, exprCodeVector(pParse, &tempX, ®Free1)); + exprToRegister(&tempX, sqlite3ExprCodeTemp(pParse, pX, ®Free1)); testcase( regFree1==0 ); - memset(&opCompare, 0, sizeof(opCompare)); opCompare.op = TK_EQ; opCompare.pLeft = &tempX; pTest = &opCompare; @@ -96673,7 +88916,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); - sqlite3VdbeGoto(v, endLabel); + sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); sqlite3ExprCachePop(pParse); sqlite3VdbeResolveLabel(v, nextCase); } @@ -96684,7 +88927,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); } - assert( pParse->db->mallocFailed || pParse->nErr>0 + assert( db->mallocFailed || pParse->nErr>0 || pParse->iCacheLevel==iCacheLevel ); sqlite3VdbeResolveLabel(v, endLabel); break; @@ -96804,25 +89047,13 @@ SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); }else{ inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - assert( pParse->pVdbe!=0 || pParse->db->mallocFailed ); + assert( pParse->pVdbe || pParse->db->mallocFailed ); if( inReg!=target && pParse->pVdbe ){ sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); } } } -/* -** Make a transient copy of expression pExpr and then code it using -** sqlite3ExprCode(). This routine works just like sqlite3ExprCode() -** except that the input expression is guaranteed to be unchanged. -*/ -SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ - sqlite3 *db = pParse->db; - pExpr = sqlite3ExprDup(db, pExpr, 0); - if( !db->mallocFailed ) sqlite3ExprCode(pParse, pExpr, target); - sqlite3ExprDelete(db, pExpr); -} - /* ** Generate code that will evaluate expression pExpr and store the ** results in register target. The results are guaranteed to appear @@ -96861,6 +89092,268 @@ SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int targ exprToRegister(pExpr, iMem); } +#ifdef SQLITE_DEBUG +/* +** Generate a human-readable explanation of an expression tree. +*/ +SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ + const char *zBinOp = 0; /* Binary operator */ + const char *zUniOp = 0; /* Unary operator */ + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( pExpr==0 ){ + sqlite3TreeViewLine(pView, "nil"); + sqlite3TreeViewPop(pView); + return; + } + switch( pExpr->op ){ + case TK_AGG_COLUMN: { + sqlite3TreeViewLine(pView, "AGG{%d:%d}", + pExpr->iTable, pExpr->iColumn); + break; + } + case TK_COLUMN: { + if( pExpr->iTable<0 ){ + /* This only happens when coding check constraints */ + sqlite3TreeViewLine(pView, "COLUMN(%d)", pExpr->iColumn); + }else{ + sqlite3TreeViewLine(pView, "{%d:%d}", + pExpr->iTable, pExpr->iColumn); + } + break; + } + case TK_INTEGER: { + if( pExpr->flags & EP_IntValue ){ + sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue); + }else{ + sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken); + } + break; + } +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_STRING: { + sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); + break; + } + case TK_NULL: { + sqlite3TreeViewLine(pView,"NULL"); + break; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_VARIABLE: { + sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", + pExpr->u.zToken, pExpr->iColumn); + break; + } + case TK_REGISTER: { + sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); + break; + } + case TK_AS: { + sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + case TK_ID: { + sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken); + break; + } +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } +#endif /* SQLITE_OMIT_CAST */ + case TK_LT: zBinOp = "LT"; break; + case TK_LE: zBinOp = "LE"; break; + case TK_GT: zBinOp = "GT"; break; + case TK_GE: zBinOp = "GE"; break; + case TK_NE: zBinOp = "NE"; break; + case TK_EQ: zBinOp = "EQ"; break; + case TK_IS: zBinOp = "IS"; break; + case TK_ISNOT: zBinOp = "ISNOT"; break; + case TK_AND: zBinOp = "AND"; break; + case TK_OR: zBinOp = "OR"; break; + case TK_PLUS: zBinOp = "ADD"; break; + case TK_STAR: zBinOp = "MUL"; break; + case TK_MINUS: zBinOp = "SUB"; break; + case TK_REM: zBinOp = "REM"; break; + case TK_BITAND: zBinOp = "BITAND"; break; + case TK_BITOR: zBinOp = "BITOR"; break; + case TK_SLASH: zBinOp = "DIV"; break; + case TK_LSHIFT: zBinOp = "LSHIFT"; break; + case TK_RSHIFT: zBinOp = "RSHIFT"; break; + case TK_CONCAT: zBinOp = "CONCAT"; break; + case TK_DOT: zBinOp = "DOT"; break; + + case TK_UMINUS: zUniOp = "UMINUS"; break; + case TK_UPLUS: zUniOp = "UPLUS"; break; + case TK_BITNOT: zUniOp = "BITNOT"; break; + case TK_NOT: zUniOp = "NOT"; break; + case TK_ISNULL: zUniOp = "ISNULL"; break; + case TK_NOTNULL: zUniOp = "NOTNULL"; break; + + case TK_COLLATE: { + sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + + case TK_AGG_FUNCTION: + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + if( ExprHasProperty(pExpr, EP_TokenOnly) ){ + pFarg = 0; + }else{ + pFarg = pExpr->x.pList; + } + if( pExpr->op==TK_AGG_FUNCTION ){ + sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", + pExpr->op2, pExpr->u.zToken); + }else{ + sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); + } + if( pFarg ){ + sqlite3TreeViewExprList(pView, pFarg, 0, 0); + } + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: { + sqlite3TreeViewLine(pView, "EXISTS-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_SELECT: { + sqlite3TreeViewLine(pView, "SELECT-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_IN: { + sqlite3TreeViewLine(pView, "IN"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + }else{ + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + } + break; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + Expr *pX = pExpr->pLeft; + Expr *pY = pExpr->x.pList->a[0].pExpr; + Expr *pZ = pExpr->x.pList->a[1].pExpr; + sqlite3TreeViewLine(pView, "BETWEEN"); + sqlite3TreeViewExpr(pView, pX, 1); + sqlite3TreeViewExpr(pView, pY, 1); + sqlite3TreeViewExpr(pView, pZ, 0); + break; + } + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + */ + sqlite3TreeViewLine(pView, "%s(%d)", + pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); + break; + } + case TK_CASE: { + sqlite3TreeViewLine(pView, "CASE"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + const char *zType = "unk"; + switch( pExpr->affinity ){ + case OE_Rollback: zType = "rollback"; break; + case OE_Abort: zType = "abort"; break; + case OE_Fail: zType = "fail"; break; + case OE_Ignore: zType = "ignore"; break; + } + sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); + break; + } +#endif + default: { + sqlite3TreeViewLine(pView, "op=%d", pExpr->op); + break; + } + } + if( zBinOp ){ + sqlite3TreeViewLine(pView, "%s", zBinOp); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); + }else if( zUniOp ){ + sqlite3TreeViewLine(pView, "%s", zUniOp); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + } + sqlite3TreeViewPop(pView); +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* +** Generate a human-readable explanation of an expression list. +*/ +SQLITE_PRIVATE void sqlite3TreeViewExprList( + TreeView *pView, + const ExprList *pList, + u8 moreToFollow, + const char *zLabel +){ + int i; + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; + if( pList==0 ){ + sqlite3TreeViewLine(pView, "%s (empty)", zLabel); + }else{ + sqlite3TreeViewLine(pView, "%s", zLabel); + for(i=0; inExpr; i++){ + sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); +#if 0 + if( pList->a[i].zName ){ + sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); + } + if( pList->a[i].bSpanIsTab ){ + sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); + } +#endif + } + } + sqlite3TreeViewPop(pView); +} +#endif /* SQLITE_DEBUG */ + /* ** Generate code that pushes the value of every element of the given ** expression list into a sequence of registers beginning at target. @@ -96872,22 +89365,16 @@ SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int targ ** ** The SQLITE_ECEL_FACTOR argument allows constant arguments to be ** factored out into initialization code. -** -** The SQLITE_ECEL_REF flag means that expressions in the list with -** ExprList.a[].u.x.iOrderByCol>0 have already been evaluated and stored -** in registers at srcReg, and so the value can be copied from there. */ SQLITE_PRIVATE int sqlite3ExprCodeExprList( Parse *pParse, /* Parsing context */ ExprList *pList, /* The expression list to be coded */ int target, /* Where to write results */ - int srcReg, /* Source registers if SQLITE_ECEL_REF */ u8 flags /* SQLITE_ECEL_* flags */ ){ struct ExprList_item *pItem; - int i, j, n; + int i, n; u8 copyOp = (flags & SQLITE_ECEL_DUP) ? OP_Copy : OP_SCopy; - Vdbe *v = pParse->pVdbe; assert( pList!=0 ); assert( target>0 ); assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ @@ -96895,14 +89382,13 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; ipExpr; - if( (flags & SQLITE_ECEL_REF)!=0 && (j = pList->a[i].u.x.iOrderByCol)>0 ){ - sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); - }else if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ + if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ sqlite3ExprCodeAtInit(pParse, pExpr, target+i, 0); }else{ int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); if( inReg!=target+i ){ VdbeOp *pOp; + Vdbe *v = pParse->pVdbe; if( copyOp==OP_Copy && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg @@ -96929,33 +89415,20 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( ** ** Code it as such, taking care to do the common subexpression ** elimination of x. -** -** The xJumpIf parameter determines details: -** -** NULL: Store the boolean result in reg[dest] -** sqlite3ExprIfTrue: Jump to dest if true -** sqlite3ExprIfFalse: Jump to dest if false -** -** The jumpIfNull parameter is ignored if xJumpIf is NULL. */ static void exprCodeBetween( Parse *pParse, /* Parsing and code generating context */ Expr *pExpr, /* The BETWEEN expression */ - int dest, /* Jump destination or storage location */ - void (*xJump)(Parse*,Expr*,int,int), /* Action to take */ + int dest, /* Jump here if the jump is taken */ + int jumpIfTrue, /* Take the jump if the BETWEEN is true */ int jumpIfNull /* Take the jump if the BETWEEN is NULL */ ){ - Expr exprAnd; /* The AND operator in x>=y AND x<=z */ + Expr exprAnd; /* The AND operator in x>=y AND x<=z */ Expr compLeft; /* The x>=y term */ Expr compRight; /* The x<=z term */ Expr exprX; /* The x subexpression */ int regFree1 = 0; /* Temporary use register */ - - memset(&compLeft, 0, sizeof(Expr)); - memset(&compRight, 0, sizeof(Expr)); - memset(&exprAnd, 0, sizeof(Expr)); - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); exprX = *pExpr->pLeft; exprAnd.op = TK_AND; @@ -96967,25 +89440,23 @@ static void exprCodeBetween( compRight.op = TK_LE; compRight.pLeft = &exprX; compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprToRegister(&exprX, exprCodeVector(pParse, &exprX, ®Free1)); - if( xJump ){ - xJump(pParse, &exprAnd, dest, jumpIfNull); + exprToRegister(&exprX, sqlite3ExprCodeTemp(pParse, &exprX, ®Free1)); + if( jumpIfTrue ){ + sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); }else{ - exprX.flags |= EP_FromJoin; - sqlite3ExprCodeTarget(pParse, &exprAnd, dest); + sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull); } sqlite3ReleaseTempReg(pParse, regFree1); /* Ensure adequate test coverage */ - testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1==0 ); - testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1!=0 ); - testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1==0 ); - testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1!=0 ); - testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1==0 ); - testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1!=0 ); - testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1==0 ); - testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1!=0 ); - testcase( xJump==0 ); + testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 ); + testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 ); + testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 ); + testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 ); + testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 ); + testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 ); + testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 ); + testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 ); } /* @@ -97037,20 +89508,12 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); break; } - case TK_IS: - case TK_ISNOT: - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - op = (op==TK_IS) ? TK_EQ : TK_NE; - jumpIfNull = SQLITE_NULLEQ; - /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { - if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); @@ -97060,12 +89523,23 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); - VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); - VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); - VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); - VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_IS: + case TK_ISNOT: { + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + op = (op==TK_IS) ? TK_EQ : TK_NE; + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -97083,7 +89557,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int } case TK_BETWEEN: { testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfTrue, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull); break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -97091,15 +89565,14 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int int destIfFalse = sqlite3VdbeMakeLabel(v); int destIfNull = jumpIfNull ? dest : destIfFalse; sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); - sqlite3VdbeGoto(v, dest); + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); sqlite3VdbeResolveLabel(v, destIfFalse); break; } #endif default: { - default_expr: if( exprAlwaysTrue(pExpr) ){ - sqlite3VdbeGoto(v, dest); + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); }else if( exprAlwaysFalse(pExpr) ){ /* No-op */ }else{ @@ -97191,20 +89664,12 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); break; } - case TK_IS: - case TK_ISNOT: - testcase( pExpr->op==TK_IS ); - testcase( pExpr->op==TK_ISNOT ); - op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; - jumpIfNull = SQLITE_NULLEQ; - /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { - if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); @@ -97214,12 +89679,23 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); - VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); - VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); - VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); - VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_IS: + case TK_ISNOT: { + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -97235,7 +89711,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int } case TK_BETWEEN: { testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfFalse, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull); break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -97251,9 +89727,8 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int } #endif default: { - default_expr: if( exprAlwaysFalse(pExpr) ){ - sqlite3VdbeGoto(v, dest); + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); }else if( exprAlwaysTrue(pExpr) ){ /* no-op */ }else{ @@ -97270,21 +89745,6 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int sqlite3ReleaseTempReg(pParse, regFree2); } -/* -** Like sqlite3ExprIfFalse() except that a copy is made of pExpr before -** code generation, and that copy is deleted after code generation. This -** ensures that the original pExpr is unchanged. -*/ -SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse *pParse, Expr *pExpr, int dest,int jumpIfNull){ - sqlite3 *db = pParse->db; - Expr *pCopy = sqlite3ExprDup(db, pExpr, 0); - if( db->mallocFailed==0 ){ - sqlite3ExprIfFalse(pParse, pCopy, dest, jumpIfNull); - } - sqlite3ExprDelete(db, pCopy); -} - - /* ** Do a deep comparison of two expression trees. Return 0 if the two ** expressions are completely identical. Return 1 if they differ only @@ -97328,10 +89788,8 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ } return 2; } - if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){ - if( pA->op==TK_FUNCTION ){ - if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; - }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken ){ + if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ return pA->op==TK_COLLATE ? 1 : 2; } } @@ -97418,61 +89876,6 @@ SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Expr *pE1, Expr *pE2, int iTab){ return 0; } -/* -** An instance of the following structure is used by the tree walker -** to determine if an expression can be evaluated by reference to the -** index only, without having to do a search for the corresponding -** table entry. The IdxCover.pIdx field is the index. IdxCover.iCur -** is the cursor for the table. -*/ -struct IdxCover { - Index *pIdx; /* The index to be tested for coverage */ - int iCur; /* Cursor number for the table corresponding to the index */ -}; - -/* -** Check to see if there are references to columns in table -** pWalker->u.pIdxCover->iCur can be satisfied using the index -** pWalker->u.pIdxCover->pIdx. -*/ -static int exprIdxCover(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_COLUMN - && pExpr->iTable==pWalker->u.pIdxCover->iCur - && sqlite3ColumnOfIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 - ){ - pWalker->eCode = 1; - return WRC_Abort; - } - return WRC_Continue; -} - -/* -** Determine if an index pIdx on table with cursor iCur contains will -** the expression pExpr. Return true if the index does cover the -** expression and false if the pExpr expression references table columns -** that are not found in the index pIdx. -** -** An index covering an expression means that the expression can be -** evaluated using only the index and without having to lookup the -** corresponding table entry. -*/ -SQLITE_PRIVATE int sqlite3ExprCoveredByIndex( - Expr *pExpr, /* The index to be tested */ - int iCur, /* The cursor number for the corresponding table */ - Index *pIdx /* The index that might be used for coverage */ -){ - Walker w; - struct IdxCover xcov; - memset(&w, 0, sizeof(w)); - xcov.iCur = iCur; - xcov.pIdx = pIdx; - w.xExprCallback = exprIdxCover; - w.u.pIdxCover = &xcov; - sqlite3WalkExpr(&w, pExpr); - return !w.eCode; -} - - /* ** An instance of the following structure is used by the tree walker ** to count references to table columns in the arguments of an @@ -97670,7 +90073,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ pItem->iMem = ++pParse->nMem; assert( !ExprHasProperty(pExpr, EP_IntValue) ); pItem->pFunc = sqlite3FindFunction(pParse->db, - pExpr->u.zToken, + pExpr->u.zToken, sqlite3Strlen30(pExpr->u.zToken), pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); if( pExpr->flags & EP_Distinct ){ pItem->iDistinct = pParse->nTab++; @@ -97756,7 +90159,7 @@ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ if( iReg && pParse->nTempRegaTempReg) ){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ + for(i=0, p=pParse->aColCache; iiReg==iReg ){ p->tempReg = 1; return; @@ -97767,11 +90170,10 @@ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ } /* -** Allocate or deallocate a block of nReg consecutive registers. +** Allocate or deallocate a block of nReg consecutive registers */ SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ int i, n; - if( nReg==1 ) return sqlite3GetTempReg(pParse); i = pParse->iRangeReg; n = pParse->nRangeReg; if( nReg<=n ){ @@ -97785,10 +90187,6 @@ SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ return i; } SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ - if( nReg==1 ){ - sqlite3ReleaseTempReg(pParse, iReg); - return; - } sqlite3ExprCacheRemove(pParse, iReg, nReg); if( nReg>pParse->nRangeReg ){ pParse->nRangeReg = nReg; @@ -97804,29 +90202,6 @@ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nRangeReg = 0; } -/* -** Validate that no temporary register falls within the range of -** iFirst..iLast, inclusive. This routine is only call from within assert() -** statements. -*/ -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ - int i; - if( pParse->nRangeReg>0 - && pParse->iRangeReg+pParse->nRangeRegiRangeReg>=iFirst - ){ - return 0; - } - for(i=0; inTempReg; i++){ - if( pParse->aTempReg[i]>=iFirst && pParse->aTempReg[i]<=iLast ){ - return 0; - } - } - return 1; -} -#endif /* SQLITE_DEBUG */ - /************** End of expr.c ************************************************/ /************** Begin file alter.c *******************************************/ /* @@ -97843,7 +90218,6 @@ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ ** This file contains C code routines that used to generate VDBE code ** that implements the ALTER TABLE command. */ -/* #include "sqliteInt.h" */ /* ** The code in this file only exists if we are not omitting the @@ -98060,7 +90434,7 @@ static void renameTriggerFunc( ** Register built-in functions used to help implement ALTER TABLE */ SQLITE_PRIVATE void sqlite3AlterFunctions(void){ - static FuncDef aAlterTableFuncs[] = { + static SQLITE_WSD FuncDef aAlterTableFuncs[] = { FUNCTION(sqlite_rename_table, 2, 0, 0, renameTableFunc), #ifndef SQLITE_OMIT_TRIGGER FUNCTION(sqlite_rename_trigger, 2, 0, 0, renameTriggerFunc), @@ -98069,7 +90443,13 @@ SQLITE_PRIVATE void sqlite3AlterFunctions(void){ FUNCTION(sqlite_rename_parent, 3, 0, 0, renameParentFunc), #endif }; - sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAlterTableFuncs); + + for(i=0; ia[0]); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - zDb = db->aDb[iDb].zDbSName; + zDb = db->aDb[iDb].zName; db->flags |= SQLITE_PreferBuiltin; /* Get a NULL terminated version of the new table name. */ @@ -98316,7 +90696,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( #ifndef SQLITE_OMIT_VIRTUALTABLE if( pVTab ){ int i = ++pParse->nMem; - sqlite3VdbeLoadString(v, i, zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, i, 0, zName, 0); sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); sqlite3MayAbort(pParse); } @@ -98413,6 +90793,33 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( db->flags = savedDbFlags; } + +/* +** Generate code to make sure the file format number is at least minFormat. +** The generated code will increase the file format number if necessary. +*/ +SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse *pParse, int iDb, int minFormat){ + Vdbe *v; + v = sqlite3GetVdbe(pParse); + /* The VDBE should have been allocated before this routine is called. + ** If that allocation failed, we would have quit before reaching this + ** point */ + if( ALWAYS(v) ){ + int r1 = sqlite3GetTempReg(pParse); + int r2 = sqlite3GetTempReg(pParse); + int j1; + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_Integer, minFormat, r2); + j1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, r2); + sqlite3VdbeJumpHere(v, j1); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempReg(pParse, r2); + } +} + /* ** This function is called after an "ALTER TABLE ... ADD" statement ** has been parsed. Argument pColDef contains the text of the new @@ -98431,18 +90838,15 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ Column *pCol; /* The new column */ Expr *pDflt; /* Default value for the new column */ sqlite3 *db; /* The database connection; */ - Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ - int r1; /* Temporary registers */ db = pParse->db; if( pParse->nErr || db->mallocFailed ) return; - assert( v!=0 ); pNew = pParse->pNewTable; assert( pNew ); assert( sqlite3BtreeHoldsAllMutexes(db) ); iDb = sqlite3SchemaToIndex(db, pNew->pSchema); - zDb = db->aDb[iDb].zDbSName; + zDb = db->aDb[iDb].zName; zTab = &pNew->zName[16]; /* Skip the "sqlite_altertab_" prefix on the name */ pCol = &pNew->aCol[pNew->nCol-1]; pDflt = pCol->pDflt; @@ -98460,8 +90864,7 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** literal NULL, then set pDflt to 0. This simplifies checking ** for an SQL NULL default below. */ - assert( pDflt==0 || pDflt->op==TK_SPAN ); - if( pDflt && pDflt->pLeft->op==TK_NULL ){ + if( pDflt && pDflt->op==TK_NULL ){ pDflt = 0; } @@ -98494,10 +90897,10 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ if( pDflt ){ sqlite3_value *pVal = 0; int rc; - rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); + rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal); assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); if( rc!=SQLITE_OK ){ - assert( db->mallocFailed == 1 ); + db->mallocFailed = 1; return; } if( !pVal ){ @@ -98527,18 +90930,11 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ db->flags = savedDbFlags; } - /* Make sure the schema version is at least 3. But do not upgrade - ** from less than 3 to 4, as that will corrupt any preexisting DESC - ** index. + /* If the default value of the new column is NULL, then set the file + ** format to 2. If the default value of the new column is not NULL, + ** the file format becomes 3. */ - r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); - sqlite3VdbeUsesBtree(v, iDb); - sqlite3VdbeAddOp2(v, OP_AddImm, r1, -2); - sqlite3VdbeAddOp2(v, OP_IfPos, r1, sqlite3VdbeCurrentAddr(v)+2); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); - sqlite3ReleaseTempReg(pParse, r1); + sqlite3MinimumFileFormat(pParse, iDb, pDflt ? 3 : 2); /* Reload the schema of the modified table. */ reloadTableSchema(pParse, pTab, pTab->zName); @@ -98612,7 +91008,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); if( !pNew->aCol || !pNew->zName ){ - assert( db->mallocFailed ); + db->mallocFailed = 1; goto exit_begin_add_column; } memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); @@ -98620,7 +91016,9 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ Column *pCol = &pNew->aCol[i]; pCol->zName = sqlite3DbStrDup(db, pCol->zName); pCol->zColl = 0; + pCol->zType = 0; pCol->pDflt = 0; + pCol->zDflt = 0; } pNew->pSchema = db->aDb[iDb].pSchema; pNew->addColOffset = pTab->addColOffset; @@ -98782,7 +91180,6 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ ** integer in the equivalent columns in sqlite_stat4. */ #ifndef SQLITE_OMIT_ANALYZE -/* #include "sqliteInt.h" */ #if defined(SQLITE_ENABLE_STAT4) # define IsStat4 1 @@ -98852,14 +91249,14 @@ static void openStatTable( for(i=0; izDbSName))==0 ){ + if( (pStat = sqlite3FindTable(db, zTab, pDb->zName))==0 ){ if( aTable[i].zCols ){ /* The sqlite_statN table does not exist. Create it. Note that a ** side-effect of the CREATE TABLE statement is to leave the rootpage ** of the new table in register pParse->regRoot. This is important ** because the OpenWrite opcode below will be needing it. */ sqlite3NestedParse(pParse, - "CREATE TABLE %Q.%s(%s)", pDb->zDbSName, zTab, aTable[i].zCols + "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols ); aRoot[i] = pParse->regRoot; aCreateTbl[i] = OPFLAG_P2ISREG; @@ -98874,7 +91271,7 @@ static void openStatTable( if( zWhere ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE %s=%Q", - pDb->zDbSName, zTab, zWhereType, zWhere + pDb->zName, zTab, zWhereType, zWhere ); }else{ /* The sqlite_stat[134] table already exists. Delete all rows. */ @@ -98955,7 +91352,7 @@ static void sampleClear(sqlite3 *db, Stat4Sample *p){ static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ assert( db!=0 ); if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); - p->u.aRowid = sqlite3DbMallocRawNN(db, n); + p->u.aRowid = sqlite3DbMallocRaw(db, n); if( p->u.aRowid ){ p->nRowid = n; memcpy(p->u.aRowid, pData, n); @@ -99120,10 +91517,12 @@ static const FuncDef statInitFuncdef = { SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statInit, /* xSFunc */ + statInit, /* xFunc */ + 0, /* xStep */ 0, /* xFinalize */ "stat_init", /* zName */ - {0} + 0, /* pHash */ + 0 /* pDestructor */ }; #ifdef SQLITE_ENABLE_STAT4 @@ -99419,10 +91818,12 @@ static const FuncDef statPushFuncdef = { SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statPush, /* xSFunc */ + statPush, /* xFunc */ + 0, /* xStep */ 0, /* xFinalize */ "stat_push", /* zName */ - {0} + 0, /* pHash */ + 0 /* pDestructor */ }; #define STAT_GET_STAT1 0 /* "stat" column of stat1 table */ @@ -99564,10 +91965,12 @@ static const FuncDef statGetFuncdef = { SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - statGet, /* xSFunc */ + statGet, /* xFunc */ + 0, /* xStep */ 0, /* xFinalize */ "stat_get", /* zName */ - {0} + 0, /* pHash */ + 0 /* pDestructor */ }; static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ @@ -99579,8 +91982,8 @@ static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ #else UNUSED_PARAMETER( iParam ); #endif - sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4, regOut, - (char*)&statGetFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regOut); + sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 1 + IsStat34); } @@ -99626,7 +92029,7 @@ static void analyzeOneTable( /* Do not gather statistics on views or virtual tables */ return; } - if( sqlite3_strlike("sqlite_%", pTab->zName, 0)==0 ){ + if( sqlite3_strnicmp(pTab->zName, "sqlite_", 7)==0 ){ /* Do not gather statistics on system tables */ return; } @@ -99636,7 +92039,7 @@ static void analyzeOneTable( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_ANALYZE, pTab->zName, 0, - db->aDb[iDb].zDbSName ) ){ + db->aDb[iDb].zName ) ){ return; } #endif @@ -99650,7 +92053,7 @@ static void analyzeOneTable( iIdxCur = iTab++; pParse->nTab = MAX(pParse->nTab, iTab); sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeLoadString(v, regTabname, pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ @@ -99672,7 +92075,7 @@ static void analyzeOneTable( } /* Populate the register containing the index name. */ - sqlite3VdbeLoadString(v, regIdxname, zIdxName); + sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0); VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); /* @@ -99734,8 +92137,8 @@ static void analyzeOneTable( #endif sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); - sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4+1, regStat4, - (char*)&statInitFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); + sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); /* Implementation of the following: @@ -99754,7 +92157,7 @@ static void analyzeOneTable( if( nColTest>0 ){ int endDistinctTest = sqlite3VdbeMakeLabel(v); int *aGotoChng; /* Array of jump instruction addresses */ - aGotoChng = sqlite3DbMallocRawNN(db, sizeof(int)*nColTest); + aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest); if( aGotoChng==0 ) continue; /* @@ -99786,7 +92189,7 @@ static void analyzeOneTable( VdbeCoverage(v); } sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); - sqlite3VdbeGoto(v, endDistinctTest); + sqlite3VdbeAddOp2(v, OP_Goto, 0, endDistinctTest); /* @@ -99822,7 +92225,6 @@ static void analyzeOneTable( regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); - assert( k>=0 && knCol ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); } @@ -99831,8 +92233,8 @@ static void analyzeOneTable( } #endif assert( regChng==(regStat4+1) ); - sqlite3VdbeAddOp4(v, OP_Function0, 1, regStat4, regTemp, - (char*)&statPushFuncdef, P4_FUNCDEF); + sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp); + sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); @@ -99872,10 +92274,12 @@ static void analyzeOneTable( ** be taken */ VdbeCoverageNeverTaken(v); #ifdef SQLITE_ENABLE_STAT3 - sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, 0, regSample); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, + pIdx->aiColumn[0], regSample); #else for(i=0; iaiColumn[i]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); #endif @@ -100026,7 +92430,7 @@ SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ /* Form 3: Analyze the fully qualified table name */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); if( iDb>=0 ){ - zDb = db->aDb[iDb].zDbSName; + zDb = db->aDb[iDb].zName; z = sqlite3NameFromToken(db, pTableName); if( z ){ if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ @@ -100162,7 +92566,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** the old data with the new instead of allocating a new array. */ if( pIndex->aiRowEst==0 ){ pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero(sizeof(tRowcnt) * nCol); - if( pIndex->aiRowEst==0 ) sqlite3OomFault(pInfo->db); + if( pIndex->aiRowEst==0 ) pInfo->db->mallocFailed = 1; } aiRowEst = pIndex->aiRowEst; #endif @@ -100309,10 +92713,10 @@ static int loadStatTbl( Index *pPrevIdx = 0; /* Previous index in the loop */ IndexSample *pSample; /* A slot in pIdx->aSample[] */ - assert( db->lookaside.bDisable ); + assert( db->lookaside.bEnabled==0 ); zSql = sqlite3MPrintf(db, zSql1, zDb); if( !zSql ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); sqlite3DbFree(db, zSql); @@ -100352,7 +92756,7 @@ static int loadStatTbl( pIdx->aSample = sqlite3DbMallocZero(db, nByte); if( pIdx->aSample==0 ){ sqlite3_finalize(pStmt); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } pSpace = (tRowcnt*)&pIdx->aSample[nSample]; pIdx->aAvgEq = pSpace; pSpace += nIdxCol; @@ -100368,7 +92772,7 @@ static int loadStatTbl( zSql = sqlite3MPrintf(db, zSql2, zDb); if( !zSql ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); sqlite3DbFree(db, zSql); @@ -100406,7 +92810,7 @@ static int loadStatTbl( pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); if( pSample->p==0 ){ sqlite3_finalize(pStmt); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); pIdx->nSample++; @@ -100423,7 +92827,7 @@ static int loadStatTbl( static int loadStat4(sqlite3 *db, const char *zDb){ int rc = SQLITE_OK; /* Result codes from subroutines */ - assert( db->lookaside.bDisable ); + assert( db->lookaside.bEnabled==0 ); if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ rc = loadStatTbl(db, 0, "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", @@ -100468,7 +92872,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ analysisInfo sInfo; HashElem *i; char *zSql; - int rc = SQLITE_OK; + int rc; assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 ); @@ -100477,40 +92881,38 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - pIdx->aiRowLogEst[0] = 0; + sqlite3DefaultRowEst(pIdx); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; #endif } - /* Load new statistics out of the sqlite_stat1 table */ + /* Check to make sure the sqlite_stat1 table exists */ sInfo.db = db; - sInfo.zDatabase = db->aDb[iDb].zDbSName; - if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)!=0 ){ - zSql = sqlite3MPrintf(db, - "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); - if( zSql==0 ){ - rc = SQLITE_NOMEM_BKPT; - }else{ - rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); - sqlite3DbFree(db, zSql); - } + sInfo.zDatabase = db->aDb[iDb].zName; + if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)==0 ){ + return SQLITE_ERROR; } - /* Set appropriate defaults on all indexes not in the sqlite_stat1 table */ - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ - Index *pIdx = sqliteHashData(i); - if( pIdx->aiRowLogEst[0]==0 ) sqlite3DefaultRowEst(pIdx); + /* Load new statistics out of the sqlite_stat1 table */ + zSql = sqlite3MPrintf(db, + "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); + sqlite3DbFree(db, zSql); } + /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ - db->lookaside.bDisable++; + int lookasideEnabled = db->lookaside.bEnabled; + db->lookaside.bEnabled = 0; rc = loadStat4(db, sInfo.zDatabase); - db->lookaside.bDisable--; + db->lookaside.bEnabled = lookasideEnabled; } for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); @@ -100520,7 +92922,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ #endif if( rc==SQLITE_NOMEM ){ - sqlite3OomFault(db); + db->mallocFailed = 1; } return rc; } @@ -100543,7 +92945,6 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ ************************************************************************* ** This file contains code used to implement the ATTACH and DETACH commands. */ -/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_ATTACH /* @@ -100629,7 +93030,7 @@ static void attachFunc( goto attach_error; } for(i=0; inDb; i++){ - char *z = db->aDb[i].zDbSName; + char *z = db->aDb[i].zName; assert( z && zName ); if( sqlite3StrICmp(z, zName)==0 ){ zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); @@ -100641,7 +93042,7 @@ static void attachFunc( ** hash tables. */ if( db->aDb==db->aDbStatic ){ - aNew = sqlite3DbMallocRawNN(db, sizeof(db->aDb[0])*3 ); + aNew = sqlite3DbMallocRaw(db, sizeof(db->aDb[0])*3 ); if( aNew==0 ) return; memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); }else{ @@ -100659,7 +93060,7 @@ static void attachFunc( flags = db->openFlags; rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); return; @@ -100676,7 +93077,7 @@ static void attachFunc( Pager *pPager; aNew->pSchema = sqlite3SchemaGet(db, aNew->pBt); if( !aNew->pSchema ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){ zErrDyn = sqlite3MPrintf(db, "attached databases must use the same text encoding as main database"); @@ -100688,15 +93089,14 @@ static void attachFunc( sqlite3BtreeSecureDelete(aNew->pBt, sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); #ifndef SQLITE_OMIT_PAGER_PRAGMAS - sqlite3BtreeSetPagerFlags(aNew->pBt, - PAGER_SYNCHRONOUS_FULL | (db->flags & PAGER_FLAGS_MASK)); + sqlite3BtreeSetPagerFlags(aNew->pBt, 3 | (db->flags & PAGER_FLAGS_MASK)); #endif sqlite3BtreeLeave(aNew->pBt); } - aNew->safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; - aNew->zDbSName = sqlite3DbStrDup(db, zName); - if( rc==SQLITE_OK && aNew->zDbSName==0 ){ - rc = SQLITE_NOMEM_BKPT; + aNew->safety_level = 3; + aNew->zName = sqlite3DbStrDup(db, zName); + if( rc==SQLITE_OK && aNew->zName==0 ){ + rc = SQLITE_NOMEM; } @@ -100724,7 +93124,7 @@ static void attachFunc( case SQLITE_NULL: /* No key specified. Use the key from the main database */ sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); - if( nKey || sqlite3BtreeGetOptimalReserve(db->aDb[0].pBt)>0 ){ + if( nKey>0 || sqlite3BtreeGetOptimalReserve(db->aDb[0].pBt)>0 ){ rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); } break; @@ -100762,7 +93162,7 @@ static void attachFunc( sqlite3ResetAllSchemasOfConnection(db); db->nDb = iDb; if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - sqlite3OomFault(db); + db->mallocFailed = 1; sqlite3DbFree(db, zErrDyn); zErrDyn = sqlite3MPrintf(db, "out of memory"); }else if( zErrDyn==0 ){ @@ -100807,7 +93207,7 @@ static void detachFunc( for(i=0; inDb; i++){ pDb = &db->aDb[i]; if( pDb->pBt==0 ) continue; - if( sqlite3StrICmp(pDb->zDbSName, zName)==0 ) break; + if( sqlite3StrICmp(pDb->zName, zName)==0 ) break; } if( i>=db->nDb ){ @@ -100857,7 +93257,6 @@ static void codeAttach( sqlite3* db = pParse->db; int regArgs; - if( pParse->nErr ) goto attach_end; memset(&sName, 0, sizeof(NameContext)); sName.pParse = pParse; @@ -100893,11 +93292,11 @@ static void codeAttach( assert( v || db->mallocFailed ); if( v ){ - sqlite3VdbeAddOp4(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3, - (char *)pFunc, P4_FUNCDEF); + sqlite3VdbeAddOp3(v, OP_Function, 0, regArgs+3-pFunc->nArg, regArgs+3); assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); - + sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF); + /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this ** statement only). For DETACH, set it to false (expire all existing ** statements). @@ -100922,10 +93321,12 @@ SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - detachFunc, /* xSFunc */ + detachFunc, /* xFunc */ + 0, /* xStep */ 0, /* xFinalize */ "sqlite_detach", /* zName */ - {0} + 0, /* pHash */ + 0 /* pDestructor */ }; codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); } @@ -100941,10 +93342,12 @@ SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *p SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ - attachFunc, /* xSFunc */ + attachFunc, /* xFunc */ + 0, /* xStep */ 0, /* xFinalize */ "sqlite_attach", /* zName */ - {0} + 0, /* pHash */ + 0 /* pDestructor */ }; codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); } @@ -100966,7 +93369,7 @@ SQLITE_PRIVATE void sqlite3FixInit( db = pParse->db; assert( db->nDb>iDb ); pFix->pParse = pParse; - pFix->zDb = db->aDb[iDb].zDbSName; + pFix->zDb = db->aDb[iDb].zName; pFix->pSchema = db->aDb[iDb].pSchema; pFix->zType = zType; pFix->pName = pName; @@ -101063,7 +93466,7 @@ SQLITE_PRIVATE int sqlite3FixExpr( return 1; } } - if( ExprHasProperty(pExpr, EP_TokenOnly|EP_Leaf) ) break; + if( ExprHasProperty(pExpr, EP_TokenOnly) ) break; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ if( sqlite3FixSelect(pFix, pExpr->x.pSelect) ) return 1; }else{ @@ -101131,7 +93534,6 @@ SQLITE_PRIVATE int sqlite3FixTriggerStep( ** systems that do not need this facility may omit it by recompiling ** the library with -DSQLITE_OMIT_AUTHORIZATION=1 */ -/* #include "sqliteInt.h" */ /* ** All of the code in this file may be omitted by defining a single @@ -101184,7 +93586,7 @@ SQLITE_PRIVATE int sqlite3FixTriggerStep( ** Setting the auth function to NULL disables this hook. The default ** setting of the auth function is NULL. */ -SQLITE_API int sqlite3_set_authorizer( +SQLITE_API int SQLITE_STDCALL sqlite3_set_authorizer( sqlite3 *db, int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pArg @@ -101224,11 +93626,10 @@ SQLITE_PRIVATE int sqlite3AuthReadCol( const char *zCol, /* Column name */ int iDb /* Index of containing database. */ ){ - sqlite3 *db = pParse->db; /* Database handle */ - char *zDb = db->aDb[iDb].zDbSName; /* Schema name of attached database */ - int rc; /* Auth callback return code */ + sqlite3 *db = pParse->db; /* Database handle */ + char *zDb = db->aDb[iDb].zName; /* Name of attached database */ + int rc; /* Auth callback return code */ - if( db->init.busy ) return SQLITE_OK; rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext #ifdef SQLITE_USER_AUTHENTICATION ,db->auth.zAuthUser @@ -101403,7 +93804,15 @@ SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext *pContext){ ** COMMIT ** ROLLBACK */ -/* #include "sqliteInt.h" */ + +/* +** This routine is called when a new SQL statement is beginning to +** be parsed. Initialize the pParse structure as needed. +*/ +SQLITE_PRIVATE void sqlite3BeginParse(Parse *pParse, int explainFlag){ + pParse->explain = (u8)explainFlag; + pParse->nVar = 0; +} #ifndef SQLITE_OMIT_SHARED_CACHE /* @@ -101459,7 +93868,7 @@ SQLITE_PRIVATE void sqlite3TableLock( p->zName = zName; }else{ pToplevel->nTableLock = 0; - sqlite3OomFault(pToplevel->db); + pToplevel->db->mallocFailed = 1; } } @@ -101527,14 +93936,15 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ assert( !pParse->isMultiWrite || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ + while( sqlite3VdbeDeletePriorOpcode(v, OP_Close) ){} sqlite3VdbeAddOp0(v, OP_Halt); #if SQLITE_USER_AUTHENTICATION if( pParse->nTableLock>0 && db->init.busy==0 ){ sqlite3UserAuthInit(db); if( db->auth.authLevelrc = SQLITE_AUTH_USER; + sqlite3ErrorMsg(pParse, "user not authenticated"); return; } } @@ -101553,20 +93963,16 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); sqlite3VdbeJumpHere(v, 0); for(iDb=0; iDbnDb; iDb++){ - Schema *pSchema; if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; sqlite3VdbeUsesBtree(v, iDb); - pSchema = db->aDb[iDb].pSchema; sqlite3VdbeAddOp4Int(v, OP_Transaction, /* Opcode */ iDb, /* P1 */ DbMaskTest(pParse->writeMask,iDb), /* P2 */ - pSchema->schema_cookie, /* P3 */ - pSchema->iGeneration /* P4 */ + pParse->cookieValue[iDb], /* P3 */ + db->aDb[iDb].pSchema->iGeneration /* P4 */ ); if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); - VdbeComment((v, - "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); } #ifndef SQLITE_OMIT_VIRTUALTABLE for(i=0; inVtabLock; i++){ @@ -101596,7 +94002,7 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ } /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeGoto(v, 1); + sqlite3VdbeAddOp2(v, OP_Goto, 0, 1); } } @@ -101610,9 +94016,15 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; + pParse->colNamesSet = 0; }else{ pParse->rc = SQLITE_ERROR; } + pParse->nTab = 0; + pParse->nMem = 0; + pParse->nSet = 0; + pParse->nVar = 0; + DbMaskZero(pParse->cookieMask); } /* @@ -101632,7 +94044,8 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ char *zSql; char *zErrMsg = 0; sqlite3 *db = pParse->db; - char saveBuf[PARSE_TAIL_SZ]; +# define SAVE_SZ (sizeof(Parse) - offsetof(Parse,nVar)) + char saveBuf[SAVE_SZ]; if( pParse->nErr ) return; assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ @@ -101643,12 +94056,12 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ return; /* A malloc must have failed */ } pParse->nested++; - memcpy(saveBuf, PARSE_TAIL(pParse), PARSE_TAIL_SZ); - memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); + memcpy(saveBuf, &pParse->nVar, SAVE_SZ); + memset(&pParse->nVar, 0, SAVE_SZ); sqlite3RunParser(pParse, zSql, &zErrMsg); sqlite3DbFree(db, zErrMsg); sqlite3DbFree(db, zSql); - memcpy(PARSE_TAIL(pParse), saveBuf, PARSE_TAIL_SZ); + memcpy(&pParse->nVar, saveBuf, SAVE_SZ); pParse->nested--; } @@ -101689,11 +94102,10 @@ SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const cha #endif for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDatabase==0 || sqlite3StrICmp(zDatabase, db->aDb[j].zDbSName)==0 ){ - assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); - if( p ) break; - } + if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); + if( p ) break; } return p; } @@ -101710,7 +94122,7 @@ SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const cha */ SQLITE_PRIVATE Table *sqlite3LocateTable( Parse *pParse, /* context in which to report errors */ - u32 flags, /* LOCATE_VIEW or LOCATE_NOERR */ + int isView, /* True if looking for a VIEW rather than a TABLE */ const char *zName, /* Name of the table we are looking for */ const char *zDbase /* Name of the database. Might be NULL */ ){ @@ -101724,28 +94136,20 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( p = sqlite3FindTable(pParse->db, zName, zDbase); if( p==0 ){ - const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table"; -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( sqlite3FindDbName(pParse->db, zDbase)<1 ){ - /* If zName is the not the name of a table in the schema created using - ** CREATE, then check to see if it is the name of an virtual table that - ** can be an eponymous virtual table. */ - Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); - if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ - return pMod->pEpoTab; - } - } -#endif - if( (flags & LOCATE_NOERR)==0 ){ - if( zDbase ){ - sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); - }else{ - sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); - } - pParse->checkSchema = 1; + const char *zMsg = isView ? "no such view" : "no such table"; + if( zDbase ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); } + pParse->checkSchema = 1; } - +#if SQLITE_USER_AUTHENICATION + else if( pParse->db->auth.authLevelpSchema==0 || p->zDatabase==0 ); if( p->pSchema ){ int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); - zDb = pParse->db->aDb[iDb].zDbSName; + zDb = pParse->db->aDb[iDb].zName; }else{ zDb = p->zDatabase; } - return sqlite3LocateTable(pParse, flags, p->zName, zDb); + return sqlite3LocateTable(pParse, isView, p->zName, zDb); } /* @@ -101795,7 +94199,7 @@ SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const cha int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ Schema *pSchema = db->aDb[j].pSchema; assert( pSchema ); - if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zDbSName) ) continue; + if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); p = sqlite3HashFind(&pSchema->idxHash, zName); if( p ) break; @@ -101811,9 +94215,8 @@ static void freeIndex(sqlite3 *db, Index *p){ sqlite3DeleteIndexSamples(db, p); #endif sqlite3ExprDelete(db, p->pPartIdxWhere); - sqlite3ExprListDelete(db, p->aColExpr); sqlite3DbFree(db, p->zColAff); - if( p->isResized ) sqlite3DbFree(db, (void *)p->azColl); + if( p->isResized ) sqlite3DbFree(db, p->azColl); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3_free(p->aiRowEst); #endif @@ -101864,8 +94267,8 @@ SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3 *db){ for(i=j=2; inDb; i++){ struct Db *pDb = &db->aDb[i]; if( pDb->pBt==0 ){ - sqlite3DbFree(db, pDb->zDbSName); - pDb->zDbSName = 0; + sqlite3DbFree(db, pDb->zName); + pDb->zName = 0; continue; } if( jaDb[j], 0, (db->nDb-j)*sizeof(db->aDb[j])); db->nDb = j; if( db->nDb<=2 && db->aDb!=db->aDbStatic ){ memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0])); @@ -101937,7 +94341,7 @@ SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ ** Delete memory allocated for the column names of a table or view (the ** Table.aCol[] array). */ -SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ +static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; assert( pTable!=0 ); @@ -101945,6 +94349,8 @@ SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ for(i=0; inCol; i++, pCol++){ sqlite3DbFree(db, pCol->zName); sqlite3ExprDelete(db, pCol->pDflt); + sqlite3DbFree(db, pCol->zDflt); + sqlite3DbFree(db, pCol->zType); sqlite3DbFree(db, pCol->zColl); } sqlite3DbFree(db, pTable->aCol); @@ -101966,10 +94372,16 @@ SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ ** db parameter can be used with db->pnBytesFreed to measure the memory ** used by the Table object. */ -static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ +SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ Index *pIndex, *pNext; TESTONLY( int nLookaside; ) /* Used to verify lookaside not used for schema */ + assert( !pTable || pTable->nRef>0 ); + + /* Do not delete the table until the reference count reaches zero. */ + if( !pTable ) return; + if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; + /* Record the number of outstanding lookaside allocations in schema Tables ** prior to doing any free() operations. Since schema Tables do not use ** lookaside, this number should not change. */ @@ -101979,9 +94391,8 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ pNext = pIndex->pNext; - assert( pIndex->pSchema==pTable->pSchema - || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); - if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){ + assert( pIndex->pSchema==pTable->pSchema ); + if( !db || db->pnBytesFreed==0 ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( &pIndex->pSchema->idxHash, zName, 0 @@ -101997,11 +94408,13 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ /* Delete the Table structure itself. */ - sqlite3DeleteColumnNames(db, pTable); + sqliteDeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); +#ifndef SQLITE_OMIT_CHECK sqlite3ExprListDelete(db, pTable->pCheck); +#endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); #endif @@ -102010,13 +94423,6 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ /* Verify that no lookaside memory was used by schema tables */ assert( nLookaside==0 || nLookaside==db->lookaside.nOut ); } -SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ - /* Do not delete the table until the reference count reaches zero. */ - if( !pTable ) return; - if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; - deleteTable(db, pTable); -} - /* ** Unlink the given table from the hash tables and the delete the @@ -102084,8 +94490,12 @@ SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *db, const char *zName){ int i = -1; /* Database number */ if( zName ){ Db *pDb; + int n = sqlite3Strlen30(zName); for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ - if( 0==sqlite3StrICmp(pDb->zDbSName, zName) ) break; + if( (!OMIT_TEMPDB || i!=1 ) && n==sqlite3Strlen30(pDb->zName) && + 0==sqlite3StrICmp(pDb->zName, zName) ){ + break; + } } } return i; @@ -102131,8 +94541,7 @@ SQLITE_PRIVATE int sqlite3TwoPartName( int iDb; /* Database holding the object */ sqlite3 *db = pParse->db; - assert( pName2!=0 ); - if( pName2->n>0 ){ + if( ALWAYS(pName2!=0) && pName2->n>0 ){ if( db->init.busy ) { sqlite3ErrorMsg(pParse, "corrupt database"); return -1; @@ -102144,7 +94553,7 @@ SQLITE_PRIVATE int sqlite3TwoPartName( return -1; } }else{ - assert( db->init.iDb==0 || db->init.busy || (db->flags & SQLITE_Vacuum)!=0); + assert( db->init.iDb==0 || db->init.busy ); iDb = db->init.iDb; *pUnqual = pName1; } @@ -102221,46 +94630,62 @@ SQLITE_PRIVATE void sqlite3StartTable( int iDb; /* Database number to create the table in */ Token *pName; /* Unqualified name of the table to create */ - if( db->init.busy && db->init.newTnum==1 ){ - /* Special case: Parsing the sqlite_master or sqlite_temp_master schema */ - iDb = db->init.iDb; - zName = sqlite3DbStrDup(db, SCHEMA_TABLE(iDb)); - pName = pName1; - }else{ - /* The common case */ - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); - if( iDb<0 ) return; - if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ - /* If creating a temp table, the name may not be qualified. Unless - ** the database name is "temp" anyway. */ - sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); - return; - } - if( !OMIT_TEMPDB && isTemp ) iDb = 1; - zName = sqlite3NameFromToken(db, pName); + /* The table or view name to create is passed to this routine via tokens + ** pName1 and pName2. If the table name was fully qualified, for example: + ** + ** CREATE TABLE xxx.yyy (...); + ** + ** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if + ** the table name is not fully qualified, i.e.: + ** + ** CREATE TABLE yyy(...); + ** + ** Then pName1 is set to "yyy" and pName2 is "". + ** + ** The call below sets the pName pointer to point at the token (pName1 or + ** pName2) that stores the unqualified table name. The variable iDb is + ** set to the index of the database that the table or view is to be + ** created in. + */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) return; + if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ + /* If creating a temp table, the name may not be qualified. Unless + ** the database name is "temp" anyway. */ + sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); + return; } + if( !OMIT_TEMPDB && isTemp ) iDb = 1; + pParse->sNameToken = *pName; + zName = sqlite3NameFromToken(db, pName); if( zName==0 ) return; if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto begin_table_error; } if( db->init.iDb==1 ) isTemp = 1; #ifndef SQLITE_OMIT_AUTHORIZATION - assert( isTemp==0 || isTemp==1 ); - assert( isView==0 || isView==1 ); + assert( (isTemp & 1)==isTemp ); { - static const u8 aCode[] = { - SQLITE_CREATE_TABLE, - SQLITE_CREATE_TEMP_TABLE, - SQLITE_CREATE_VIEW, - SQLITE_CREATE_TEMP_VIEW - }; - char *zDb = db->aDb[iDb].zDbSName; + int code; + char *zDb = db->aDb[iDb].zName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ goto begin_table_error; } - if( !isVirtual && sqlite3AuthCheck(pParse, (int)aCode[isTemp+2*isView], - zName, 0, zDb) ){ + if( isView ){ + if( !OMIT_TEMPDB && isTemp ){ + code = SQLITE_CREATE_TEMP_VIEW; + }else{ + code = SQLITE_CREATE_VIEW; + } + }else{ + if( !OMIT_TEMPDB && isTemp ){ + code = SQLITE_CREATE_TEMP_TABLE; + }else{ + code = SQLITE_CREATE_TABLE; + } + } + if( !isVirtual && sqlite3AuthCheck(pParse, code, zName, 0, zDb) ){ goto begin_table_error; } } @@ -102274,7 +94699,7 @@ SQLITE_PRIVATE void sqlite3StartTable( ** collisions. */ if( !IN_DECLARE_VTAB ){ - char *zDb = db->aDb[iDb].zDbSName; + char *zDb = db->aDb[iDb].zName; if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto begin_table_error; } @@ -102296,8 +94721,8 @@ SQLITE_PRIVATE void sqlite3StartTable( pTable = sqlite3DbMallocZero(db, sizeof(Table)); if( pTable==0 ){ - assert( db->mallocFailed ); - pParse->rc = SQLITE_NOMEM_BKPT; + db->mallocFailed = 1; + pParse->rc = SQLITE_NOMEM; pParse->nErr++; goto begin_table_error; } @@ -102329,12 +94754,10 @@ SQLITE_PRIVATE void sqlite3StartTable( ** now. */ if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ - int addr1; + int j1; int fileFormat; int reg1, reg2, reg3; - /* nullRow[] is an OP_Record encoding of a row containing 5 NULLs */ - static const char nullRow[] = { 6, 0, 0, 0, 0, 0 }; - sqlite3BeginWriteOperation(pParse, 1, iDb); + sqlite3BeginWriteOperation(pParse, 0, iDb); #ifndef SQLITE_OMIT_VIRTUALTABLE if( isVirtual ){ @@ -102350,12 +94773,14 @@ SQLITE_PRIVATE void sqlite3StartTable( reg3 = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); - addr1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); + j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? 1 : SQLITE_MAX_FILE_FORMAT; - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, fileFormat); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, ENC(db)); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, reg3); + sqlite3VdbeAddOp2(v, OP_Integer, ENC(db), reg3); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, reg3); + sqlite3VdbeJumpHere(v, j1); /* This just creates a place-holder record in the sqlite_master table. ** The record created does not contain anything yet. It will be replaced @@ -102376,7 +94801,7 @@ SQLITE_PRIVATE void sqlite3StartTable( } sqlite3OpenMasterTable(pParse, iDb); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); - sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC); + sqlite3VdbeAddOp2(v, OP_Null, 0, reg3); sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeAddOp0(v, OP_Close); @@ -102391,19 +94816,18 @@ SQLITE_PRIVATE void sqlite3StartTable( return; } -/* Set properties of a table column based on the (magical) -** name of the column. +/* +** This macro is used to compare two strings in a case-insensitive manner. +** It is slightly faster than calling sqlite3StrICmp() directly, but +** produces larger code. +** +** WARNING: This macro is not compatible with the strcmp() family. It +** returns true if the two strings are equal, otherwise false. */ -#if SQLITE_ENABLE_HIDDEN_COLUMNS -SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ - if( sqlite3_strnicmp(pCol->zName, "__hidden__", 10)==0 ){ - pCol->colFlags |= COLFLAG_HIDDEN; - }else if( pTab && pCol!=pTab->aCol && (pCol[-1].colFlags & COLFLAG_HIDDEN) ){ - pTab->tabFlags |= TF_OOOHidden; - } -} -#endif - +#define STRICMP(x, y) (\ +sqlite3UpperToLower[*(unsigned char *)(x)]== \ +sqlite3UpperToLower[*(unsigned char *)(y)] \ +&& sqlite3StrICmp((x)+1,(y)+1)==0 ) /* ** Add a new column to the table currently being constructed. @@ -102413,11 +94837,10 @@ SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ ** first to get things going. Then this routine is called for each ** column. */ -SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ +SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ Table *p; int i; char *z; - char *zType; Column *pCol; sqlite3 *db = pParse->db; if( (p = pParse->pNewTable)==0 ) return; @@ -102427,13 +94850,10 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ return; } #endif - z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2); + z = sqlite3NameFromToken(db, pName); if( z==0 ) return; - memcpy(z, pName->z, pName->n); - z[pName->n] = 0; - sqlite3Dequote(z); for(i=0; inCol; i++){ - if( sqlite3_stricmp(z, p->aCol[i].zName)==0 ){ + if( STRICMP(z, p->aCol[i].zName) ){ sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); sqlite3DbFree(db, z); return; @@ -102451,23 +94871,14 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ pCol = &p->aCol[p->nCol]; memset(pCol, 0, sizeof(p->aCol[0])); pCol->zName = z; - sqlite3ColumnPropertiesFromName(p, pCol); - if( pType->n==0 ){ - /* If there is no type specified, columns have the default affinity - ** 'BLOB'. */ - pCol->affinity = SQLITE_AFF_BLOB; - pCol->szEst = 1; - }else{ - zType = z + sqlite3Strlen30(z) + 1; - memcpy(zType, pType->z, pType->n); - zType[pType->n] = 0; - sqlite3Dequote(zType); - pCol->affinity = sqlite3AffinityType(zType, &pCol->szEst); - pCol->colFlags |= COLFLAG_HASTYPE; - } + /* If there is no type specified, columns have the default affinity + ** 'NONE'. If there is a type specified, then sqlite3AddColumnType() will + ** be called next to set pCol->affinity correctly. + */ + pCol->affinity = SQLITE_AFF_NONE; + pCol->szEst = 1; p->nCol++; - pParse->constraintName.n = 0; } /* @@ -102500,7 +94911,7 @@ SQLITE_PRIVATE void sqlite3AddNotNull(Parse *pParse, int onError){ ** 'CHAR' | SQLITE_AFF_TEXT ** 'CLOB' | SQLITE_AFF_TEXT ** 'TEXT' | SQLITE_AFF_TEXT -** 'BLOB' | SQLITE_AFF_BLOB +** 'BLOB' | SQLITE_AFF_NONE ** 'REAL' | SQLITE_AFF_REAL ** 'FLOA' | SQLITE_AFF_REAL ** 'DOUB' | SQLITE_AFF_REAL @@ -102513,7 +94924,7 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ char aff = SQLITE_AFF_NUMERIC; const char *zChar = 0; - assert( zIn!=0 ); + if( zIn==0 ) return aff; while( zIn[0] ){ h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; zIn++; @@ -102526,7 +94937,7 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ aff = SQLITE_AFF_TEXT; }else if( h==(('b'<<24)+('l'<<16)+('o'<<8)+'b') /* BLOB */ && (aff==SQLITE_AFF_NUMERIC || aff==SQLITE_AFF_REAL) ){ - aff = SQLITE_AFF_BLOB; + aff = SQLITE_AFF_NONE; if( zIn[0]=='(' ) zChar = zIn; #ifndef SQLITE_OMIT_FLOATING_POINT }else if( h==(('r'<<24)+('e'<<16)+('a'<<8)+'l') /* REAL */ @@ -102570,6 +94981,28 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ return aff; } +/* +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. The pFirst token is the first +** token in the sequence of tokens that describe the type of the +** column currently under construction. pLast is the last token +** in the sequence. Use this information to construct a string +** that contains the typename of the column and store that string +** in zType. +*/ +SQLITE_PRIVATE void sqlite3AddColumnType(Parse *pParse, Token *pType){ + Table *p; + Column *pCol; + + p = pParse->pNewTable; + if( p==0 || NEVER(p->nCol<1) ) return; + pCol = &p->aCol[p->nCol-1]; + assert( pCol->zType==0 || CORRUPT_DB ); + sqlite3DbFree(pParse->db, pCol->zType); + pCol->zType = sqlite3NameFromToken(pParse->db, pType); + pCol->affinity = sqlite3AffinityType(pCol->zType, &pCol->szEst); +} + /* ** The expression is the default value for the most recently added column ** of the table currently under construction. @@ -102595,45 +95028,16 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ ** tokens that point to volatile memory. The 'span' of the expression ** is required by pragma table_info. */ - Expr x; sqlite3ExprDelete(db, pCol->pDflt); - memset(&x, 0, sizeof(x)); - x.op = TK_SPAN; - x.u.zToken = sqlite3DbStrNDup(db, (char*)pSpan->zStart, - (int)(pSpan->zEnd - pSpan->zStart)); - x.pLeft = pSpan->pExpr; - x.flags = EP_Skip; - pCol->pDflt = sqlite3ExprDup(db, &x, EXPRDUP_REDUCE); - sqlite3DbFree(db, x.u.zToken); + pCol->pDflt = sqlite3ExprDup(db, pSpan->pExpr, EXPRDUP_REDUCE); + sqlite3DbFree(db, pCol->zDflt); + pCol->zDflt = sqlite3DbStrNDup(db, (char*)pSpan->zStart, + (int)(pSpan->zEnd - pSpan->zStart)); } } sqlite3ExprDelete(db, pSpan->pExpr); } -/* -** Backwards Compatibility Hack: -** -** Historical versions of SQLite accepted strings as column names in -** indexes and PRIMARY KEY constraints and in UNIQUE constraints. Example: -** -** CREATE TABLE xyz(a,b,c,d,e,PRIMARY KEY('a'),UNIQUE('b','c' COLLATE trim) -** CREATE INDEX abc ON xyz('c','d' DESC,'e' COLLATE nocase DESC); -** -** This is goofy. But to preserve backwards compatibility we continue to -** accept it. This routine does the necessary conversion. It converts -** the expression given in its argument from a TK_STRING into a TK_ID -** if the expression is just a TK_STRING with an optional COLLATE clause. -** If the epxression is anything other than TK_STRING, the expression is -** unchanged. -*/ -static void sqlite3StringToId(Expr *p){ - if( p->op==TK_STRING ){ - p->op = TK_ID; - }else if( p->op==TK_COLLATE && p->pLeft->op==TK_STRING ){ - p->pLeft->op = TK_ID; - } -} - /* ** Designate the PRIMARY KEY for the table. pList is a list of names ** of columns that form the primary key. If pList is NULL, then the @@ -102660,10 +95064,10 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( int sortOrder /* SQLITE_SO_ASC or SQLITE_SO_DESC */ ){ Table *pTab = pParse->pNewTable; - Column *pCol = 0; + char *zType = 0; int iCol = -1, i; int nTerm; - if( pTab==0 ) goto primary_key_exit; + if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; if( pTab->tabFlags & TF_HasPrimaryKey ){ sqlite3ErrorMsg(pParse, "table \"%s\" has more than one primary key", pTab->zName); @@ -102672,31 +95076,24 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( pTab->tabFlags |= TF_HasPrimaryKey; if( pList==0 ){ iCol = pTab->nCol - 1; - pCol = &pTab->aCol[iCol]; - pCol->colFlags |= COLFLAG_PRIMKEY; + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; + zType = pTab->aCol[iCol].zType; nTerm = 1; }else{ nTerm = pList->nExpr; for(i=0; ia[i].pExpr); - assert( pCExpr!=0 ); - sqlite3StringToId(pCExpr); - if( pCExpr->op==TK_ID ){ - const char *zCName = pCExpr->u.zToken; - for(iCol=0; iColnCol; iCol++){ - if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ - pCol = &pTab->aCol[iCol]; - pCol->colFlags |= COLFLAG_PRIMKEY; - break; - } + for(iCol=0; iColnCol; iCol++){ + if( sqlite3StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; + zType = pTab->aCol[iCol].zType; + break; } } } } if( nTerm==1 - && pCol - && sqlite3StrICmp(sqlite3ColumnType(pCol,""), "INTEGER")==0 - && sortOrder!=SQLITE_SO_DESC + && zType && sqlite3StrICmp(zType, "INTEGER")==0 + && sortOrder==SQLITE_SO_ASC ){ pTab->iPKey = iCol; pTab->keyConf = (u8)onError; @@ -102709,8 +95106,15 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( "INTEGER PRIMARY KEY"); #endif }else{ - sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, - 0, sortOrder, 0, SQLITE_IDXTYPE_PRIMARYKEY); + Vdbe *v = pParse->pVdbe; + Index *p; + if( v ) pParse->addrSkipPK = sqlite3VdbeAddOp0(v, OP_Noop); + p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, + 0, sortOrder, 0); + if( p ){ + p->idxType = SQLITE_IDXTYPE_PRIMARYKEY; + if( v ) sqlite3VdbeJumpHere(v, pParse->addrSkipPK); + } pList = 0; } @@ -102829,16 +95233,15 @@ SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ ** set back to prior value. But schema changes are infrequent ** and the probability of hitting the same cookie value is only ** 1 chance in 2^32. So we're safe enough. -** -** IMPLEMENTATION-OF: R-34230-56049 SQLite automatically increments -** the schema-version whenever the schema changes. */ SQLITE_PRIVATE void sqlite3ChangeCookie(Parse *pParse, int iDb){ + int r1 = sqlite3GetTempReg(pParse); sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, - db->aDb[iDb].pSchema->schema_cookie+1); + sqlite3VdbeAddOp2(v, OP_Integer, db->aDb[iDb].pSchema->schema_cookie+1, r1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, r1); + sqlite3ReleaseTempReg(pParse, r1); } /* @@ -102920,7 +95323,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ n += 35 + 6*p->nCol; zStmt = sqlite3DbMallocRaw(0, n); if( zStmt==0 ){ - sqlite3OomFault(db); + db->mallocFailed = 1; return 0; } sqlite3_snprintf(n, zStmt, "CREATE TABLE "); @@ -102929,7 +95332,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ zStmt[k++] = '('; for(pCol=p->aCol, i=0; inCol; i++, pCol++){ static const char * const azType[] = { - /* SQLITE_AFF_BLOB */ "", + /* SQLITE_AFF_NONE */ "", /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NUMERIC */ " NUM", /* SQLITE_AFF_INTEGER */ " INT", @@ -102942,17 +95345,17 @@ static char *createTableStmt(sqlite3 *db, Table *p){ k += sqlite3Strlen30(&zStmt[k]); zSep = zSep2; identPut(zStmt, &k, pCol->zName); - assert( pCol->affinity-SQLITE_AFF_BLOB >= 0 ); - assert( pCol->affinity-SQLITE_AFF_BLOB < ArraySize(azType) ); - testcase( pCol->affinity==SQLITE_AFF_BLOB ); + assert( pCol->affinity-SQLITE_AFF_NONE >= 0 ); + assert( pCol->affinity-SQLITE_AFF_NONE < ArraySize(azType) ); + testcase( pCol->affinity==SQLITE_AFF_NONE ); testcase( pCol->affinity==SQLITE_AFF_TEXT ); testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); - zType = azType[pCol->affinity - SQLITE_AFF_BLOB]; + zType = azType[pCol->affinity - SQLITE_AFF_NONE]; len = sqlite3Strlen30(zType); - assert( pCol->affinity==SQLITE_AFF_BLOB + assert( pCol->affinity==SQLITE_AFF_NONE || pCol->affinity==sqlite3AffinityType(zType, 0) ); memcpy(&zStmt[k], zType, len); k += len; @@ -102973,9 +95376,9 @@ static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ assert( pIdx->isResized==0 ); nByte = (sizeof(char*) + sizeof(i16) + 1)*N; zExtra = sqlite3DbMallocZero(db, nByte); - if( zExtra==0 ) return SQLITE_NOMEM_BKPT; + if( zExtra==0 ) return SQLITE_NOMEM; memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); - pIdx->azColl = (const char**)zExtra; + pIdx->azColl = (char**)zExtra; zExtra += sizeof(char*)*N; memcpy(zExtra, pIdx->aiColumn, sizeof(i16)*pIdx->nColumn); pIdx->aiColumn = (i16*)zExtra; @@ -103030,23 +95433,21 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){ ** are appropriate for a WITHOUT ROWID table instead of a rowid table. ** Changes include: ** -** (1) Set all columns of the PRIMARY KEY schema object to be NOT NULL. -** (2) Convert the OP_CreateTable into an OP_CreateIndex. There is +** (1) Convert the OP_CreateTable into an OP_CreateIndex. There is ** no rowid btree for a WITHOUT ROWID. Instead, the canonical ** data storage is a covering index btree. -** (3) Bypass the creation of the sqlite_master table entry +** (2) Bypass the creation of the sqlite_master table entry ** for the PRIMARY KEY as the primary key index is now ** identified by the sqlite_master table entry of the table itself. -** (4) Set the Index.tnum of the PRIMARY KEY Index object in the +** (3) Set the Index.tnum of the PRIMARY KEY Index object in the ** schema to the rootpage from the main table. +** (4) Set all columns of the PRIMARY KEY schema object to be NOT NULL. ** (5) Add all table columns to the PRIMARY KEY Index object ** so that the PRIMARY KEY is a covering index. The surplus ** columns are part of KeyInfo.nXField and are not used for ** sorting or lookup or uniqueness checks. ** (6) Replace the rowid tail on all automatically generated UNIQUE ** indices with the PRIMARY KEY columns. -** -** For virtual tables, only (1) is performed. */ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ Index *pIdx; @@ -103056,27 +95457,21 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; - /* Mark every PRIMARY KEY column as NOT NULL (except for imposter tables) - */ - if( !db->init.imposterTable ){ - for(i=0; inCol; i++){ - if( (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 ){ - pTab->aCol[i].notNull = OE_Abort; - } - } - } - - /* The remaining transformations only apply to b-tree tables, not to - ** virtual tables */ - if( IN_DECLARE_VTAB ) return; - /* Convert the OP_CreateTable opcode that would normally create the ** root-page for the table into an OP_CreateIndex opcode. The index ** created will become the PRIMARY KEY index. */ if( pParse->addrCrTab ){ assert( v ); - sqlite3VdbeChangeOpcode(v, pParse->addrCrTab, OP_CreateIndex); + sqlite3VdbeGetOp(v, pParse->addrCrTab)->opcode = OP_CreateIndex; + } + + /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master + ** table entry. + */ + if( pParse->addrSkipPK ){ + assert( v ); + sqlite3VdbeGetOp(v, pParse->addrSkipPK)->opcode = OP_Goto; } /* Locate the PRIMARY KEY index. Or, if this table was originally @@ -103084,30 +95479,18 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ */ if( pTab->iPKey>=0 ){ ExprList *pList; - Token ipkToken; - sqlite3TokenInit(&ipkToken, pTab->aCol[pTab->iPKey].zName); - pList = sqlite3ExprListAppend(pParse, 0, - sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); + pList = sqlite3ExprListAppend(pParse, 0, 0); if( pList==0 ) return; + pList->a[0].zName = sqlite3DbStrDup(pParse->db, + pTab->aCol[pTab->iPKey].zName); pList->a[0].sortOrder = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); - sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0, - SQLITE_IDXTYPE_PRIMARYKEY); - if( db->mallocFailed ) return; - pPk = sqlite3PrimaryKeyIndex(pTab); + pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); + if( pPk==0 ) return; + pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY; pTab->iPKey = -1; }else{ pPk = sqlite3PrimaryKeyIndex(pTab); - - /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master - ** table entry. This is only required if currently generating VDBE - ** code for a CREATE TABLE (not when parsing one as part of reading - ** a database schema). */ - if( v ){ - assert( db->init.busy==0 ); - sqlite3VdbeChangeOpcode(v, pPk->tnum, OP_Goto); - } - /* ** Remove all redundant columns from the PRIMARY KEY. For example, change ** "PRIMARY KEY(a,b,a,b,c,b,c,d)" into just "PRIMARY KEY(a,b,c,d)". Later @@ -103122,11 +95505,19 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ } pPk->nKeyCol = j; } - assert( pPk!=0 ); pPk->isCovering = 1; - if( !db->init.imposterTable ) pPk->uniqNotNull = 1; + assert( pPk!=0 ); nPk = pPk->nKeyCol; + /* Make sure every column of the PRIMARY KEY is NOT NULL. (Except, + ** do not enforce this for imposter tables.) */ + if( !db->init.imposterTable ){ + for(i=0; iaCol[pPk->aiColumn[i]].notNull = 1; + } + pPk->uniqNotNull = 1; + } + /* The root page of the PRIMARY KEY is the table root page */ pPk->tnum = pTab->tnum; @@ -103164,7 +95555,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( !hasColumn(pPk->aiColumn, j, i) ){ assert( jnColumn ); pPk->aiColumn[j] = i; - pPk->azColl[j] = sqlite3StrBINARY; + pPk->azColl[j] = "BINARY"; j++; } } @@ -103207,10 +95598,9 @@ SQLITE_PRIVATE void sqlite3EndTable( int iDb; /* Database in which the table lives */ Index *pIdx; /* An implied index of the table */ - if( pEnd==0 && pSelect==0 ){ + if( (pEnd==0 && pSelect==0) || db->mallocFailed ){ return; } - assert( !db->mallocFailed ); p = pParse->pNewTable; if( p==0 ) return; @@ -103221,13 +95611,9 @@ SQLITE_PRIVATE void sqlite3EndTable( ** So do not write to the disk again. Extract the root page number ** for the table from the db->init.newTnum field. (The page number ** should have been put there by the sqliteOpenCb routine.) - ** - ** If the root page number is 1, that means this is the sqlite_master - ** table itself. So mark it read-only. */ if( db->init.busy ){ p->tnum = db->init.newTnum; - if( p->tnum==1 ) p->tabFlags |= TF_Readonly; } /* Special processing for WITHOUT ROWID Tables */ @@ -103240,7 +95626,7 @@ SQLITE_PRIVATE void sqlite3EndTable( if( (p->tabFlags & TF_HasPrimaryKey)==0 ){ sqlite3ErrorMsg(pParse, "PRIMARY KEY missing on table %s", p->zName); }else{ - p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; + p->tabFlags |= TF_WithoutRowid; convertToWithoutRowidTable(pParse, p); } } @@ -103308,46 +95694,26 @@ SQLITE_PRIVATE void sqlite3EndTable( ** be redundant. */ if( pSelect ){ - SelectDest dest; /* Where the SELECT should store results */ - int regYield; /* Register holding co-routine entry-point */ - int addrTop; /* Top of the co-routine */ - int regRec; /* A record to be insert into the new table */ - int regRowid; /* Rowid of the next row to insert */ - int addrInsLoop; /* Top of the loop for inserting rows */ - Table *pSelTab; /* A table that describes the SELECT results */ - - regYield = ++pParse->nMem; - regRec = ++pParse->nMem; - regRowid = ++pParse->nMem; + SelectDest dest; + Table *pSelTab; + assert(pParse->nTab==1); - sqlite3MayAbort(pParse); sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); pParse->nTab = 2; - addrTop = sqlite3VdbeCurrentAddr(v) + 1; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); - sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + sqlite3SelectDestInit(&dest, SRT_Table, 1); sqlite3Select(pParse, pSelect, &dest); - sqlite3VdbeEndCoroutine(v, regYield); - sqlite3VdbeJumpHere(v, addrTop - 1); - if( pParse->nErr ) return; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); - if( pSelTab==0 ) return; - assert( p->aCol==0 ); - p->nCol = pSelTab->nCol; - p->aCol = pSelTab->aCol; - pSelTab->nCol = 0; - pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); - addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); - sqlite3TableAffinity(v, p, 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); - sqlite3VdbeGoto(v, addrInsLoop); - sqlite3VdbeJumpHere(v, addrInsLoop); sqlite3VdbeAddOp1(v, OP_Close, 1); + if( pParse->nErr==0 ){ + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); + if( pSelTab==0 ) return; + assert( p->aCol==0 ); + p->nCol = pSelTab->nCol; + p->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + } } /* Compute the complete text of the CREATE statement */ @@ -103370,7 +95736,7 @@ SQLITE_PRIVATE void sqlite3EndTable( "UPDATE %Q.%s " "SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q " "WHERE rowid=#%d", - db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zType, p->zName, p->zName, @@ -103385,13 +95751,13 @@ SQLITE_PRIVATE void sqlite3EndTable( /* Check to see if we need to create an sqlite_sequence table for ** keeping track of autoincrement keys. */ - if( (p->tabFlags & TF_Autoincrement)!=0 ){ + if( p->tabFlags & TF_Autoincrement ){ Db *pDb = &db->aDb[iDb]; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( pDb->pSchema->pSeqTab==0 ){ sqlite3NestedParse(pParse, "CREATE TABLE %Q.sqlite_sequence(name,seq)", - pDb->zDbSName + pDb->zName ); } } @@ -103412,7 +95778,7 @@ SQLITE_PRIVATE void sqlite3EndTable( pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, p); if( pOld ){ assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ - sqlite3OomFault(db); + db->mallocFailed = 1; return; } pParse->pNewTable = 0; @@ -103442,7 +95808,6 @@ SQLITE_PRIVATE void sqlite3CreateView( Token *pBegin, /* The CREATE token that begins the statement */ Token *pName1, /* The token that holds the name of the view */ Token *pName2, /* The token that holds the name of the view */ - ExprList *pCNames, /* Optional list of view column names */ Select *pSelect, /* A SELECT statement that will become the new view */ int isTemp, /* TRUE for a TEMPORARY view */ int noErr /* Suppress error messages if VIEW already exists */ @@ -103458,15 +95823,22 @@ SQLITE_PRIVATE void sqlite3CreateView( if( pParse->nVar>0 ){ sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); - goto create_view_fail; + sqlite3SelectDelete(db, pSelect); + return; } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); p = pParse->pNewTable; - if( p==0 || pParse->nErr ) goto create_view_fail; + if( p==0 || pParse->nErr ){ + sqlite3SelectDelete(db, pSelect); + return; + } sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); - if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; + if( sqlite3FixSelect(&sFix, pSelect) ){ + sqlite3SelectDelete(db, pSelect); + return; + } /* Make a copy of the entire SELECT statement that defines the view. ** This will force all the Expr.token.z values to be dynamically @@ -103474,31 +95846,30 @@ SQLITE_PRIVATE void sqlite3CreateView( ** they will persist after the current sqlite3_exec() call returns. */ p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); - p->pCheck = sqlite3ExprListDup(db, pCNames, EXPRDUP_REDUCE); - if( db->mallocFailed ) goto create_view_fail; + sqlite3SelectDelete(db, pSelect); + if( db->mallocFailed ){ + return; + } + if( !db->init.busy ){ + sqlite3ViewGetColumnNames(pParse, p); + } /* Locate the end of the CREATE VIEW statement. Make sEnd point to ** the end. */ sEnd = pParse->sLastToken; - assert( sEnd.z[0]!=0 ); - if( sEnd.z[0]!=';' ){ + if( ALWAYS(sEnd.z[0]!=0) && sEnd.z[0]!=';' ){ sEnd.z += sEnd.n; } sEnd.n = 0; n = (int)(sEnd.z - pBegin->z); - assert( n>0 ); z = pBegin->z; - while( sqlite3Isspace(z[n-1]) ){ n--; } + while( ALWAYS(n>0) && sqlite3Isspace(z[n-1]) ){ n--; } sEnd.z = &z[n-1]; sEnd.n = 1; /* Use sqlite3EndTable() to add the view to the SQLITE_MASTER table */ sqlite3EndTable(pParse, 0, &sEnd, 0, 0); - -create_view_fail: - sqlite3SelectDelete(db, pSelect); - sqlite3ExprListDelete(db, pCNames); return; } #endif /* SQLITE_OMIT_VIEW */ @@ -103515,9 +95886,7 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ int nErr = 0; /* Number of errors encountered */ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ -#ifndef SQLITE_OMIT_AUTHORIZATION sqlite3_xauth xAuth; /* Saved xAuth pointer */ -#endif assert( pTable ); @@ -103565,10 +95934,11 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ assert( pTable->pSelect ); pSel = sqlite3SelectDup(db, pTable->pSelect, 0); if( pSel ){ + u8 enableLookaside = db->lookaside.bEnabled; n = pParse->nTab; sqlite3SrcListAssignCursors(pParse, pSel->pSrc); pTable->nCol = -1; - db->lookaside.bDisable++; + db->lookaside.bEnabled = 0; #ifndef SQLITE_OMIT_AUTHORIZATION xAuth = db->xAuth; db->xAuth = 0; @@ -103577,43 +95947,25 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #else pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); #endif + db->lookaside.bEnabled = enableLookaside; pParse->nTab = n; - if( pTable->pCheck ){ - /* CREATE VIEW name(arglist) AS ... - ** The names of the columns in the table are taken from - ** arglist which is stored in pTable->pCheck. The pCheck field - ** normally holds CHECK constraints on an ordinary table, but for - ** a VIEW it holds the list of column names. - */ - sqlite3ColumnsFromExprList(pParse, pTable->pCheck, - &pTable->nCol, &pTable->aCol); - if( db->mallocFailed==0 - && pParse->nErr==0 - && pTable->nCol==pSel->pEList->nExpr - ){ - sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel); - } - }else if( pSelTab ){ - /* CREATE VIEW name AS... without an argument list. Construct - ** the column names from the SELECT statement that defines the view. - */ + if( pSelTab ){ assert( pTable->aCol==0 ); pTable->nCol = pSelTab->nCol; pTable->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); + pTable->pSchema->schemaFlags |= DB_UnresetViews; }else{ pTable->nCol = 0; nErr++; } - sqlite3DeleteTable(db, pSelTab); sqlite3SelectDelete(db, pSel); - db->lookaside.bDisable--; } else { nErr++; } - pTable->pSchema->schemaFlags |= DB_UnresetViews; #endif /* SQLITE_OMIT_VIEW */ return nErr; } @@ -103630,7 +95982,7 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); if( pTab->pSelect ){ - sqlite3DeleteColumnNames(db, pTab); + sqliteDeleteColumnNames(db, pTab); pTab->aCol = 0; pTab->nCol = 0; } @@ -103692,7 +96044,6 @@ SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3 *db, int iDb, int iFrom, int iT static void destroyRootPage(Parse *pParse, int iTable, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); int r1 = sqlite3GetTempReg(pParse); - assert( iTable>1 ); sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); sqlite3MayAbort(pParse); #ifndef SQLITE_OMIT_AUTOVACUUM @@ -103707,7 +96058,7 @@ static void destroyRootPage(Parse *pParse, int iTable, int iDb){ */ sqlite3NestedParse(pParse, "UPDATE %Q.%s SET rootpage=%d WHERE #%d AND rootpage=#%d", - pParse->db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), iTable, r1, r1); + pParse->db->aDb[iDb].zName, SCHEMA_TABLE(iDb), iTable, r1, r1); #endif sqlite3ReleaseTempReg(pParse, r1); } @@ -103783,7 +96134,7 @@ static void sqlite3ClearStatTables( const char *zName /* Name of index or table */ ){ int i; - const char *zDbName = pParse->db->aDb[iDb].zDbSName; + const char *zDbName = pParse->db->aDb[iDb].zName; for(i=1; i<=4; i++){ char zTab[24]; sqlite3_snprintf(sizeof(zTab),zTab,"sqlite_stat%d",i); @@ -103836,7 +96187,7 @@ SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, in if( pTab->tabFlags & TF_Autoincrement ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.sqlite_sequence WHERE name=%Q", - pDb->zDbSName, pTab->zName + pDb->zName, pTab->zName ); } #endif @@ -103850,7 +96201,7 @@ SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, in */ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", - pDb->zDbSName, SCHEMA_TABLE(iDb), pTab->zName); + pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); if( !isView && !IsVirtual(pTab) ){ destroyTable(pParse, pTab); } @@ -103883,7 +96234,6 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, assert( pName->nSrc==1 ); if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; if( noErr ) db->suppressErr++; - assert( isView==0 || isView==LOCATE_VIEW ); pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); if( noErr ) db->suppressErr--; @@ -103904,7 +96254,7 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, { int code; const char *zTab = SCHEMA_TABLE(iDb); - const char *zDb = db->aDb[iDb].zDbSName; + const char *zDb = db->aDb[iDb].zName; const char *zArg2 = 0; if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb)){ goto exit_drop_table; @@ -104079,7 +96429,7 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( pFKey->zTo, (void *)pFKey ); if( pNextTo==pFKey ){ - sqlite3OomFault(db); + db->mallocFailed = 1; goto fk_end; } if( pNextTo ){ @@ -104145,7 +96495,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, - db->aDb[iDb].zDbSName ) ){ + db->aDb[iDb].zName ) ){ return; } #endif @@ -104161,7 +96511,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ tnum = pIndex->tnum; } pKey = sqlite3KeyInfoOfIndex(pParse, pIndex); - assert( pKey!=0 || db->mallocFailed || pParse->nErr ); /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; @@ -104185,9 +96534,10 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); - if( IsUniqueIndex(pIndex) ){ + assert( pKey!=0 || db->mallocFailed || pParse->nErr ); + if( IsUniqueIndex(pIndex) && pKey!=0 ){ int j2 = sqlite3VdbeCurrentAddr(v) + 3; - sqlite3VdbeGoto(v, j2); + sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, pIndex->nKeyCol); VdbeCoverage(v); @@ -104232,7 +96582,7 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( p = sqlite3DbMallocZero(db, nByte + nExtra); if( p ){ char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); - p->azColl = (const char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; @@ -104254,8 +96604,12 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( ** pList is a list of columns to be indexed. pList will be NULL if this ** is a primary key or unique-constraint on the most recent column added ** to the table currently under construction. +** +** If the index is created successfully, return a pointer to the new Index +** structure. This is used by sqlite3AddPrimaryKey() to mark the index +** as the tables primary key (Index.idxType==SQLITE_IDXTYPE_PRIMARYKEY) */ -SQLITE_PRIVATE void sqlite3CreateIndex( +SQLITE_PRIVATE Index *sqlite3CreateIndex( Parse *pParse, /* All information about this parse */ Token *pName1, /* First part of index name. May be NULL */ Token *pName2, /* Second part of index name. May be NULL */ @@ -104265,9 +96619,9 @@ SQLITE_PRIVATE void sqlite3CreateIndex( Token *pStart, /* The CREATE token that begins this statement */ Expr *pPIWhere, /* WHERE clause for partial indices */ int sortOrder, /* Sort order of primary key when pList==NULL */ - int ifNotExist, /* Omit error if index already exists */ - u8 idxType /* The index type */ + int ifNotExist /* Omit error if index already exists */ ){ + Index *pRet = 0; /* Pointer to return */ Table *pTab = 0; /* Table to be indexed */ Index *pIndex = 0; /* The index to be created */ char *zName = 0; /* Name of the index */ @@ -104280,15 +96634,13 @@ SQLITE_PRIVATE void sqlite3CreateIndex( int iDb; /* Index of the database that is being written */ Token *pName = 0; /* Unqualified name of the index to create */ struct ExprList_item *pListItem; /* For looping over pList */ + const Column *pTabCol; /* A column in the table */ int nExtra = 0; /* Space allocated for zExtra[] */ int nExtraCol; /* Number of extra columns needed */ char *zExtra = 0; /* Extra space after the Index object */ Index *pPk = 0; /* PRIMARY KEY index for WITHOUT ROWID tables */ - if( db->mallocFailed || pParse->nErr>0 ){ - goto exit_create_index; - } - if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){ + if( db->mallocFailed || IN_DECLARE_VTAB || pParse->nErr>0 ){ goto exit_create_index; } if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ @@ -104397,7 +96749,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( goto exit_create_index; } } - if( sqlite3FindIndex(db, zName, pDb->zDbSName)!=0 ){ + if( sqlite3FindIndex(db, zName, pDb->zName)!=0 ){ if( !ifNotExist ){ sqlite3ErrorMsg(pParse, "index %s already exists", zName); }else{ @@ -104414,20 +96766,13 @@ SQLITE_PRIVATE void sqlite3CreateIndex( if( zName==0 ){ goto exit_create_index; } - - /* Automatic index names generated from within sqlite3_declare_vtab() - ** must have names that are distinct from normal automatic index names. - ** The following statement converts "sqlite3_autoindex..." into - ** "sqlite3_butoindex..." in order to make the names distinct. - ** The "vtab_err.test" test demonstrates the need of this statement. */ - if( IN_DECLARE_VTAB ) zName[7]++; } /* Check for authorization to create an index. */ #ifndef SQLITE_OMIT_AUTHORIZATION { - const char *zDb = pDb->zDbSName; + const char *zDb = pDb->zName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ goto exit_create_index; } @@ -104444,15 +96789,11 @@ SQLITE_PRIVATE void sqlite3CreateIndex( ** So create a fake list to simulate this. */ if( pList==0 ){ - Token prevCol; - sqlite3TokenInit(&prevCol, pTab->aCol[pTab->nCol-1].zName); - pList = sqlite3ExprListAppend(pParse, 0, - sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); + pList = sqlite3ExprListAppend(pParse, 0, 0); if( pList==0 ) goto exit_create_index; - assert( pList->nExpr==1 ); - sqlite3ExprListSetSortOrder(pList, sortOrder); - }else{ - sqlite3ExprListCheckLength(pParse, pList, "index"); + pList->a[0].zName = sqlite3DbStrDup(pParse->db, + pTab->aCol[pTab->nCol-1].zName); + pList->a[0].sortOrder = (u8)sortOrder; } /* Figure out how many bytes of space are required to store explicitly @@ -104460,8 +96801,8 @@ SQLITE_PRIVATE void sqlite3CreateIndex( */ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - assert( pExpr!=0 ); - if( pExpr->op==TK_COLLATE ){ + if( pExpr ){ + assert( pExpr->op==TK_COLLATE ); nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -104484,7 +96825,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( pIndex->pTable = pTab; pIndex->onError = (u8)onError; pIndex->uniqNotNull = onError!=OE_None; - pIndex->idxType = idxType; + pIndex->idxType = pName ? SQLITE_IDXTYPE_APPDEF : SQLITE_IDXTYPE_UNIQUE; pIndex->pSchema = db->aDb[iDb].pSchema; pIndex->nKeyCol = pList->nExpr; if( pPIWhere ){ @@ -104502,54 +96843,35 @@ SQLITE_PRIVATE void sqlite3CreateIndex( sortOrderMask = 0; /* Ignore DESC */ } - /* Analyze the list of expressions that form the terms of the index and - ** report any errors. In the common case where the expression is exactly - ** a table column, store that column in aiColumn[]. For general expressions, - ** populate pIndex->aColExpr and store XN_EXPR (-2) in aiColumn[]. + /* Scan the names of the columns of the table to be indexed and + ** load the column indices into the Index structure. Report an error + ** if any column is not found. ** - ** TODO: Issue a warning if two or more columns of the index are identical. - ** TODO: Issue a warning if the table primary key is used as part of the - ** index key. + ** TODO: Add a test to make sure that the same column is not named + ** more than once within the same index. Only the first instance of + ** the column will ever be used by the optimizer. Note that using the + ** same column more than once cannot be an error because that would + ** break backwards compatibility - it needs to be a warning. */ for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ - Expr *pCExpr; /* The i-th index expression */ - int requestedSortOrder; /* ASC or DESC on the i-th expression */ - const char *zColl; /* Collation sequence name */ - - sqlite3StringToId(pListItem->pExpr); - sqlite3ResolveSelfReference(pParse, pTab, NC_IdxExpr, pListItem->pExpr, 0); - if( pParse->nErr ) goto exit_create_index; - pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); - if( pCExpr->op!=TK_COLUMN ){ - if( pTab==pParse->pNewTable ){ - sqlite3ErrorMsg(pParse, "expressions prohibited in PRIMARY KEY and " - "UNIQUE constraints"); - goto exit_create_index; - } - if( pIndex->aColExpr==0 ){ - ExprList *pCopy = sqlite3ExprListDup(db, pList, 0); - pIndex->aColExpr = pCopy; - if( !db->mallocFailed ){ - assert( pCopy!=0 ); - pListItem = &pCopy->a[i]; - } - } - j = XN_EXPR; - pIndex->aiColumn[i] = XN_EXPR; - pIndex->uniqNotNull = 0; - }else{ - j = pCExpr->iColumn; - assert( j<=0x7fff ); - if( j<0 ){ - j = pTab->iPKey; - }else if( pTab->aCol[j].notNull==0 ){ - pIndex->uniqNotNull = 0; - } - pIndex->aiColumn[i] = (i16)j; + const char *zColName = pListItem->zName; + int requestedSortOrder; + char *zColl; /* Collation sequence name */ + + for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ + if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break; } - zColl = 0; - if( pListItem->pExpr->op==TK_COLLATE ){ + if( j>=pTab->nCol ){ + sqlite3ErrorMsg(pParse, "table %s has no column named %s", + pTab->zName, zColName); + pParse->checkSchema = 1; + goto exit_create_index; + } + assert( j<=0x7fff ); + pIndex->aiColumn[i] = (i16)j; + if( pListItem->pExpr ){ int nColl; + assert( pListItem->pExpr->op==TK_COLLATE ); zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); @@ -104557,26 +96879,21 @@ SQLITE_PRIVATE void sqlite3CreateIndex( zColl = zExtra; zExtra += nColl; nExtra -= nColl; - }else if( j>=0 ){ + }else{ zColl = pTab->aCol[j].zColl; + if( !zColl ) zColl = "BINARY"; } - if( !zColl ) zColl = sqlite3StrBINARY; if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; } pIndex->azColl[i] = zColl; requestedSortOrder = pListItem->sortOrder & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; + if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0; } - - /* Append the table key to the end of the index. For WITHOUT ROWID - ** tables (when pPk!=0) this will be the declared PRIMARY KEY. For - ** normal tables (when pPk==0) this will be the rowid. - */ if( pPk ){ for(j=0; jnKeyCol; j++){ int x = pPk->aiColumn[j]; - assert( x>=0 ); if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ pIndex->nColumn--; }else{ @@ -104588,26 +96905,12 @@ SQLITE_PRIVATE void sqlite3CreateIndex( } assert( i==pIndex->nColumn ); }else{ - pIndex->aiColumn[i] = XN_ROWID; - pIndex->azColl[i] = sqlite3StrBINARY; + pIndex->aiColumn[i] = -1; + pIndex->azColl[i] = "BINARY"; } sqlite3DefaultRowEst(pIndex); if( pParse->pNewTable==0 ) estimateIndexWidth(pIndex); - /* If this index contains every column of its table, then mark - ** it as a covering index */ - assert( HasRowid(pTab) - || pTab->iPKey<0 || sqlite3ColumnOfIndex(pIndex, pTab->iPKey)>=0 ); - if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ - pIndex->isCovering = 1; - for(j=0; jnCol; j++){ - if( j==pTab->iPKey ) continue; - if( sqlite3ColumnOfIndex(pIndex,j)>=0 ) continue; - pIndex->isCovering = 0; - break; - } - } - if( pTab==pParse->pNewTable ){ /* This routine has been called to create an automatic index as a ** result of a PRIMARY KEY or UNIQUE clause on a column definition, or @@ -104641,11 +96944,10 @@ SQLITE_PRIVATE void sqlite3CreateIndex( for(k=0; knKeyCol; k++){ const char *z1; const char *z2; - assert( pIdx->aiColumn[k]>=0 ); if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; z1 = pIdx->azColl[k]; z2 = pIndex->azColl[k]; - if( sqlite3StrICmp(z1, z2) ) break; + if( z1!=z2 && sqlite3StrICmp(z1, z2) ) break; } if( k==pIdx->nKeyCol ){ if( pIdx->onError!=pIndex->onError ){ @@ -104664,7 +96966,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( pIdx->onError = pIndex->onError; } } - if( idxType==SQLITE_IDXTYPE_PRIMARYKEY ) pIdx->idxType = idxType; + pRet = pIdx; goto exit_create_index; } } @@ -104673,16 +96975,14 @@ SQLITE_PRIVATE void sqlite3CreateIndex( /* Link the new Index structure to its table and to the other ** in-memory database structures. */ - assert( pParse->nErr==0 ); if( db->init.busy ){ Index *p; - assert( !IN_DECLARE_VTAB ); assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); p = sqlite3HashInsert(&pIndex->pSchema->idxHash, pIndex->zName, pIndex); if( p ){ assert( p==pIndex ); /* Malloc must have failed */ - sqlite3OomFault(db); + db->mallocFailed = 1; goto exit_create_index; } db->flags |= SQLITE_InternChanges; @@ -104704,7 +97004,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( ** has just been created, it contains no data and the index initialization ** step can be skipped. */ - else if( HasRowid(pTab) || pTblName!=0 ){ + else if( pParse->nErr==0 && (HasRowid(pTab) || pTblName!=0) ){ Vdbe *v; char *zStmt; int iMem = ++pParse->nMem; @@ -104712,15 +97012,10 @@ SQLITE_PRIVATE void sqlite3CreateIndex( v = sqlite3GetVdbe(pParse); if( v==0 ) goto exit_create_index; - sqlite3BeginWriteOperation(pParse, 1, iDb); - /* Create the rootpage for the index using CreateIndex. But before - ** doing so, code a Noop instruction and store its address in - ** Index.tnum. This is required in case this index is actually a - ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In - ** that case the convertToWithoutRowidTable() routine will replace - ** the Noop with a Goto to jump over the VDBE code generated below. */ - pIndex->tnum = sqlite3VdbeAddOp0(v, OP_Noop); + /* Create the rootpage for the index + */ + sqlite3BeginWriteOperation(pParse, 1, iDb); sqlite3VdbeAddOp2(v, OP_CreateIndex, iDb, iMem); /* Gather the complete text of the CREATE INDEX statement into @@ -104742,7 +97037,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( */ sqlite3NestedParse(pParse, "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", - db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName, pTab->zName, iMem, @@ -104758,10 +97053,8 @@ SQLITE_PRIVATE void sqlite3CreateIndex( sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddParseSchemaOp(v, iDb, sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); - sqlite3VdbeAddOp0(v, OP_Expire); + sqlite3VdbeAddOp1(v, OP_Expire, 0); } - - sqlite3VdbeJumpHere(v, pIndex->tnum); } /* When adding an index to the list of indices for a table, make @@ -104783,6 +97076,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( pIndex->pNext = pOther->pNext; pOther->pNext = pIndex; } + pRet = pIndex; pIndex = 0; } @@ -104793,6 +97087,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( sqlite3ExprListDelete(db, pList); sqlite3SrcListDelete(db, pTblName); sqlite3DbFree(db, zName); + return pRet; } /* @@ -104821,11 +97116,10 @@ SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ int i; /* Set the first entry (number of rows in the index) to the estimated - ** number of rows in the table, or half the number of rows in the table - ** for a partial index. But do not let the estimate drop below 10. */ + ** number of rows in the table. Or 10, if the estimated number of rows + ** in the table is less than that. */ a[0] = pIdx->pTable->nRowLogEst; - if( pIdx->pPartIdxWhere!=0 ) a[0] -= 10; assert( 10==sqlite3LogEst(2) ); - if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is ** 6 and each subsequent value (if any) is 5. */ @@ -104876,7 +97170,7 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists { int code = SQLITE_DROP_INDEX; Table *pTab = pIndex->pTable; - const char *zDb = db->aDb[iDb].zDbSName; + const char *zDb = db->aDb[iDb].zName; const char *zTab = SCHEMA_TABLE(iDb); if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ goto exit_drop_index; @@ -104894,7 +97188,7 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists sqlite3BeginWriteOperation(pParse, 1, iDb); sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", - db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), pIndex->zName + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName ); sqlite3ClearStatTables(pParse, iDb, "idx", pIndex->zName); sqlite3ChangeCookie(pParse, iDb); @@ -105110,12 +97404,10 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend( ){ struct SrcList_item *pItem; assert( pDatabase==0 || pTable!=0 ); /* Cannot have C without B */ - assert( db!=0 ); if( pList==0 ){ - pList = sqlite3DbMallocRawNN(db, sizeof(SrcList) ); + pList = sqlite3DbMallocZero(db, sizeof(SrcList) ); if( pList==0 ) return 0; pList->nAlloc = 1; - pList->nSrc = 0; } pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); if( db->mallocFailed ){ @@ -105165,8 +97457,7 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3DbFree(db, pItem->zDatabase); sqlite3DbFree(db, pItem->zName); sqlite3DbFree(db, pItem->zAlias); - if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); - if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); + sqlite3DbFree(db, pItem->zIndex); sqlite3DeleteTable(db, pItem->pTab); sqlite3SelectDelete(db, pItem->pSelect); sqlite3ExprDelete(db, pItem->pOn); @@ -105239,37 +97530,17 @@ SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pI assert( pIndexedBy!=0 ); if( p && ALWAYS(p->nSrc>0) ){ struct SrcList_item *pItem = &p->a[p->nSrc-1]; - assert( pItem->fg.notIndexed==0 ); - assert( pItem->fg.isIndexedBy==0 ); - assert( pItem->fg.isTabFunc==0 ); + assert( pItem->notIndexed==0 && pItem->zIndex==0 ); if( pIndexedBy->n==1 && !pIndexedBy->z ){ /* A "NOT INDEXED" clause was supplied. See parse.y ** construct "indexed_opt" for details. */ - pItem->fg.notIndexed = 1; + pItem->notIndexed = 1; }else{ - pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); - pItem->fg.isIndexedBy = (pItem->u1.zIndexedBy!=0); + pItem->zIndex = sqlite3NameFromToken(pParse->db, pIndexedBy); } } } -/* -** Add the list of function arguments to the SrcList entry for a -** table-valued-function. -*/ -SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList *pList){ - if( p ){ - struct SrcList_item *pItem = &p->a[p->nSrc-1]; - assert( pItem->fg.notIndexed==0 ); - assert( pItem->fg.isIndexedBy==0 ); - assert( pItem->fg.isTabFunc==0 ); - pItem->u1.pFuncArg = pList; - pItem->fg.isTabFunc = 1; - }else{ - sqlite3ExprListDelete(pParse->db, pList); - } -} - /* ** When building up a FROM clause in the parser, the join operator ** is initially attached to the left operand. But the code generator @@ -105289,14 +97560,14 @@ SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ if( p ){ int i; for(i=p->nSrc-1; i>0; i--){ - p->a[i].fg.jointype = p->a[i-1].fg.jointype; + p->a[i].jointype = p->a[i-1].jointype; } - p->a[0].fg.jointype = 0; + p->a[0].jointype = 0; } } /* -** Generate VDBE code for a BEGIN statement. +** Begin a transaction */ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ sqlite3 *db; @@ -105306,6 +97577,7 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ assert( pParse!=0 ); db = pParse->db; assert( db!=0 ); +/* if( db->aDb[0].pBt==0 ) return; */ if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "BEGIN", 0, 0) ){ return; } @@ -105317,11 +97589,11 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ sqlite3VdbeUsesBtree(v, i); } } - sqlite3VdbeAddOp0(v, OP_AutoCommit); + sqlite3VdbeAddOp2(v, OP_AutoCommit, 0, 0); } /* -** Generate VDBE code for a COMMIT statement. +** Commit a transaction */ SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ Vdbe *v; @@ -105333,12 +97605,12 @@ SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ } v = sqlite3GetVdbe(pParse); if( v ){ - sqlite3VdbeAddOp1(v, OP_AutoCommit, 1); + sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 0); } } /* -** Generate VDBE code for a ROLLBACK statement. +** Rollback a transaction */ SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse *pParse){ Vdbe *v; @@ -105400,7 +97672,7 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ db->aDb[1].pBt = pBt; assert( db->aDb[1].pSchema ); if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ - sqlite3OomFault(db); + db->mallocFailed = 1; return 1; } } @@ -105415,13 +97687,15 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ */ SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); + sqlite3 *db = pToplevel->db; - assert( iDb>=0 && iDbdb->nDb ); - assert( pParse->db->aDb[iDb].pBt!=0 || iDb==1 ); + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 || iDb==1 ); assert( iDbdb, iDb, 0) ); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( DbMaskTest(pToplevel->cookieMask, iDb)==0 ){ DbMaskSet(pToplevel->cookieMask, iDb); + pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; if( !OMIT_TEMPDB && iDb==1 ){ sqlite3OpenTempDatabase(pToplevel); } @@ -105437,7 +97711,7 @@ SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb) int i; for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; - if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zDbSName)) ){ + if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zName)) ){ sqlite3CodeVerifySchema(pParse, i); } } @@ -105515,7 +97789,7 @@ SQLITE_PRIVATE void sqlite3HaltConstraint( sqlite3MayAbort(pParse); } sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); - sqlite3VdbeChangeP5(v, p5Errmsg); + if( p5Errmsg ) sqlite3VdbeChangeP5(v, p5Errmsg); } /* @@ -105532,16 +97806,12 @@ SQLITE_PRIVATE void sqlite3UniqueConstraint( Table *pTab = pIdx->pTable; sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); - if( pIdx->aColExpr ){ - sqlite3XPrintf(&errMsg, "index '%q'", pIdx->zName); - }else{ - for(j=0; jnKeyCol; j++){ - char *zCol; - assert( pIdx->aiColumn[j]>=0 ); - zCol = pTab->aCol[pIdx->aiColumn[j]].zName; - if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); - sqlite3XPrintf(&errMsg, "%s.%s", pTab->zName, zCol); - } + for(j=0; jnKeyCol; j++){ + char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); + sqlite3StrAccumAppendAll(&errMsg, pTab->zName); + sqlite3StrAccumAppend(&errMsg, ".", 1); + sqlite3StrAccumAppendAll(&errMsg, zCol); } zErr = sqlite3StrAccumFinish(&errMsg); sqlite3HaltConstraint(pParse, @@ -105684,7 +97954,7 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ if( iDb<0 ) return; z = sqlite3NameFromToken(db, pObjName); if( z==0 ) return; - zDb = db->aDb[iDb].zDbSName; + zDb = db->aDb[iDb].zName; pTab = sqlite3FindTable(db, z, zDb); if( pTab ){ reindexTable(pParse, pTab, 0); @@ -105705,6 +97975,10 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ /* ** Return a KeyInfo structure that is appropriate for the given Index. ** +** The KeyInfo structure for an index is cached in the Index object. +** So there might be multiple references to the returned pointer. The +** caller should not try to modify the KeyInfo object. +** ** The caller should invoke sqlite3KeyInfoUnref() on the returned object ** when it has finished using it. */ @@ -105722,8 +97996,9 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ if( pKey ){ assert( sqlite3KeyInfoIsWriteable(pKey) ); for(i=0; iazColl[i]; - pKey->aColl[i] = zColl==sqlite3StrBINARY ? 0 : + char *zColl = pIdx->azColl[i]; + assert( zColl!=0 ); + pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 : sqlite3LocateCollSeq(pParse, zColl); pKey->aSortOrder[i] = pIdx->aSortOrder[i]; } @@ -105769,9 +98044,10 @@ SQLITE_PRIVATE With *sqlite3WithAdd( }else{ pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); } - assert( (pNew!=0 && zName!=0) || db->mallocFailed ); + assert( zName!=0 || pNew==0 ); + assert( db->mallocFailed==0 || pNew==0 ); - if( db->mallocFailed ){ + if( pNew==0 ){ sqlite3ExprListDelete(db, pArglist); sqlite3SelectDelete(db, pQuery); sqlite3DbFree(db, zName); @@ -105780,7 +98056,7 @@ SQLITE_PRIVATE With *sqlite3WithAdd( pNew->a[pNew->nCte].pSelect = pQuery; pNew->a[pNew->nCte].pCols = pArglist; pNew->a[pNew->nCte].zName = zName; - pNew->a[pNew->nCte].zCteErr = 0; + pNew->a[pNew->nCte].zErr = 0; pNew->nCte++; } @@ -105822,7 +98098,6 @@ SQLITE_PRIVATE void sqlite3WithDelete(sqlite3 *db, With *pWith){ ** of user defined functions and collation sequences. */ -/* #include "sqliteInt.h" */ /* ** Invoke the 'collation needed' callback to request a collation sequence @@ -105985,7 +98260,7 @@ static CollSeq *findCollSeqEntry( */ assert( pDel==0 || pDel==pColl ); if( pDel!=0 ){ - sqlite3OomFault(db); + db->mallocFailed = 1; sqlite3DbFree(db, pDel); pColl = 0; } @@ -106051,8 +98326,8 @@ SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( ** 5: UTF16 byte order conversion required - argument count matches exactly ** 6: Perfect match: encoding and argument count match exactly. ** -** If nArg==(-2) then any function with a non-null xSFunc is -** a perfect match and any function with xSFunc NULL is +** If nArg==(-2) then any function with a non-null xStep or xFunc is +** a perfect match and any function with both xStep and xFunc NULL is ** a non-match. */ #define FUNC_PERFECT_MATCH 6 /* The score for a perfect match */ @@ -106064,7 +98339,7 @@ static int matchQuality( int match; /* nArg of -2 is a special case */ - if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; + if( nArg==(-2) ) return (p->xFunc==0 && p->xStep==0) ? 0 : FUNC_PERFECT_MATCH; /* Wrong number of arguments means "no match" */ if( p->nArg!=nArg && p->nArg>=0 ) return 0; @@ -106092,12 +98367,14 @@ static int matchQuality( ** a pointer to the matching FuncDef if found, or 0 if there is no match. */ static FuncDef *functionSearch( + FuncDefHash *pHash, /* Hash table to search */ int h, /* Hash of the name */ - const char *zFunc /* Name of function */ + const char *zFunc, /* Name of function */ + int nFunc /* Number of bytes in zFunc */ ){ FuncDef *p; - for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){ - if( sqlite3StrICmp(p->zName, zFunc)==0 ){ + for(p=pHash->a[h]; p; p=p->pHash){ + if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ return p; } } @@ -106107,26 +98384,23 @@ static FuncDef *functionSearch( /* ** Insert a new FuncDef into a FuncDefHash hash table. */ -SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs( - FuncDef *aDef, /* List of global functions to be inserted */ - int nDef /* Length of the apDef[] list */ +SQLITE_PRIVATE void sqlite3FuncDefInsert( + FuncDefHash *pHash, /* The hash table into which to insert */ + FuncDef *pDef /* The function definition to insert */ ){ - int i; - for(i=0; ipNext!=&aDef[i] ); - aDef[i].pNext = pOther->pNext; - pOther->pNext = &aDef[i]; - }else{ - aDef[i].pNext = 0; - aDef[i].u.pHash = sqlite3BuiltinFunctions.a[h]; - sqlite3BuiltinFunctions.a[h] = &aDef[i]; - } + FuncDef *pOther; + int nName = sqlite3Strlen30(pDef->zName); + u8 c1 = (u8)pDef->zName[0]; + int h = (sqlite3UpperToLower[c1] + nName) % ArraySize(pHash->a); + pOther = functionSearch(pHash, h, pDef->zName, nName); + if( pOther ){ + assert( pOther!=pDef && pOther->pNext!=pDef ); + pDef->pNext = pOther->pNext; + pOther->pNext = pDef; + }else{ + pDef->pNext = 0; + pDef->pHash = pHash->a[h]; + pHash->a[h] = pDef; } } @@ -106143,7 +98417,7 @@ SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs( ** no matching function previously existed. ** ** If nArg is -2, then the first valid function found is returned. A -** function is valid if xSFunc is non-zero. The nArg==(-2) +** function is valid if either xFunc or xStep is non-zero. The nArg==(-2) ** case is used to see if zName is a valid function name for some number ** of arguments. If nArg is -2, then createFlag must be 0. ** @@ -106153,7 +98427,8 @@ SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs( */ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( sqlite3 *db, /* An open database */ - const char *zName, /* Name of the function. zero-terminated */ + const char *zName, /* Name of the function. Not null-terminated */ + int nName, /* Number of characters in the name */ int nArg, /* Number of arguments. -1 means any number */ u8 enc, /* Preferred text encoding */ u8 createFlag /* Create new entry if true and does not otherwise exist */ @@ -106162,15 +98437,14 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( FuncDef *pBest = 0; /* Best match found so far */ int bestScore = 0; /* Score of best match */ int h; /* Hash value */ - int nName; /* Length of the name */ assert( nArg>=(-2) ); assert( nArg>=(-1) || createFlag==0 ); - nName = sqlite3Strlen30(zName); + h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % ArraySize(db->aFunc.a); /* First search for a match amongst the application-defined functions. */ - p = (FuncDef*)sqlite3HashFind(&db->aFunc, zName); + p = functionSearch(&db->aFunc, h, zName, nName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ @@ -106193,9 +98467,9 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( ** So we must not search for built-ins when creating a new function. */ if( !createFlag && (pBest==0 || (db->flags & SQLITE_PreferBuiltin)!=0) ){ + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); bestScore = 0; - h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % SQLITE_FUNC_HASH_SZ; - p = functionSearch(h, zName); + p = functionSearch(pHash, h, zName, nName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ @@ -106212,22 +98486,15 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( */ if( createFlag && bestScorezName = (const char*)&pBest[1]; + pBest->zName = (char *)&pBest[1]; pBest->nArg = (u16)nArg; pBest->funcFlags = enc; - memcpy((char*)&pBest[1], zName, nName+1); - pOther = (FuncDef*)sqlite3HashInsert(&db->aFunc, pBest->zName, pBest); - if( pOther==pBest ){ - sqlite3DbFree(db, pBest); - sqlite3OomFault(db); - return 0; - }else{ - pBest->pNext = pOther; - } + memcpy(pBest->zName, zName, nName); + pBest->zName[nName] = 0; + sqlite3FuncDefInsert(&db->aFunc, pBest); } - if( pBest && (pBest->xSFunc || createFlag) ){ + if( pBest && (pBest->xStep || pBest->xFunc || createFlag) ){ return pBest; } return 0; @@ -106281,7 +98548,7 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); } if( !p ){ - sqlite3OomFault(db); + db->mallocFailed = 1; }else if ( 0==p->file_format ){ sqlite3HashInit(&p->tblHash); sqlite3HashInit(&p->idxHash); @@ -106308,7 +98575,6 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ ** This file contains C code routines that are called by the parser ** in order to generate code for DELETE FROM statements. */ -/* #include "sqliteInt.h" */ /* ** While a SrcList can in general represent multiple tables and subqueries @@ -106398,12 +98664,11 @@ SQLITE_PRIVATE void sqlite3MaterializeView( if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); assert( pFrom->a[0].pOn==0 ); assert( pFrom->a[0].pUsing==0 ); } - pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, - SF_IncludeHidden, 0, 0); + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); sqlite3Select(pParse, pSel, &dest); sqlite3SelectDelete(db, pSel); @@ -106439,7 +98704,7 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( */ if( pOrderBy && (pLimit == 0) ) { sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); - goto limit_where_cleanup; + goto limit_where_cleanup_2; } /* We only need to generate a select expression if there @@ -106461,16 +98726,16 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( */ pSelectRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - if( pSelectRowid == 0 ) goto limit_where_cleanup; + if( pSelectRowid == 0 ) goto limit_where_cleanup_2; pEList = sqlite3ExprListAppend(pParse, 0, pSelectRowid); - if( pEList == 0 ) goto limit_where_cleanup; + if( pEList == 0 ) goto limit_where_cleanup_2; /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree ** and the SELECT subtree. */ pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); if( pSelectSrc == 0 ) { sqlite3ExprListDelete(pParse->db, pEList); - goto limit_where_cleanup; + goto limit_where_cleanup_2; } /* generate the SELECT expression tree. */ @@ -106480,11 +98745,21 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ pWhereRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - pInClause = pWhereRowid ? sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0) : 0; - sqlite3PExprAddSelect(pParse, pInClause, pSelect); + if( pWhereRowid == 0 ) goto limit_where_cleanup_1; + pInClause = sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0); + if( pInClause == 0 ) goto limit_where_cleanup_1; + + pInClause->x.pSelect = pSelect; + pInClause->flags |= EP_xIsSelect; + sqlite3ExprSetHeightAndFlags(pParse, pInClause); return pInClause; -limit_where_cleanup: + /* something went wrong. clean up anything allocated. */ +limit_where_cleanup_1: + sqlite3SelectDelete(pParse->db, pSelect); + return 0; + +limit_where_cleanup_2: sqlite3ExprDelete(pParse->db, pWhere); sqlite3ExprListDelete(pParse->db, pOrderBy); sqlite3ExprDelete(pParse->db, pLimit); @@ -106508,6 +98783,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ){ Vdbe *v; /* The virtual database engine */ Table *pTab; /* The table from which records will be deleted */ + const char *zDb; /* Name of database holding pTab */ int i; /* Loop counter */ WhereInfo *pWInfo; /* Information about the WHERE clause */ Index *pIdx; /* For looping over indices of the table */ @@ -106521,7 +98797,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( int iDb; /* Database number */ int memCnt = -1; /* Memory cell used for change counting */ int rcauth; /* Value returned by authorization callback */ - int eOnePass; /* ONEPASS_OFF or _SINGLE or _MULTI */ + int okOnePass; /* True for one-pass algorithm without the FIFO */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ u8 *aToOpen = 0; /* Open cursor iTabCur+j if aToOpen[j] is true */ Index *pPk; /* The PRIMARY KEY index on the table */ @@ -106533,9 +98809,8 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( int iRowSet = 0; /* Register for rowset of rows to delete */ int addrBypass = 0; /* Address of jump over the delete logic */ int addrLoop = 0; /* Top of the delete loop */ + int addrDelete = 0; /* Jump directly to the delete logic */ int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ - int bComplex; /* True if there are triggers or FKs or - ** subqueries in the WHERE clause */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ @@ -106563,7 +98838,6 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( #ifndef SQLITE_OMIT_TRIGGER pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); isView = pTab->pSelect!=0; - bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); #else # define pTrigger 0 # define isView 0 @@ -106584,8 +98858,8 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); - rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, - db->aDb[iDb].zDbSName); + zDb = db->aDb[iDb].zName; + rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb); assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); if( rcauth==SQLITE_DENY ){ goto delete_from_cleanup; @@ -106647,13 +98921,8 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** It is easier just to erase the whole table. Prior to version 3.6.5, ** this optimization caused the row change count (the value returned by ** API function sqlite3_count_changes) to be set incorrectly. */ - if( rcauth==SQLITE_OK - && pWhere==0 - && !bComplex - && !IsVirtual(pTab) -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK - && db->xPreUpdateCallback==0 -#endif + if( rcauth==SQLITE_OK && pWhere==0 && !pTrigger && !IsVirtual(pTab) + && 0==sqlite3FkRequired(pParse, pTab, 0, 0) ){ assert( !isView ); sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); @@ -106668,9 +98937,6 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { - u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK|WHERE_SEEK_TABLE; - if( sNC.ncFlags & NC_VarSelect ) bComplex = 1; - wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ pPk = 0; @@ -106691,18 +98957,13 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } /* Construct a query to find the rowid or primary key for every row - ** to be deleted, based on the WHERE clause. Set variable eOnePass - ** to indicate the strategy used to implement this delete: - ** - ** ONEPASS_OFF: Two-pass approach - use a FIFO for rowids/PK values. - ** ONEPASS_SINGLE: One-pass approach - at most one row deleted. - ** ONEPASS_MULTI: One-pass approach - any number of rows may be deleted. + ** to be deleted, based on the WHERE clause. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, + WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK, + iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; - eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); - assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); - assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); + okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); /* Keep track of the number of rows to be deleted */ if( db->flags & SQLITE_CountRows ){ @@ -106712,7 +98973,6 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( /* Extract the rowid or primary key for the current row */ if( pPk ){ for(i=0; iaiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pPk->aiColumn[i], iPk+i); } @@ -106723,12 +98983,13 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( iKey>pParse->nMem ) pParse->nMem = iKey; } - if( eOnePass!=ONEPASS_OFF ){ - /* For ONEPASS, no need to store the rowid/primary-key. There is only + if( okOnePass ){ + /* For ONEPASS, no need to store the rowid/primary-key. There is only ** one, so just keep it in its register(s) and fall through to the - ** delete code. */ + ** delete code. + */ nKey = nPk; /* OP_Found will use an unpacked key */ - aToOpen = sqlite3DbMallocRawNN(db, nIdx+2); + aToOpen = sqlite3DbMallocRaw(db, nIdx+2); if( aToOpen==0 ){ sqlite3WhereEnd(pWInfo); goto delete_from_cleanup; @@ -106738,27 +98999,27 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); + addrDelete = sqlite3VdbeAddOp0(v, OP_Goto); /* Jump to DELETE logic */ + }else if( pPk ){ + /* Construct a composite key for the row to be deleted and remember it */ + iKey = ++pParse->nMem; + nKey = 0; /* Zero tells OP_Found to use a composite key */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, + sqlite3IndexAffinityStr(v, pPk), nPk); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); }else{ - if( pPk ){ - /* Add the PK key for this row to the temporary table */ - iKey = ++pParse->nMem; - nKey = 0; /* Zero tells OP_Found to use a composite key */ - sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, - sqlite3IndexAffinityStr(pParse->db, pPk), nPk); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); - }else{ - /* Add the rowid of the row to be deleted to the RowSet */ - nKey = 1; /* OP_Seek always uses a single rowid */ - sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); - } + /* Get the rowid of the row to be deleted and remember it in the RowSet */ + nKey = 1; /* OP_Seek always uses a single rowid */ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); } - /* If this DELETE cannot use the ONEPASS strategy, this is the - ** end of the WHERE loop */ - if( eOnePass!=ONEPASS_OFF ){ + /* End of the WHERE loop */ + sqlite3WhereEnd(pWInfo); + if( okOnePass ){ + /* Bypass the delete logic below if the WHERE loop found zero rows */ addrBypass = sqlite3VdbeMakeLabel(v); - }else{ - sqlite3WhereEnd(pWInfo); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBypass); + sqlite3VdbeJumpHere(v, addrDelete); } /* Unless this is a view, open cursors for the table we are @@ -106767,24 +99028,21 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** triggers. */ if( !isView ){ - int iAddrOnce = 0; - if( eOnePass==ONEPASS_MULTI ){ - iAddrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); - } testcase( IsVirtual(pTab) ); - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, OPFLAG_FORDELETE, - iTabCur, aToOpen, &iDataCur, &iIdxCur); + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, + &iDataCur, &iIdxCur); assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); - if( eOnePass==ONEPASS_MULTI ) sqlite3VdbeJumpHere(v, iAddrOnce); } /* Set up a loop over the rowids/primary-keys that were found in the ** where-clause loop above. */ - if( eOnePass!=ONEPASS_OFF ){ + if( okOnePass ){ + /* Just one row. Hence the top-of-loop is a no-op */ assert( nKey==nPk ); /* OP_Found will use an unpacked key */ - if( !IsVirtual(pTab) && aToOpen[iDataCur-iTabCur] ){ + assert( !IsVirtual(pTab) ); + if( aToOpen[iDataCur-iTabCur] ){ assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); @@ -106806,32 +99064,23 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, OE_Abort); - assert( eOnePass==ONEPASS_OFF || eOnePass==ONEPASS_SINGLE ); sqlite3MayAbort(pParse); - if( eOnePass==ONEPASS_SINGLE && sqlite3IsToplevel(pParse) ){ - pParse->isMultiWrite = 0; - } }else #endif { int count = (pParse->nested==0); /* True to count changes */ - int iIdxNoSeek = -1; - if( bComplex==0 && aiCurOnePass[1]!=iDataCur ){ - iIdxNoSeek = aiCurOnePass[1]; - } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); + iKey, nKey, count, OE_Default, okOnePass); } /* End of the loop over all rowids/primary-keys. */ - if( eOnePass!=ONEPASS_OFF ){ + if( okOnePass ){ sqlite3VdbeResolveLabel(v, addrBypass); - sqlite3WhereEnd(pWInfo); }else if( pPk ){ sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrLoop); }else{ - sqlite3VdbeGoto(v, addrLoop); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrLoop); sqlite3VdbeJumpHere(v, addrLoop); } @@ -106898,25 +99147,6 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** sequence of nPk memory cells starting at iPk. If nPk==0 that means ** that a search record formed from OP_MakeRecord is contained in the ** single memory location iPk. -** -** eMode: -** Parameter eMode may be passed either ONEPASS_OFF (0), ONEPASS_SINGLE, or -** ONEPASS_MULTI. If eMode is not ONEPASS_OFF, then the cursor -** iDataCur already points to the row to delete. If eMode is ONEPASS_OFF -** then this function must seek iDataCur to the entry identified by iPk -** and nPk before reading from it. -** -** If eMode is ONEPASS_MULTI, then this call is being made as part -** of a ONEPASS delete that affects multiple rows. In this case, if -** iIdxNoSeek is a valid cursor number (>=0), then its position should -** be preserved following the delete operation. Or, if iIdxNoSeek is not -** a valid cursor number, the position of iDataCur should be preserved -** instead. -** -** iIdxNoSeek: -** If iIdxNoSeek is a valid cursor number (>=0), then it identifies an -** index cursor (from within array of cursors starting at iIdxCur) that -** already points to the index entry to be deleted. */ SQLITE_PRIVATE void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -106928,8 +99158,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( i16 nPk, /* Number of PRIMARY KEY memory cells */ u8 count, /* If non-zero, increment the row change counter */ u8 onconf, /* Default ON CONFLICT policy for triggers */ - u8 eMode, /* ONEPASS_OFF, _SINGLE, or _MULTI. See above */ - int iIdxNoSeek /* Cursor number of cursor that does not need seeking */ + u8 bNoSeek /* iDataCur is already pointing to the row to delete */ ){ Vdbe *v = pParse->pVdbe; /* Vdbe */ int iOld = 0; /* First register in OLD.* array */ @@ -106946,7 +99175,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( ** not attempt to delete it or fire any DELETE triggers. */ iLabel = sqlite3VdbeMakeLabel(v); opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - if( eMode==ONEPASS_OFF ){ + if( !bNoSeek ){ sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); VdbeCoverageIf(v, opSeek==OP_NotExists); VdbeCoverageIf(v, opSeek==OP_NotFound); @@ -107004,27 +99233,13 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( /* Delete the index and table entries. Skip this step if pTab is really ** a view (in which case the only effect of the DELETE statement is to - ** fire the INSTEAD OF triggers). - ** - ** If variable 'count' is non-zero, then this OP_Delete instruction should - ** invoke the update-hook. The pre-update-hook, on the other hand should - ** be invoked unless table pTab is a system table. The difference is that - ** the update-hook is not invoked for rows removed by REPLACE, but the - ** pre-update-hook is. - */ + ** fire the INSTEAD OF triggers). */ if( pTab->pSelect==0 ){ - u8 p5 = 0; - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); - sqlite3VdbeChangeP4(v, -1, (char*)pTab, P4_TABLE); - if( eMode!=ONEPASS_OFF ){ - sqlite3VdbeChangeP5(v, OPFLAG_AUXDELETE); + if( count ){ + sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); } - if( iIdxNoSeek>=0 ){ - sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); - } - if( eMode==ONEPASS_MULTI ) p5 |= OPFLAG_SAVEPOSITION; - sqlite3VdbeChangeP5(v, p5); } /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to @@ -107067,8 +99282,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( Table *pTab, /* Table containing the row to be deleted */ int iDataCur, /* Cursor of table holding data. */ int iIdxCur, /* First index cursor */ - int *aRegIdx, /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ - int iIdxNoSeek /* Do not delete from this cursor */ + int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ ){ int i; /* Index loop counter */ int r1 = -1; /* Register holding an index key */ @@ -107084,12 +99298,11 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( assert( iIdxCur+i!=iDataCur || pPk==pIdx ); if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; if( pIdx==pPk ) continue; - if( iIdxCur+i==iIdxNoSeek ) continue; VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, - &iPartIdxLabel, pPrior, r1); + &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, - pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } @@ -107138,16 +99351,17 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( ){ Vdbe *v = pParse->pVdbe; int j; + Table *pTab = pIdx->pTable; int regBase; int nCol; if( piPartIdxLabel ){ if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(v); - pParse->iSelfTab = iDataCur; + pParse->iPartIdxTab = iDataCur; sqlite3ExprCachePush(pParse); - sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, - SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, + SQLITE_JUMPIFNULL); }else{ *piPartIdxLabel = 0; } @@ -107156,14 +99370,9 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( regBase = sqlite3GetTempRange(pParse, nCol); if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; jaiColumn[j]==pIdx->aiColumn[j] - && pPrior->aiColumn[j]!=XN_EXPR - ){ - /* This column was already computed by the previous index */ - continue; - } - sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); + if( pPrior && pPrior->aiColumn[j]==pIdx->aiColumn[j] ) continue; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pIdx->aiColumn[j], + regBase+j); /* If the column affinity is REAL but the number is an integer, then it ** might be stored in the table as an integer (using a compact ** representation) then converted to REAL by an OP_RealAffinity opcode. @@ -107208,10 +99417,8 @@ SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ ** functions of SQLite. (Some function, and in particular the date and ** time functions, are implemented separately.) */ -/* #include "sqliteInt.h" */ /* #include */ /* #include */ -/* #include "vdbeInt.h" */ /* ** Return the collating function associated with a function. @@ -107403,8 +99610,6 @@ static void instrFunc( zHaystack = sqlite3_value_text(argv[0]); zNeedle = sqlite3_value_text(argv[1]); isText = 1; - if( zNeedle==0 ) return; - assert( zHaystack ); } while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ N++; @@ -107436,8 +99641,7 @@ static void printfFunc( x.nUsed = 0; x.apArg = argv+1; sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); - str.printfFlags = SQLITE_PRINTF_SQLFUNC; - sqlite3XPrintf(&str, zFormat, &x); + sqlite3XPrintf(&str, SQLITE_PRINTF_SQLFUNC, zFormat, &x); n = str.nChar; sqlite3_result_text(context, sqlite3StrAccumFinish(&str), n, SQLITE_DYNAMIC); @@ -107765,23 +99969,25 @@ static void total_changes( ** A structure defining how to do GLOB-style comparisons. */ struct compareInfo { - u8 matchAll; /* "*" or "%" */ - u8 matchOne; /* "?" or "_" */ - u8 matchSet; /* "[" or 0 */ - u8 noCase; /* true to ignore case differences */ + u8 matchAll; + u8 matchOne; + u8 matchSet; + u8 noCase; }; /* ** For LIKE and GLOB matching on EBCDIC machines, assume that every -** character is exactly one byte in size. Also, provde the Utf8Read() -** macro for fast reading of the next character in the common case where -** the next character is ASCII. +** character is exactly one byte in size. Also, all characters are +** able to participate in upper-case-to-lower-case mappings in EBCDIC +** whereas only characters less than 0x80 do in ASCII. */ #if defined(SQLITE_EBCDIC) # define sqlite3Utf8Read(A) (*((*A)++)) -# define Utf8Read(A) (*(A++)) +# define GlobUpperToLower(A) A = sqlite3UpperToLower[A] +# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] #else -# define Utf8Read(A) (A[0]<0x80?*(A++):sqlite3Utf8Read(&A)) +# define GlobUpperToLower(A) if( A<=0x7f ){ A = sqlite3UpperToLower[A]; } +# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] #endif static const struct compareInfo globInfo = { '*', '?', '[', 0 }; @@ -107823,7 +100029,7 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; ** Ec Where E is the "esc" character and c is any other ** character, including '%', '_', and esc, match exactly c. ** -** The comments within this routine usually assume glob matching. +** The comments through this routine usually assume glob matching. ** ** This routine is usually quick, but can be N**2 in the worst case. */ @@ -107831,20 +100037,29 @@ static int patternCompare( const u8 *zPattern, /* The glob pattern */ const u8 *zString, /* The string to compare against the glob */ const struct compareInfo *pInfo, /* Information about how to do the compare */ - u32 matchOther /* The escape char (LIKE) or '[' (GLOB) */ + u32 esc /* The escape character */ ){ u32 c, c2; /* Next pattern and input string chars */ u32 matchOne = pInfo->matchOne; /* "?" or "_" */ u32 matchAll = pInfo->matchAll; /* "*" or "%" */ + u32 matchOther; /* "[" or the escape character */ u8 noCase = pInfo->noCase; /* True if uppercase==lowercase */ const u8 *zEscaped = 0; /* One past the last escaped input char */ - while( (c = Utf8Read(zPattern))!=0 ){ + /* The GLOB operator does not have an ESCAPE clause. And LIKE does not + ** have the matchSet operator. So we either have to look for one or + ** the other, never both. Hence the single variable matchOther is used + ** to store the one we have to look for. + */ + matchOther = esc ? esc : pInfo->matchSet; + + while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ if( c==matchAll ){ /* Match "*" */ /* Skip over multiple "*" characters in the pattern. If there ** are also "?" characters, skip those as well, but consume a ** single character of the input string for each "?" skipped */ - while( (c=Utf8Read(zPattern)) == matchAll || c == matchOne ){ + while( (c=sqlite3Utf8Read(&zPattern)) == matchAll + || c == matchOne ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ return 0; } @@ -107852,7 +100067,7 @@ static int patternCompare( if( c==0 ){ return 1; /* "*" at the end of the pattern matches */ }else if( c==matchOther ){ - if( pInfo->matchSet==0 ){ + if( esc ){ c = sqlite3Utf8Read(&zPattern); if( c==0 ) return 0; }else{ @@ -107860,7 +100075,7 @@ static int patternCompare( ** recursive search in this case, but it is an unusual case. */ assert( matchOther<0x80 ); /* '[' is a single-byte character */ while( *zString - && patternCompare(&zPattern[-1],zString,pInfo,matchOther)==0 ){ + && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ SQLITE_SKIP_UTF8(zString); } return *zString!=0; @@ -107886,18 +100101,18 @@ static int patternCompare( } while( (c2 = *(zString++))!=0 ){ if( c2!=c && c2!=cx ) continue; - if( patternCompare(zPattern,zString,pInfo,matchOther) ) return 1; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } }else{ - while( (c2 = Utf8Read(zString))!=0 ){ + while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ if( c2!=c ) continue; - if( patternCompare(zPattern,zString,pInfo,matchOther) ) return 1; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } } return 0; } if( c==matchOther ){ - if( pInfo->matchSet==0 ){ + if( esc ){ c = sqlite3Utf8Read(&zPattern); if( c==0 ) return 0; zEscaped = zPattern; @@ -107935,9 +100150,9 @@ static int patternCompare( continue; } } - c2 = Utf8Read(zString); + c2 = sqlite3Utf8Read(&zString); if( c==c2 ) continue; - if( noCase && sqlite3Tolower(c)==sqlite3Tolower(c2) && c<0x80 && c2<0x80 ){ + if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){ continue; } if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue; @@ -107949,15 +100164,8 @@ static int patternCompare( /* ** The sqlite3_strglob() interface. */ -SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ - return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '[')==0; -} - -/* -** The sqlite3_strlike() interface. -*/ -SQLITE_API int sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ - return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc)==0; +SQLITE_API int SQLITE_STDCALL sqlite3_strglob(const char *zGlobPattern, const char *zString){ + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; } /* @@ -107988,22 +100196,10 @@ static void likeFunc( sqlite3_value **argv ){ const unsigned char *zA, *zB; - u32 escape; + u32 escape = 0; int nPat; sqlite3 *db = sqlite3_context_db_handle(context); - struct compareInfo *pInfo = sqlite3_user_data(context); -#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS - if( sqlite3_value_type(argv[0])==SQLITE_BLOB - || sqlite3_value_type(argv[1])==SQLITE_BLOB - ){ -#ifdef SQLITE_TEST - sqlite3_like_count++; -#endif - sqlite3_result_int(context, 0); - return; - } -#endif zB = sqlite3_value_text(argv[0]); zA = sqlite3_value_text(argv[1]); @@ -108031,13 +100227,13 @@ static void likeFunc( return; } escape = sqlite3Utf8Read(&zEsc); - }else{ - escape = pInfo->matchSet; } if( zA && zB ){ + struct compareInfo *pInfo = sqlite3_user_data(context); #ifdef SQLITE_TEST sqlite3_like_count++; #endif + sqlite3_result_int(context, patternCompare(zB, zA, pInfo, escape)); } } @@ -108331,14 +100527,16 @@ static void zeroblobFunc( sqlite3_value **argv ){ i64 n; - int rc; + sqlite3 *db = sqlite3_context_db_handle(context); assert( argc==1 ); UNUSED_PARAMETER(argc); n = sqlite3_value_int64(argv[0]); - if( n<0 ) n = 0; - rc = sqlite3_result_zeroblob64(context, n); /* IMP: R-00293-64994 */ - if( rc ){ - sqlite3_result_error_code(context, rc); + testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); + if( n>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + }else{ + sqlite3_result_zeroblob(context, (int)n); /* IMP: R-00293-64994 */ } } @@ -108513,26 +100711,6 @@ static void trimFunc( } -#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION -/* -** The "unknown" function is automatically substituted in place of -** any unrecognized function name when doing an EXPLAIN or EXPLAIN QUERY PLAN -** when the SQLITE_ENABLE_UNKNOWN_FUNCTION compile-time option is used. -** When the "sqlite3" command-line shell is built using this functionality, -** that allows an EXPLAIN or EXPLAIN QUERY PLAN for complex queries -** involving application-defined functions to be examined in a generic -** sqlite3 shell. -*/ -static void unknownFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - /* no-op */ -} -#endif /*SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION*/ - - /* IMP: R-25361-16150 This function is omitted from SQLite by default. It ** is only available if the SQLITE_SOUNDEX compile-time option is used ** when SQLite is built. @@ -108603,14 +100781,6 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3 *db = sqlite3_context_db_handle(context); char *zErrMsg = 0; - /* Disallow the load_extension() SQL function unless the SQLITE_LoadExtFunc - ** flag is set. See the sqlite3_enable_load_extension() API. - */ - if( (db->flags & SQLITE_LoadExtFunc)==0 ){ - sqlite3_result_error(context, "not authorized", -1); - return; - } - if( argc==2 ){ zProc = (const char *)sqlite3_value_text(argv[1]); }else{ @@ -108836,7 +101006,7 @@ static void groupConcatFinalize(sqlite3_context *context){ ** of the built-in functions above are part of the global function set. ** This routine only deals with those that are not global. */ -SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ +SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ int rc = sqlite3_overload_function(db, "MATCH", 2); /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC @@ -108847,7 +101017,7 @@ SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ /* END SQLCIPHER */ assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); if( rc==SQLITE_NOMEM ){ - sqlite3OomFault(db); + db->mallocFailed = 1; } /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC @@ -108863,7 +101033,8 @@ SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ */ static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ FuncDef *pDef; - pDef = sqlite3FindFunction(db, zName, 2, SQLITE_UTF8, 0); + pDef = sqlite3FindFunction(db, zName, sqlite3Strlen30(zName), + 2, SQLITE_UTF8, 0); if( ALWAYS(pDef) ){ pDef->funcFlags |= flagVal; } @@ -108911,7 +101082,9 @@ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocas return 0; } assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - pDef = sqlite3FindFunction(db, pExpr->u.zToken, 2, SQLITE_UTF8, 0); + pDef = sqlite3FindFunction(db, pExpr->u.zToken, + sqlite3Strlen30(pExpr->u.zToken), + 2, SQLITE_UTF8, 0); if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_FUNC_LIKE)==0 ){ return 0; } @@ -108935,7 +101108,7 @@ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocas ** ** After this routine runs */ -SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ +SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ /* ** The following array holds FuncDef structures for all of the functions ** defined in this file. @@ -108943,27 +101116,8 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ ** The array cannot be constant since changes are made to the ** FuncDef.pHash elements at start-time. The elements of this array ** are read-only after initialization is complete. - ** - ** For peak efficiency, put the most frequently used function last. */ - static FuncDef aBuiltinFunc[] = { -#ifdef SQLITE_SOUNDEX - FUNCTION(soundex, 1, 0, 0, soundexFunc ), -#endif -#ifndef SQLITE_OMIT_LOAD_EXTENSION - VFUNCTION(load_extension, 1, 0, 0, loadExt ), - VFUNCTION(load_extension, 2, 0, 0, loadExt ), -#endif -#if SQLITE_USER_AUTHENTICATION - FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), -#endif -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), - DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), -#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ - FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), - FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), - FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + static SQLITE_WSD FuncDef aBuiltinFunc[] = { FUNCTION(ltrim, 1, 1, 0, trimFunc ), FUNCTION(ltrim, 2, 1, 0, trimFunc ), FUNCTION(rtrim, 1, 2, 0, trimFunc ), @@ -108981,6 +101135,8 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), FUNCTION(instr, 2, 0, 0, instrFunc ), + FUNCTION(substr, 2, 0, 0, substrFunc ), + FUNCTION(substr, 3, 0, 0, substrFunc ), FUNCTION(printf, -1, 0, 0, printfFunc ), FUNCTION(unicode, 1, 0, 0, unicodeFunc ), FUNCTION(char, -1, 0, 0, charFunc ), @@ -108991,22 +101147,40 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ #endif FUNCTION(upper, 1, 0, 0, upperFunc ), FUNCTION(lower, 1, 0, 0, lowerFunc ), + FUNCTION(coalesce, 1, 0, 0, 0 ), + FUNCTION(coalesce, 0, 0, 0, 0 ), + FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), FUNCTION(hex, 1, 0, 0, hexFunc ), FUNCTION2(ifnull, 2, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), + FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), - DFUNCTION(sqlite_version, 0, 0, 0, versionFunc ), - DFUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), +#if SQLITE_USER_AUTHENTICATION + FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), +#endif +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ FUNCTION(quote, 1, 0, 0, quoteFunc ), VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), VFUNCTION(changes, 0, 0, 0, changes ), VFUNCTION(total_changes, 0, 0, 0, total_changes ), FUNCTION(replace, 3, 0, 0, replaceFunc ), FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), - FUNCTION(substr, 2, 0, 0, substrFunc ), - FUNCTION(substr, 3, 0, 0, substrFunc ), + #ifdef SQLITE_SOUNDEX + FUNCTION(soundex, 1, 0, 0, soundexFunc ), + #endif + #ifndef SQLITE_OMIT_LOAD_EXTENSION + FUNCTION(load_extension, 1, 0, 0, loadExt ), + FUNCTION(load_extension, 2, 0, 0, loadExt ), + #endif AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), @@ -109017,43 +101191,28 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), -#ifdef SQLITE_CASE_SENSITIVE_LIKE + #ifdef SQLITE_CASE_SENSITIVE_LIKE LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), -#else + #else LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), -#endif -#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION - FUNCTION(unknown, -1, 0, 0, unknownFunc ), -#endif - FUNCTION(coalesce, 1, 0, 0, 0 ), - FUNCTION(coalesce, 0, 0, 0, 0 ), - FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), + #endif }; + + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aBuiltinFunc); + + for(i=0; iu.pHash){ - int n = sqlite3Strlen30(p->zName); - int h = p->zName[0] + n; - printf(" %s(%d)", p->zName, h); - } - printf("\n"); - } - } #endif } @@ -109072,7 +101231,6 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ ** This file contains code used by the compiler to add foreign key ** support to compiled SQL statements. */ -/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_FOREIGN_KEY #ifndef SQLITE_OMIT_TRIGGER @@ -109280,7 +101438,7 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( } }else if( paiCol ){ assert( nCol>1 ); - aiCol = (int *)sqlite3DbMallocRawNN(pParse->db, nCol*sizeof(int)); + aiCol = (int *)sqlite3DbMallocRaw(pParse->db, nCol*sizeof(int)); if( !aiCol ) return 1; *paiCol = aiCol; } @@ -109310,16 +101468,16 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( int i, j; for(i=0; iaiColumn[i]; /* Index of column in parent tbl */ - const char *zDfltColl; /* Def. collation for column */ + char *zDfltColl; /* Def. collation for column */ char *zIdxCol; /* Name of indexed column */ - if( iCol<0 ) break; /* No foreign keys against expression indexes */ - /* If the index uses a collation sequence that is different from ** the default collation sequence for the column, this index is ** unusable. Bail out early in this case. */ zDfltColl = pParent->aCol[iCol].zColl; - if( !zDfltColl ) zDfltColl = sqlite3StrBINARY; + if( !zDfltColl ){ + zDfltColl = "BINARY"; + } if( sqlite3StrICmp(pIdx->azColl[i], zDfltColl) ) break; zIdxCol = pParent->aCol[iCol].zName; @@ -109435,7 +101593,7 @@ static void fkLookupParent( sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); VdbeCoverage(v); - sqlite3VdbeGoto(v, iOk); + sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); sqlite3VdbeJumpHere(v, iMustBeInt); sqlite3ReleaseTempReg(pParse, regTemp); @@ -109465,7 +101623,6 @@ static void fkLookupParent( for(i=0; iaiColumn[i]+1+regData; - assert( pIdx->aiColumn[i]>=0 ); assert( aiCol[i]!=pTab->iPKey ); if( pIdx->aiColumn[i]==pTab->iPKey ){ /* The parent key is a composite key that includes the IPK column */ @@ -109474,11 +101631,11 @@ static void fkLookupParent( sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); } - sqlite3VdbeGoto(v, iOk); + sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); } sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, - sqlite3IndexAffinityStr(pParse->db,pIdx), nCol); + sqlite3IndexAffinityStr(v,pIdx), nCol); sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, regRec); @@ -109674,7 +101831,6 @@ static void fkScanChildren( assert( pIdx!=0 ); for(i=0; inKeyCol; i++){ i16 iCol = pIdx->aiColumn[i]; - assert( iCol>=0 ); pLeft = exprTableRegister(pParse, pTab, regData, iCol); pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, iCol); pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); @@ -109932,7 +102088,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( if( (db->flags&SQLITE_ForeignKeys)==0 ) return; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - zDb = db->aDb[iDb].zDbSName; + zDb = db->aDb[iDb].zName; /* Loop through all the foreign key constraints for which pTab is the ** child table (the table that the foreign key definition is part of). */ @@ -109994,7 +102150,6 @@ SQLITE_PRIVATE void sqlite3FkCheck( if( aiCol[i]==pTab->iPKey ){ aiCol[i] = -1; } - assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Request permission to read the parent key columns. If the ** authorization callback returns SQLITE_IGNORE, behave as if any @@ -110126,10 +102281,7 @@ SQLITE_PRIVATE u32 sqlite3FkOldmask( Index *pIdx = 0; sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); if( pIdx ){ - for(i=0; inKeyCol; i++){ - assert( pIdx->aiColumn[i]>=0 ); - mask |= COLUMN_MASK(pIdx->aiColumn[i]); - } + for(i=0; inKeyCol; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); } } } @@ -110223,12 +102375,10 @@ static Trigger *fkActionTrigger( int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ action = pFKey->aAction[iAction]; - if( action==OE_Restrict && (db->flags & SQLITE_DeferFKs) ){ - return 0; - } pTrigger = pFKey->apTrigger[iAction]; if( action!=OE_None && !pTrigger ){ + u8 enableLookaside; /* Copy of db->lookaside.bEnabled */ char const *zFrom; /* Name of child table */ int nFrom; /* Length in bytes of zFrom */ Index *pIdx = 0; /* Parent key index for this FK */ @@ -110254,10 +102404,11 @@ static Trigger *fkActionTrigger( iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; assert( iFromCol>=0 ); assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKeynCol) ); - assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); - sqlite3TokenInit(&tToCol, - pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName); - sqlite3TokenInit(&tFromCol, pFKey->pFrom->aCol[iFromCol].zName); + tToCol.z = pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName; + tFromCol.z = pFKey->pFrom->aCol[iFromCol].zName; + + tToCol.n = sqlite3Strlen30(tToCol.z); + tFromCol.n = sqlite3Strlen30(tFromCol.z); /* Create the expression "OLD.zToCol = zFromCol". It is important ** that the "OLD.zToCol" term is on the LHS of the = operator, so @@ -110303,10 +102454,10 @@ static Trigger *fkActionTrigger( if( pDflt ){ pNew = sqlite3ExprDup(db, pDflt, 0); }else{ - pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); + pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); } }else{ - pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); + pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); } pList = sqlite3ExprListAppend(pParse, pList, pNew); sqlite3ExprListSetName(pParse, pList, &tFromCol, 0); @@ -110337,7 +102488,8 @@ static Trigger *fkActionTrigger( } /* Disable lookaside memory allocation */ - db->lookaside.bDisable++; + enableLookaside = db->lookaside.bEnabled; + db->lookaside.bEnabled = 0; pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ @@ -110359,7 +102511,7 @@ static Trigger *fkActionTrigger( } /* Re-enable the lookaside buffer, if it was disabled earlier. */ - db->lookaside.bDisable--; + db->lookaside.bEnabled = enableLookaside; sqlite3ExprDelete(db, pWhere); sqlite3ExprDelete(db, pWhen); @@ -110433,8 +102585,7 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pFKey; /* Iterator variable */ FKey *pNext; /* Copy of pFKey->pNextFrom */ - assert( db==0 || IsVirtual(pTab) - || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ /* Remove the FK from the fkeyHash hash table. */ @@ -110484,7 +102635,6 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ ** This file contains C code routines that are called by the parser ** to handle INSERT statements in SQLite. */ -/* #include "sqliteInt.h" */ /* ** Generate code that will @@ -110514,7 +102664,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); assert( pPk!=0 ); - assert( pPk->tnum==pTab->tnum ); + assert( pPk->tnum=pTab->tnum ); sqlite3VdbeAddOp3(v, opcode, iCur, pPk->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pPk); VdbeComment((v, "%s", pTab->zName)); @@ -110528,7 +102678,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** ** Character Column affinity ** ------------------------------ -** 'A' BLOB +** 'A' NONE ** 'B' TEXT ** 'C' NUMERIC ** 'D' INTEGER @@ -110541,7 +102691,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** is managed along with the rest of the Index structure. It will be ** released when sqlite3DeleteIndex() is called. */ -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ if( !pIdx->zColAff ){ /* The first time a column affinity string for a particular index is ** required, it is allocated and populated here. It is then stored as @@ -110553,25 +102703,15 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ */ int n; Table *pTab = pIdx->pTable; + sqlite3 *db = sqlite3VdbeDb(v); pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ - sqlite3OomFault(db); + db->mallocFailed = 1; return 0; } for(n=0; nnColumn; n++){ i16 x = pIdx->aiColumn[n]; - if( x>=0 ){ - pIdx->zColAff[n] = pTab->aCol[x].affinity; - }else if( x==XN_ROWID ){ - pIdx->zColAff[n] = SQLITE_AFF_INTEGER; - }else{ - char aff; - assert( x==XN_EXPR ); - assert( pIdx->aColExpr!=0 ); - aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); - if( aff==0 ) aff = SQLITE_AFF_BLOB; - pIdx->zColAff[n] = aff; - } + pIdx->zColAff[n] = x<0 ? SQLITE_AFF_INTEGER : pTab->aCol[x].affinity; } pIdx->zColAff[n] = 0; } @@ -110581,9 +102721,9 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ /* ** Compute the affinity string for table pTab, if it has not already been -** computed. As an optimization, omit trailing SQLITE_AFF_BLOB affinities. +** computed. As an optimization, omit trailing SQLITE_AFF_NONE affinities. ** -** If the affinity exists (if it is no entirely SQLITE_AFF_BLOB values) and +** If the affinity exists (if it is no entirely SQLITE_AFF_NONE values) and ** if iReg>0 then code an OP_Affinity opcode that will set the affinities ** for register iReg and following. Or if affinities exists and iReg==0, ** then just set the P4 operand of the previous opcode (which should be @@ -110593,7 +102733,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ ** ** Character Column affinity ** ------------------------------ -** 'A' BLOB +** 'A' NONE ** 'B' TEXT ** 'C' NUMERIC ** 'D' INTEGER @@ -110606,7 +102746,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ sqlite3 *db = sqlite3VdbeDb(v); zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); if( !zColAff ){ - sqlite3OomFault(db); + db->mallocFailed = 1; return; } @@ -110615,7 +102755,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ } do{ zColAff[i--] = 0; - }while( i>=0 && zColAff[i]==SQLITE_AFF_BLOB ); + }while( i>=0 && zColAff[i]==SQLITE_AFF_NONE ); pTab->zColAff = zColAff; } i = sqlite3Strlen30(zColAff); @@ -110672,9 +102812,7 @@ static int readsTable(Parse *p, int iDb, Table *pTab){ /* ** Locate or create an AutoincInfo structure associated with table pTab ** which is in database iDb. Return the register number for the register -** that holds the maximum rowid. Return zero if pTab is not an AUTOINCREMENT -** table. (Also return zero when doing a VACUUM since we do not want to -** update the AUTOINCREMENT counters during a VACUUM.) +** that holds the maximum rowid. ** ** There is at most one AutoincInfo structure per table even if the ** same table is autoincremented multiple times due to inserts within @@ -110697,16 +102835,14 @@ static int autoIncBegin( Table *pTab /* The table we are writing to */ ){ int memId = 0; /* Register holding maximum rowid */ - if( (pTab->tabFlags & TF_Autoincrement)!=0 - && (pParse->db->flags & SQLITE_Vacuum)==0 - ){ + if( pTab->tabFlags & TF_Autoincrement ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); AutoincInfo *pInfo; pInfo = pToplevel->pAinc; while( pInfo && pInfo->pTab!=pTab ){ pInfo = pInfo->pNext; } if( pInfo==0 ){ - pInfo = sqlite3DbMallocRawNN(pParse->db, sizeof(*pInfo)); + pInfo = sqlite3DbMallocRaw(pParse->db, sizeof(*pInfo)); if( pInfo==0 ) return 0; pInfo->pNext = pToplevel->pAinc; pToplevel->pAinc = pInfo; @@ -110730,55 +102866,43 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* Database only autoinc table */ int memId; /* Register holding max rowid */ + int addr; /* A VDBE address */ Vdbe *v = pParse->pVdbe; /* VDBE under construction */ /* This routine is never called during trigger-generation. It is ** only called from the top-level */ assert( pParse->pTriggerTab==0 ); - assert( sqlite3IsToplevel(pParse) ); + assert( pParse==sqlite3ParseToplevel(pParse) ); assert( v ); /* We failed long ago if this is not so */ for(p = pParse->pAinc; p; p = p->pNext){ - static const int iLn = VDBE_OFFSET_LINENO(2); - static const VdbeOpList autoInc[] = { - /* 0 */ {OP_Null, 0, 0, 0}, - /* 1 */ {OP_Rewind, 0, 9, 0}, - /* 2 */ {OP_Column, 0, 0, 0}, - /* 3 */ {OP_Ne, 0, 7, 0}, - /* 4 */ {OP_Rowid, 0, 0, 0}, - /* 5 */ {OP_Column, 0, 1, 0}, - /* 6 */ {OP_Goto, 0, 9, 0}, - /* 7 */ {OP_Next, 0, 2, 0}, - /* 8 */ {OP_Integer, 0, 0, 0}, - /* 9 */ {OP_Close, 0, 0, 0} - }; - VdbeOp *aOp; pDb = &db->aDb[p->iDb]; memId = p->regCtr; assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); - sqlite3VdbeLoadString(v, memId-1, p->pTab->zName); - aOp = sqlite3VdbeAddOpList(v, ArraySize(autoInc), autoInc, iLn); - if( aOp==0 ) break; - aOp[0].p2 = memId; - aOp[0].p3 = memId+1; - aOp[2].p3 = memId; - aOp[3].p1 = memId-1; - aOp[3].p3 = memId; - aOp[3].p5 = SQLITE_JUMPIFNULL; - aOp[4].p2 = memId+1; - aOp[5].p3 = memId; - aOp[8].p2 = memId; + sqlite3VdbeAddOp3(v, OP_Null, 0, memId, memId+1); + addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, p->pTab->zName, 0); + sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, 0, 0, memId); + sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); + sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); + sqlite3VdbeAddOp3(v, OP_Column, 0, 1, memId); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr+9); + sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); + sqlite3VdbeAddOp0(v, OP_Close); } } /* ** Update the maximum rowid for an autoincrement calculation. ** -** This routine should be called when the regRowid register holds a +** This routine should be called when the top of the stack holds a ** new rowid that is about to be inserted. If that new rowid is ** larger than the maximum rowid in the memId memory cell, then the -** memory cell is updated. +** memory cell is updated. The stack is unchanged. */ static void autoIncStep(Parse *pParse, int memId, int regRowid){ if( memId>0 ){ @@ -110793,44 +102917,31 @@ static void autoIncStep(Parse *pParse, int memId, int regRowid){ ** table (either directly or through triggers) needs to call this ** routine just before the "exit" code. */ -static SQLITE_NOINLINE void autoIncrementEnd(Parse *pParse){ +SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ AutoincInfo *p; Vdbe *v = pParse->pVdbe; sqlite3 *db = pParse->db; assert( v ); for(p = pParse->pAinc; p; p = p->pNext){ - static const int iLn = VDBE_OFFSET_LINENO(2); - static const VdbeOpList autoIncEnd[] = { - /* 0 */ {OP_NotNull, 0, 2, 0}, - /* 1 */ {OP_NewRowid, 0, 0, 0}, - /* 2 */ {OP_MakeRecord, 0, 2, 0}, - /* 3 */ {OP_Insert, 0, 0, 0}, - /* 4 */ {OP_Close, 0, 0, 0} - }; - VdbeOp *aOp; Db *pDb = &db->aDb[p->iDb]; + int j1; int iRec; int memId = p->regCtr; iRec = sqlite3GetTempReg(pParse); assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); - aOp = sqlite3VdbeAddOpList(v, ArraySize(autoIncEnd), autoIncEnd, iLn); - if( aOp==0 ) break; - aOp[0].p1 = memId+1; - aOp[1].p2 = memId+1; - aOp[2].p1 = memId-1; - aOp[2].p3 = iRec; - aOp[3].p2 = iRec; - aOp[3].p3 = memId+1; - aOp[3].p5 = OPFLAG_APPEND; + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); + sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3VdbeAddOp0(v, OP_Close); sqlite3ReleaseTempReg(pParse, iRec); } } -SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ - if( pParse->pAinc ) autoIncrementEnd(pParse); -} #else /* ** If SQLITE_OMIT_AUTOINCREMENT is defined, then the three routines @@ -110957,6 +103068,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3 *db; /* The main database structure */ Table *pTab; /* The table to insert into. aka TABLE */ char *zTab; /* Name of the table into which we are inserting */ + const char *zDb; /* Name of the database holding this table */ int i, j, idx; /* Loop counters */ Vdbe *v; /* Generate code into this virtual machine */ Index *pIdx; /* For looping over indices of the table */ @@ -110971,6 +103083,7 @@ SQLITE_PRIVATE void sqlite3Insert( int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ SelectDest dest; /* Destination for SELECT on rhs of INSERT */ int iDb; /* Index of database holding TABLE */ + Db *pDb; /* The database containing table being inserted into */ u8 useTempTable = 0; /* Store SELECT results in intermediate table */ u8 appendFlag = 0; /* True if the insert is likely to be an append */ u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ @@ -111020,8 +103133,9 @@ SQLITE_PRIVATE void sqlite3Insert( } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); - if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, - db->aDb[iDb].zDbSName) ){ + pDb = &db->aDb[iDb]; + zDb = pDb->zName; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, zDb) ){ goto insert_cleanup; } withoutRowid = !HasRowid(pTab); @@ -111158,7 +103272,7 @@ SQLITE_PRIVATE void sqlite3Insert( rc = sqlite3Select(pParse, pSelect, &dest); regFromSelect = dest.iSdst; if( rc || db->mallocFailed || pParse->nErr ) goto insert_cleanup; - sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ assert( pSelect->pEList ); nColumn = pSelect->pEList->nExpr; @@ -111199,7 +103313,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); - sqlite3VdbeGoto(v, addrL); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrL); sqlite3VdbeJumpHere(v, addrL); sqlite3ReleaseTempReg(pParse, regRec); sqlite3ReleaseTempReg(pParse, regTempRowid); @@ -111213,13 +103327,11 @@ SQLITE_PRIVATE void sqlite3Insert( sNC.pParse = pParse; srcTab = -1; assert( useTempTable==0 ); - if( pList ){ - nColumn = pList->nExpr; - if( sqlite3ResolveExprListNames(&sNC, pList) ){ + nColumn = pList ? pList->nExpr : 0; + for(i=0; ia[i].pExpr) ){ goto insert_cleanup; } - }else{ - nColumn = 0; } } @@ -111234,8 +103346,10 @@ SQLITE_PRIVATE void sqlite3Insert( /* Make sure the number of columns in the source data matches the number ** of columns to be inserted into the table. */ - for(i=0; inCol; i++){ - nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); + if( IsVirtual(pTab) ){ + for(i=0; inCol; i++){ + nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); + } } if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){ sqlite3ErrorMsg(pParse, @@ -111258,9 +103372,9 @@ SQLITE_PRIVATE void sqlite3Insert( /* If this is not a view, open the table and and all indices */ if( !isView ){ int nIdx; - nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, -1, 0, + nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, -1, 0, &iDataCur, &iIdxCur); - aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+1)); + aRegIdx = sqlite3DbMallocRaw(db, sizeof(int)*(nIdx+1)); if( aRegIdx==0 ){ goto insert_cleanup; } @@ -111310,7 +103424,7 @@ SQLITE_PRIVATE void sqlite3Insert( if( ipkColumn<0 ){ sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); }else{ - int addr1; + int j1; assert( !withoutRowid ); if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regCols); @@ -111318,9 +103432,9 @@ SQLITE_PRIVATE void sqlite3Insert( assert( pSelect==0 ); /* Otherwise useTempTable is true */ sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regCols); } - addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeJumpHere(v, j1); sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); } @@ -111331,14 +103445,15 @@ SQLITE_PRIVATE void sqlite3Insert( /* Create the new column data */ - for(i=j=0; inCol; i++){ - if( pColumn ){ + for(i=0; inCol; i++){ + if( pColumn==0 ){ + j = i; + }else{ for(j=0; jnId; j++){ if( pColumn->a[j].idx==i ) break; } } - if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) - || (pColumn==0 && IsOrdinaryHiddenColumn(&pTab->aCol[i])) ){ + if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) ){ sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i+1); }else if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i+1); @@ -111346,7 +103461,6 @@ SQLITE_PRIVATE void sqlite3Insert( assert( pSelect==0 ); /* Otherwise useTempTable is true */ sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i+1); } - if( pColumn==0 && !IsOrdinaryHiddenColumn(&pTab->aCol[i]) ) j++; } /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger, @@ -111394,14 +103508,14 @@ SQLITE_PRIVATE void sqlite3Insert( ** to generate a unique primary key value. */ if( !appendFlag ){ - int addr1; + int j1; if( !IsVirtual(pTab) ){ - addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeJumpHere(v, j1); }else{ - addr1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, addr1+2); VdbeCoverage(v); + j1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); VdbeCoverage(v); } sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); } @@ -111430,6 +103544,7 @@ SQLITE_PRIVATE void sqlite3Insert( } if( pColumn==0 ){ if( IsHiddenColumn(&pTab->aCol[i]) ){ + assert( IsVirtual(pTab) ); j = -1; nHidden++; }else{ @@ -111468,7 +103583,7 @@ SQLITE_PRIVATE void sqlite3Insert( { int isReplace; /* Set to true if constraints may cause a replace */ sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, - regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0 + regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace ); sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, @@ -111497,7 +103612,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeJumpHere(v, addrInsTop); sqlite3VdbeAddOp1(v, OP_Close, srcTab); }else if( pSelect ){ - sqlite3VdbeGoto(v, addrCont); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrCont); sqlite3VdbeJumpHere(v, addrInsTop); } @@ -111550,59 +103665,6 @@ SQLITE_PRIVATE void sqlite3Insert( #undef tmask #endif -/* -** Meanings of bits in of pWalker->eCode for checkConstraintUnchanged() -*/ -#define CKCNSTRNT_COLUMN 0x01 /* CHECK constraint uses a changing column */ -#define CKCNSTRNT_ROWID 0x02 /* CHECK constraint references the ROWID */ - -/* This is the Walker callback from checkConstraintUnchanged(). Set -** bit 0x01 of pWalker->eCode if -** pWalker->eCode to 0 if this expression node references any of the -** columns that are being modifed by an UPDATE statement. -*/ -static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_COLUMN ){ - assert( pExpr->iColumn>=0 || pExpr->iColumn==-1 ); - if( pExpr->iColumn>=0 ){ - if( pWalker->u.aiCol[pExpr->iColumn]>=0 ){ - pWalker->eCode |= CKCNSTRNT_COLUMN; - } - }else{ - pWalker->eCode |= CKCNSTRNT_ROWID; - } - } - return WRC_Continue; -} - -/* -** pExpr is a CHECK constraint on a row that is being UPDATE-ed. The -** only columns that are modified by the UPDATE are those for which -** aiChng[i]>=0, and also the ROWID is modified if chngRowid is true. -** -** Return true if CHECK constraint pExpr does not use any of the -** changing columns (or the rowid if it is changing). In other words, -** return true if this CHECK constraint can be skipped when validating -** the new row in the UPDATE statement. -*/ -static int checkConstraintUnchanged(Expr *pExpr, int *aiChng, int chngRowid){ - Walker w; - memset(&w, 0, sizeof(w)); - w.eCode = 0; - w.xExprCallback = checkConstraintExprNode; - w.u.aiCol = aiChng; - sqlite3WalkExpr(&w, pExpr); - if( !chngRowid ){ - testcase( (w.eCode & CKCNSTRNT_ROWID)!=0 ); - w.eCode &= ~CKCNSTRNT_ROWID; - } - testcase( w.eCode==0 ); - testcase( w.eCode==CKCNSTRNT_COLUMN ); - testcase( w.eCode==CKCNSTRNT_ROWID ); - testcase( w.eCode==(CKCNSTRNT_ROWID|CKCNSTRNT_COLUMN) ); - return !w.eCode; -} - /* ** Generate code to do constraint checks prior to an INSERT or an UPDATE ** on table pTab. @@ -111697,8 +103759,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( u8 pkChng, /* Non-zero if the rowid or PRIMARY KEY changed */ u8 overrideError, /* Override onError to this if not OE_Default */ int ignoreDest, /* Jump to this label on an OE_Ignore resolution */ - int *pbMayReplace, /* OUT: Set to true if constraint may cause a replace */ - int *aiChng /* column i is unchanged if aiChng[i]<0 */ + int *pbMayReplace /* OUT: Set to true if constraint may cause a replace */ ){ Vdbe *v; /* VDBE under constrution */ Index *pIdx; /* Pointer to one of the indices */ @@ -111708,7 +103769,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int ix; /* Index loop counter */ int nCol; /* Number of columns */ int onError; /* Conflict resolution strategy */ - int addr1; /* Address of jump instruction */ + int j1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ int ipkTop = 0; /* Top of the rowid change constraint check */ @@ -111744,14 +103805,10 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( */ for(i=0; iiPKey ){ - continue; /* ROWID is never NULL */ - } - if( aiChng && aiChng[i]<0 ){ - /* Don't bother checking for NOT NULL on columns that do not change */ continue; } onError = pTab->aCol[i].notNull; - if( onError==OE_None ) continue; /* This column is allowed to be NULL */ + if( onError==OE_None ) continue; if( overrideError!=OE_Default ){ onError = overrideError; }else if( onError==OE_Default ){ @@ -111783,10 +103840,9 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( } default: { assert( onError==OE_Replace ); - addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); - VdbeCoverage(v); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); VdbeCoverage(v); sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeJumpHere(v, j1); break; } } @@ -111800,13 +103856,10 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( pParse->ckBase = regNewData+1; onError = overrideError!=OE_Default ? overrideError : OE_Abort; for(i=0; inExpr; i++){ - int allOk; - Expr *pExpr = pCheck->a[i].pExpr; - if( aiChng && checkConstraintUnchanged(pExpr, aiChng, pkChng) ) continue; - allOk = sqlite3VdbeMakeLabel(v); - sqlite3ExprIfTrue(pParse, pExpr, allOk, SQLITE_JUMPIFNULL); + int allOk = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); if( onError==OE_Ignore ){ - sqlite3VdbeGoto(v, ignoreDest); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); }else{ char *zName = pCheck->a[i].zName; if( zName==0 ) zName = pTab->zName; @@ -111904,29 +103957,17 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regNewData, 1, 0, OE_Replace, 1, -1); - }else{ -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK - if( HasRowid(pTab) ){ - /* This OP_Delete opcode fires the pre-update-hook only. It does - ** not modify the b-tree. It is more efficient to let the coming - ** OP_Insert replace the existing entry than it is to delete the - ** existing entry and then insert a new one. */ - sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP); - sqlite3VdbeChangeP4(v, -1, (char *)pTab, P4_TABLE); - } -#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ - if( pTab->pIndex ){ - sqlite3MultiWrite(pParse); - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1); - } + regNewData, 1, 0, OE_Replace, 1); + }else if( pTab->pIndex ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); } seenReplace = 1; break; } case OE_Ignore: { /*assert( seenReplace==0 );*/ - sqlite3VdbeGoto(v, ignoreDest); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); break; } } @@ -111962,8 +104003,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]); pParse->ckBase = regNewData+1; - sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, addrUniqueOk, - SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, addrUniqueOk, + SQLITE_JUMPIFNULL); pParse->ckBase = 0; } @@ -111974,22 +104015,15 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( for(i=0; inColumn; i++){ int iField = pIdx->aiColumn[i]; int x; - if( iField==XN_EXPR ){ - pParse->ckBase = regNewData+1; - sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); - pParse->ckBase = 0; - VdbeComment((v, "%s column %d", pIdx->zName, i)); + if( iField<0 || iField==pTab->iPKey ){ + if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ + x = regNewData; + regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; }else{ - if( iField==XN_ROWID || iField==pTab->iPKey ){ - if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ - x = regNewData; - regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; - }else{ - x = iField + regNewData + 1; - } - sqlite3VdbeAddOp2(v, iField<0 ? OP_IntCopy : OP_SCopy, x, regIdx+i); - VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); + x = iField + regNewData + 1; } + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); @@ -112039,7 +104073,6 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** store it in registers regR..regR+nPk-1 */ if( pIdx!=pPk ){ for(i=0; inKeyCol; i++){ - assert( pPk->aiColumn[i]>=0 ); x = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); VdbeComment((v, "%s.%s", pTab->zName, @@ -112061,7 +104094,6 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( for(i=0; inKeyCol; i++){ char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); x = pPk->aiColumn[i]; - assert( x>=0 ); if( i==(pPk->nKeyCol-1) ){ addrJump = addrUniqueOk; op = OP_Eq; @@ -112088,7 +104120,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( break; } case OE_Ignore: { - sqlite3VdbeGoto(v, ignoreDest); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); break; } default: { @@ -112099,8 +104131,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regR, nPkField, 0, OE_Replace, - (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), -1); + regR, nPkField, 0, OE_Replace, pIdx==pPk); seenReplace = 1; break; } @@ -112110,7 +104141,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); } if( ipkTop ){ - sqlite3VdbeGoto(v, ipkTop+1); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ipkTop+1); sqlite3VdbeJumpHere(v, ipkBottom); } @@ -112163,7 +104194,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; } - sqlite3VdbeChangeP5(v, pik_flags); + if( pik_flags ) sqlite3VdbeChangeP5(v, pik_flags); } if( !HasRowid(pTab) ) return; regData = regNewData + 1; @@ -112185,7 +104216,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( } sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, regRec, regNewData); if( !pParse->nested ){ - sqlite3VdbeChangeP4(v, -1, (char *)pTab, P4_TABLE); + sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); } sqlite3VdbeChangeP5(v, pik_flags); } @@ -112215,7 +104246,6 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( Parse *pParse, /* Parsing context */ Table *pTab, /* Table to be opened */ int op, /* OP_OpenRead or OP_OpenWrite */ - u8 p5, /* P5 value for OP_Open* opcodes (except on WITHOUT ROWID) */ int iBase, /* Use this for the table cursor, if there is one */ u8 *aToOpen, /* If not NULL: boolean for each table and index */ int *piDataCur, /* Write the database source cursor number here */ @@ -112228,7 +104258,6 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( Vdbe *v; assert( op==OP_OpenRead || op==OP_OpenWrite ); - assert( op==OP_OpenWrite || p5==0 ); if( IsVirtual(pTab) ){ /* This routine is a no-op for virtual tables. Leave the output ** variables *piDataCur and *piIdxCur uninitialized so that valgrind @@ -112250,14 +104279,12 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ int iIdxCur = iBase++; assert( pIdx->pSchema==pTab->pSchema ); - if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - if( piDataCur ) *piDataCur = iIdxCur; - p5 = 0; + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) && piDataCur ){ + *piDataCur = iIdxCur; } if( aToOpen==0 || aToOpen[i+1] ){ sqlite3VdbeAddOp3(v, op, iIdxCur, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - sqlite3VdbeChangeP5(v, p5); VdbeComment((v, "%s", pIdx->zName)); } } @@ -112278,6 +104305,20 @@ SQLITE_API int sqlite3_xferopt_count; #ifndef SQLITE_OMIT_XFER_OPT +/* +** Check to collation names to see if they are compatible. +*/ +static int xferCompatibleCollation(const char *z1, const char *z2){ + if( z1==0 ){ + return z2==0; + } + if( z2==0 ){ + return 0; + } + return sqlite3StrICmp(z1, z2)==0; +} + + /* ** Check to see if index pSrc is compatible as a source of data ** for index pDest in an insert transfer optimization. The rules @@ -112303,17 +104344,10 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ return 0; /* Different columns indexed */ } - if( pSrc->aiColumn[i]==XN_EXPR ){ - assert( pSrc->aColExpr!=0 && pDest->aColExpr!=0 ); - if( sqlite3ExprCompare(pSrc->aColExpr->a[i].pExpr, - pDest->aColExpr->a[i].pExpr, -1)!=0 ){ - return 0; /* Different expressions in the index */ - } - } if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ return 0; /* Different sort orders */ } - if( sqlite3_stricmp(pSrc->azColl[i],pDest->azColl[i])!=0 ){ + if( !xferCompatibleCollation(pSrc->azColl[i],pDest->azColl[i]) ){ return 0; /* Different collating sequences */ } } @@ -112428,7 +104462,7 @@ static int xferOptimization( return 0; /* The result set must have exactly one column */ } assert( pEList->a[0].pExpr ); - if( pEList->a[0].pExpr->op!=TK_ASTERISK ){ + if( pEList->a[0].pExpr->op!=TK_ALL ){ return 0; /* The result set must be the special operator "*" */ } @@ -112464,32 +104498,21 @@ static int xferOptimization( for(i=0; inCol; i++){ Column *pDestCol = &pDest->aCol[i]; Column *pSrcCol = &pSrc->aCol[i]; -#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS - if( (db->flags & SQLITE_Vacuum)==0 - && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN - ){ - return 0; /* Neither table may have __hidden__ columns */ - } -#endif if( pDestCol->affinity!=pSrcCol->affinity ){ return 0; /* Affinity must be the same on all columns */ } - if( sqlite3_stricmp(pDestCol->zColl, pSrcCol->zColl)!=0 ){ + if( !xferCompatibleCollation(pDestCol->zColl, pSrcCol->zColl) ){ return 0; /* Collating sequence must be the same on all columns */ } if( pDestCol->notNull && !pSrcCol->notNull ){ return 0; /* tab2 must be NOT NULL if tab1 is */ } /* Default values for second and subsequent columns need to match. */ - if( i>0 ){ - assert( pDestCol->pDflt==0 || pDestCol->pDflt->op==TK_SPAN ); - assert( pSrcCol->pDflt==0 || pSrcCol->pDflt->op==TK_SPAN ); - if( (pDestCol->pDflt==0)!=(pSrcCol->pDflt==0) - || (pDestCol->pDflt && strcmp(pDestCol->pDflt->u.zToken, - pSrcCol->pDflt->u.zToken)!=0) - ){ - return 0; /* Default values must be the same for all columns */ - } + if( i>0 + && ((pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) + || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)!=0)) + ){ + return 0; /* Default values must be the same for all columns */ } } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ @@ -112564,7 +104587,7 @@ static int xferOptimization( ** (3) onError is something other than OE_Abort and OE_Rollback. */ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); VdbeCoverage(v); - emptyDestTest = sqlite3VdbeAddOp0(v, OP_Goto); + emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); sqlite3VdbeJumpHere(v, addr1); } if( HasRowid(pSrc) ){ @@ -112584,9 +104607,9 @@ static int xferOptimization( assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); - sqlite3VdbeAddOp4(v, OP_Insert, iDest, regData, regRowid, - (char*)pDest, P4_TABLE); + sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); + sqlite3VdbeChangeP4(v, -1, pDest->zName, 0); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); @@ -112595,7 +104618,7 @@ static int xferOptimization( sqlite3TableLock(pParse, iDbSrc, pSrc->tnum, 0, pSrc->zName); } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ - u8 idxInsFlags = 0; + u8 useSeekResult = 0; for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; } @@ -112625,21 +104648,17 @@ static int xferOptimization( ** a VACUUM command. In that case keys may not be written in strictly ** sorted order. */ for(i=0; inColumn; i++){ - const char *zColl = pSrcIdx->azColl[i]; - assert( sqlite3_stricmp(sqlite3StrBINARY, zColl)!=0 - || sqlite3StrBINARY==zColl ); - if( sqlite3_stricmp(sqlite3StrBINARY, zColl) ) break; + char *zColl = pSrcIdx->azColl[i]; + assert( zColl!=0 ); + if( sqlite3_stricmp("BINARY", zColl) ) break; } if( i==pSrcIdx->nColumn ){ - idxInsFlags = OPFLAG_USESEEKRESULT; + useSeekResult = OPFLAG_USESEEKRESULT; sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); } } - if( !HasRowid(pSrc) && pDestIdx->idxType==2 ){ - idxInsFlags |= OPFLAG_NCHANGE; - } sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); - sqlite3VdbeChangeP5(v, idxInsFlags); + sqlite3VdbeChangeP5(v, useSeekResult); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); @@ -112649,7 +104668,6 @@ static int xferOptimization( sqlite3ReleaseTempReg(pParse, regRowid); sqlite3ReleaseTempReg(pParse, regData); if( emptyDestTest ){ - sqlite3AutoincrementEnd(pParse); sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); sqlite3VdbeJumpHere(v, emptyDestTest); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); @@ -112679,7 +104697,6 @@ static int xferOptimization( ** accessed by users of the library. */ -/* #include "sqliteInt.h" */ /* ** Execute SQL code. Return one of the SQLITE_ success/failure @@ -112691,7 +104708,7 @@ static int xferOptimization( ** argument to xCallback(). If xCallback=NULL then no callback ** is invoked, even for queries. */ -SQLITE_API int sqlite3_exec( +SQLITE_API int SQLITE_STDCALL sqlite3_exec( sqlite3 *db, /* The database on which the SQL executes */ const char *zSql, /* The SQL to be executed */ sqlite3_callback xCallback, /* Invoke this callback routine */ @@ -112754,7 +104771,7 @@ SQLITE_API int sqlite3_exec( for(i=0; imallocFailed = 1; goto exec_out; } } @@ -112795,7 +104812,7 @@ SQLITE_API int sqlite3_exec( if( *pzErrMsg ){ memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); }else{ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; sqlite3Error(db, SQLITE_NOMEM); } }else if( pzErrMsg ){ @@ -112846,9 +104863,10 @@ SQLITE_API int sqlite3_exec( ** as extensions by SQLite should #include this file instead of ** sqlite3.h. */ -#ifndef SQLITE3EXT_H -#define SQLITE3EXT_H -/* #include "sqlite3.h" */ +#ifndef _SQLITE3EXT_H_ +#define _SQLITE3EXT_H_ + +typedef struct sqlite3_api_routines sqlite3_api_routines; /* ** The following structure holds pointers to all of the SQLite API @@ -113096,35 +105114,8 @@ struct sqlite3_api_routines { void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64, void(*)(void*), unsigned char); int (*strglob)(const char*,const char*); - /* Version 3.8.11 and later */ - sqlite3_value *(*value_dup)(const sqlite3_value*); - void (*value_free)(sqlite3_value*); - int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); - int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); - /* Version 3.9.0 and later */ - unsigned int (*value_subtype)(sqlite3_value*); - void (*result_subtype)(sqlite3_context*,unsigned int); - /* Version 3.10.0 and later */ - int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int); - int (*strlike)(const char*,const char*,unsigned int); - int (*db_cacheflush)(sqlite3*); - /* Version 3.12.0 and later */ - int (*system_errno)(sqlite3*); - /* Version 3.14.0 and later */ - int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*); - char *(*expanded_sql)(sqlite3_stmt*); }; -/* -** This is the function signature used for all extension entry points. It -** is also defined in the file "loadext.c". -*/ -typedef int (*sqlite3_loadext_entry)( - sqlite3 *db, /* Handle to the database. */ - char **pzErrMsg, /* Used to set error string on failure. */ - const sqlite3_api_routines *pThunk /* Extension API function pointers. */ -); - /* ** The following macros redefine the API routines so that they are ** redirected through the global sqlite3_api structure. @@ -113136,7 +105127,7 @@ typedef int (*sqlite3_loadext_entry)( ** the API. So the redefinition macros are only valid if the ** SQLITE_CORE macros is undefined. */ -#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) +#ifndef SQLITE_CORE #define sqlite3_aggregate_context sqlite3_api->aggregate_context #ifndef SQLITE_OMIT_DEPRECATED #define sqlite3_aggregate_count sqlite3_api->aggregate_count @@ -113263,7 +105254,6 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_value_text16le sqlite3_api->value_text16le #define sqlite3_value_type sqlite3_api->value_type #define sqlite3_vmprintf sqlite3_api->vmprintf -#define sqlite3_vsnprintf sqlite3_api->vsnprintf #define sqlite3_overload_function sqlite3_api->overload_function #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 @@ -113354,26 +105344,9 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_result_blob64 sqlite3_api->result_blob64 #define sqlite3_result_text64 sqlite3_api->result_text64 #define sqlite3_strglob sqlite3_api->strglob -/* Version 3.8.11 and later */ -#define sqlite3_value_dup sqlite3_api->value_dup -#define sqlite3_value_free sqlite3_api->value_free -#define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 -#define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 -/* Version 3.9.0 and later */ -#define sqlite3_value_subtype sqlite3_api->value_subtype -#define sqlite3_result_subtype sqlite3_api->result_subtype -/* Version 3.10.0 and later */ -#define sqlite3_status64 sqlite3_api->status64 -#define sqlite3_strlike sqlite3_api->strlike -#define sqlite3_db_cacheflush sqlite3_api->db_cacheflush -/* Version 3.12.0 and later */ -#define sqlite3_system_errno sqlite3_api->system_errno -/* Version 3.14.0 and later */ -#define sqlite3_trace_v2 sqlite3_api->trace_v2 -#define sqlite3_expanded_sql sqlite3_api->expanded_sql -#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ - -#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) +#endif /* SQLITE_CORE */ + +#ifndef SQLITE_CORE /* This case when the file really is being compiled as a loadable ** extension */ # define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; @@ -113388,14 +105361,14 @@ typedef int (*sqlite3_loadext_entry)( # define SQLITE_EXTENSION_INIT3 /*no-op*/ #endif -#endif /* SQLITE3EXT_H */ +#endif /* _SQLITE3EXT_H_ */ /************** End of sqlite3ext.h ******************************************/ /************** Continuing where we left off in loadext.c ********************/ -/* #include "sqliteInt.h" */ /* #include */ #ifndef SQLITE_OMIT_LOAD_EXTENSION + /* ** Some API routines are omitted when various features are ** excluded from a build of SQLite. Substitute a NULL pointer @@ -113465,7 +105438,7 @@ typedef int (*sqlite3_loadext_entry)( # define sqlite3_enable_shared_cache 0 #endif -#if defined(SQLITE_OMIT_TRACE) || defined(SQLITE_OMIT_DEPRECATED) +#ifdef SQLITE_OMIT_TRACE # define sqlite3_profile 0 # define sqlite3_trace 0 #endif @@ -113485,10 +105458,6 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_blob_reopen 0 #endif -#if defined(SQLITE_OMIT_TRACE) -# define sqlite3_trace_v2 0 -#endif - /* ** The following structure contains pointers to all SQLite API routines. ** A pointer to this structure is passed into extensions when they are @@ -113780,24 +105749,7 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_reset_auto_extension, sqlite3_result_blob64, sqlite3_result_text64, - sqlite3_strglob, - /* Version 3.8.11 and later */ - (sqlite3_value*(*)(const sqlite3_value*))sqlite3_value_dup, - sqlite3_value_free, - sqlite3_result_zeroblob64, - sqlite3_bind_zeroblob64, - /* Version 3.9.0 and later */ - sqlite3_value_subtype, - sqlite3_result_subtype, - /* Version 3.10.0 and later */ - sqlite3_status64, - sqlite3_strlike, - sqlite3_db_cacheflush, - /* Version 3.12.0 and later */ - sqlite3_system_errno, - /* Version 3.14.0 and later */ - sqlite3_trace_v2, - sqlite3_expanded_sql + sqlite3_strglob }; /* @@ -113820,14 +105772,13 @@ static int sqlite3LoadExtension( ){ sqlite3_vfs *pVfs = db->pVfs; void *handle; - sqlite3_loadext_entry xInit; + int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); char *zErrmsg = 0; const char *zEntry; char *zAltEntry = 0; void **aHandle; u64 nMsg = 300 + sqlite3Strlen30(zFile); int ii; - int rc; /* Shared library endings to try if zFile cannot be loaded as written */ static const char *azEndings[] = { @@ -113846,9 +105797,8 @@ static int sqlite3LoadExtension( /* Ticket #1863. To avoid a creating security problems for older ** applications that relink against newer versions of SQLite, the ** ability to run load_extension is turned off by default. One - ** must call either sqlite3_enable_load_extension(db) or - ** sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, 1, 0) - ** to turn on extension loading. + ** must call sqlite3_enable_load_extension() to turn on extension + ** loading. Otherwise you get the following error. */ if( (db->flags & SQLITE_LoadExtension)==0 ){ if( pzErrMsg ){ @@ -113863,7 +105813,7 @@ static int sqlite3LoadExtension( #if SQLITE_OS_UNIX || SQLITE_OS_WIN for(ii=0; ii=0 && zFile[iFile]!='/'; iFile--){} @@ -113911,7 +105862,8 @@ static int sqlite3LoadExtension( } memcpy(zAltEntry+iEntry, "_init", 6); zEntry = zAltEntry; - xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + sqlite3OsDlSym(pVfs, handle, zEntry); } if( xInit==0 ){ if( pzErrMsg ){ @@ -113928,9 +105880,7 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } sqlite3_free(zAltEntry); - rc = xInit(db, &zErrmsg, &sqlite3Apis); - if( rc ){ - if( rc==SQLITE_OK_LOAD_PERMANENTLY ) return SQLITE_OK; + if( xInit(db, &zErrmsg, &sqlite3Apis) ){ if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } @@ -113942,7 +105892,7 @@ static int sqlite3LoadExtension( /* Append the new shared library handle to the db->aExtension array. */ aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1)); if( aHandle==0 ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } if( db->nExtension>0 ){ memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension); @@ -113953,7 +105903,7 @@ static int sqlite3LoadExtension( db->aExtension[db->nExtension++] = handle; return SQLITE_OK; } -SQLITE_API int sqlite3_load_extension( +SQLITE_API int SQLITE_STDCALL sqlite3_load_extension( sqlite3 *db, /* Load the extension into this database connection */ const char *zFile, /* Name of the shared library containing extension */ const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ @@ -113984,18 +105934,29 @@ SQLITE_PRIVATE void sqlite3CloseExtensions(sqlite3 *db){ ** Enable or disable extension loading. Extension loading is disabled by ** default so as not to open security holes in older applications. */ -SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ +SQLITE_API int SQLITE_STDCALL sqlite3_enable_load_extension(sqlite3 *db, int onoff){ sqlite3_mutex_enter(db->mutex); if( onoff ){ - db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; + db->flags |= SQLITE_LoadExtension; }else{ - db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc); + db->flags &= ~SQLITE_LoadExtension; } sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } -#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */ +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ + +/* +** The auto-extension code added regardless of whether or not extension +** loading is supported. We need a dummy sqlite3Apis pointer for that +** code if regular extension loading is not available. This is that +** dummy pointer. +*/ +#ifdef SQLITE_OMIT_LOAD_EXTENSION +static const sqlite3_api_routines sqlite3Apis = { 0 }; +#endif + /* ** The following object holds the list of automatically loaded @@ -114030,9 +105991,7 @@ static SQLITE_WSD struct sqlite3AutoExtList { ** Register a statically linked extension that is automatically ** loaded by every new database connection. */ -SQLITE_API int sqlite3_auto_extension( - void (*xInit)(void) -){ +SQLITE_API int SQLITE_STDCALL sqlite3_auto_extension(void (*xInit)(void)){ int rc = SQLITE_OK; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); @@ -114055,7 +106014,7 @@ SQLITE_API int sqlite3_auto_extension( void (**aNew)(void); aNew = sqlite3_realloc64(wsdAutoext.aExt, nByte); if( aNew==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; }else{ wsdAutoext.aExt = aNew; wsdAutoext.aExt[wsdAutoext.nExt] = xInit; @@ -114077,9 +106036,7 @@ SQLITE_API int sqlite3_auto_extension( ** Return 1 if xInit was found on the list and removed. Return 0 if xInit ** was not on the list. */ -SQLITE_API int sqlite3_cancel_auto_extension( - void (*xInit)(void) -){ +SQLITE_API int SQLITE_STDCALL sqlite3_cancel_auto_extension(void (*xInit)(void)){ #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); #endif @@ -114102,7 +106059,7 @@ SQLITE_API int sqlite3_cancel_auto_extension( /* ** Reset the automatic extension loading mechanism. */ -SQLITE_API void sqlite3_reset_auto_extension(void){ +SQLITE_API void SQLITE_STDCALL sqlite3_reset_auto_extension(void){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize()==SQLITE_OK ) #endif @@ -114128,7 +106085,7 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ u32 i; int go = 1; int rc; - sqlite3_loadext_entry xInit; + int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); wsdAutoextInit; if( wsdAutoext.nExt==0 ){ @@ -114139,22 +106096,18 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ char *zErrmsg; #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); -#endif -#ifdef SQLITE_OMIT_LOAD_EXTENSION - const sqlite3_api_routines *pThunk = 0; -#else - const sqlite3_api_routines *pThunk = &sqlite3Apis; #endif sqlite3_mutex_enter(mutex); if( i>=wsdAutoext.nExt ){ xInit = 0; go = 0; }else{ - xInit = (sqlite3_loadext_entry)wsdAutoext.aExt[i]; + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + wsdAutoext.aExt[i]; } sqlite3_mutex_leave(mutex); zErrmsg = 0; - if( xInit && (rc = xInit(db, &zErrmsg, pThunk))!=0 ){ + if( xInit && (rc = xInit(db, &zErrmsg, &sqlite3Apis))!=0 ){ sqlite3ErrorWithMsg(db, rc, "automatic extension loading failed: %s", zErrmsg); go = 0; @@ -114178,7 +106131,6 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ ************************************************************************* ** This file contains code used to implement the PRAGMA command. */ -/* #include "sqliteInt.h" */ #if !defined(SQLITE_ENABLE_LOCKING_STYLE) # if defined(__APPLE__) @@ -114207,44 +106159,43 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ #define PragTyp_FLAG 2 #define PragTyp_BUSY_TIMEOUT 3 #define PragTyp_CACHE_SIZE 4 -#define PragTyp_CACHE_SPILL 5 -#define PragTyp_CASE_SENSITIVE_LIKE 6 -#define PragTyp_COLLATION_LIST 7 -#define PragTyp_COMPILE_OPTIONS 8 -#define PragTyp_DATA_STORE_DIRECTORY 9 -#define PragTyp_DATABASE_LIST 10 -#define PragTyp_DEFAULT_CACHE_SIZE 11 -#define PragTyp_ENCODING 12 -#define PragTyp_FOREIGN_KEY_CHECK 13 -#define PragTyp_FOREIGN_KEY_LIST 14 -#define PragTyp_INCREMENTAL_VACUUM 15 -#define PragTyp_INDEX_INFO 16 -#define PragTyp_INDEX_LIST 17 -#define PragTyp_INTEGRITY_CHECK 18 -#define PragTyp_JOURNAL_MODE 19 -#define PragTyp_JOURNAL_SIZE_LIMIT 20 -#define PragTyp_LOCK_PROXY_FILE 21 -#define PragTyp_LOCKING_MODE 22 -#define PragTyp_PAGE_COUNT 23 -#define PragTyp_MMAP_SIZE 24 -#define PragTyp_PAGE_SIZE 25 -#define PragTyp_SECURE_DELETE 26 -#define PragTyp_SHRINK_MEMORY 27 -#define PragTyp_SOFT_HEAP_LIMIT 28 -#define PragTyp_STATS 29 -#define PragTyp_SYNCHRONOUS 30 -#define PragTyp_TABLE_INFO 31 -#define PragTyp_TEMP_STORE 32 -#define PragTyp_TEMP_STORE_DIRECTORY 33 -#define PragTyp_THREADS 34 -#define PragTyp_WAL_AUTOCHECKPOINT 35 -#define PragTyp_WAL_CHECKPOINT 36 -#define PragTyp_ACTIVATE_EXTENSIONS 37 -#define PragTyp_HEXKEY 38 -#define PragTyp_KEY 39 -#define PragTyp_REKEY 40 -#define PragTyp_LOCK_STATUS 41 -#define PragTyp_PARSER_TRACE 42 +#define PragTyp_CASE_SENSITIVE_LIKE 5 +#define PragTyp_COLLATION_LIST 6 +#define PragTyp_COMPILE_OPTIONS 7 +#define PragTyp_DATA_STORE_DIRECTORY 8 +#define PragTyp_DATABASE_LIST 9 +#define PragTyp_DEFAULT_CACHE_SIZE 10 +#define PragTyp_ENCODING 11 +#define PragTyp_FOREIGN_KEY_CHECK 12 +#define PragTyp_FOREIGN_KEY_LIST 13 +#define PragTyp_INCREMENTAL_VACUUM 14 +#define PragTyp_INDEX_INFO 15 +#define PragTyp_INDEX_LIST 16 +#define PragTyp_INTEGRITY_CHECK 17 +#define PragTyp_JOURNAL_MODE 18 +#define PragTyp_JOURNAL_SIZE_LIMIT 19 +#define PragTyp_LOCK_PROXY_FILE 20 +#define PragTyp_LOCKING_MODE 21 +#define PragTyp_PAGE_COUNT 22 +#define PragTyp_MMAP_SIZE 23 +#define PragTyp_PAGE_SIZE 24 +#define PragTyp_SECURE_DELETE 25 +#define PragTyp_SHRINK_MEMORY 26 +#define PragTyp_SOFT_HEAP_LIMIT 27 +#define PragTyp_STATS 28 +#define PragTyp_SYNCHRONOUS 29 +#define PragTyp_TABLE_INFO 30 +#define PragTyp_TEMP_STORE 31 +#define PragTyp_TEMP_STORE_DIRECTORY 32 +#define PragTyp_THREADS 33 +#define PragTyp_WAL_AUTOCHECKPOINT 34 +#define PragTyp_WAL_CHECKPOINT 35 +#define PragTyp_ACTIVATE_EXTENSIONS 36 +#define PragTyp_HEXKEY 37 +#define PragTyp_KEY 38 +#define PragTyp_REKEY 39 +#define PragTyp_LOCK_STATUS 40 +#define PragTyp_PARSER_TRACE 41 #define PragFlag_NeedSchema 0x01 #define PragFlag_ReadOnly 0x02 static const struct sPragmaNames { @@ -114291,18 +106242,14 @@ static const struct sPragmaNames { #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "cache_spill", - /* ePragTyp: */ PragTyp_CACHE_SPILL, + /* ePragTyp: */ PragTyp_FLAG, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ SQLITE_CacheSpill }, #endif { /* zName: */ "case_sensitive_like", /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, /* ePragFlag: */ 0, /* iArg: */ 0 }, - { /* zName: */ "cell_size_check", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_CellSizeCk }, #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "checkpoint_fullfsync", /* ePragTyp: */ PragTyp_FLAG, @@ -114507,7 +106454,7 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif -#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_PARSER_TRACE) +#if defined(SQLITE_DEBUG) { /* zName: */ "parser_trace", /* ePragTyp: */ PragTyp_PARSER_TRACE, /* ePragFlag: */ 0, @@ -114660,15 +106607,15 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 60 on by default, 73 total. */ +/* Number of pragmas: 59 on by default, 72 total. */ /************** End of pragma.h **********************************************/ /************** Continuing where we left off in pragma.c *********************/ /* ** Interpret the given string as a safety level. Return 0 for OFF, -** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or -** unrecognized string argument. The FULL and EXTRA option is disallowed +** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or +** unrecognized string argument. The FULL option is disallowed ** if the omitFull parameter it 1. ** ** Note that the values returned are one less that the values that @@ -114677,21 +106624,18 @@ static const struct sPragmaNames { ** and older scripts may have used numbers 0 for OFF and 1 for ON. */ static u8 getSafetyLevel(const char *z, int omitFull, u8 dflt){ - /* 123456789 123456789 123 */ - static const char zText[] = "onoffalseyestruextrafull"; - static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 15, 20}; - static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 5, 4}; - static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 3, 2}; - /* on no off false yes true extra full */ + /* 123456789 123456789 */ + static const char zText[] = "onoffalseyestruefull"; + static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; + static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 4}; + static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2}; int i, n; if( sqlite3Isdigit(*z) ){ return (u8)sqlite3Atoi(z); } n = sqlite3Strlen30(z); - for(i=0; inMem; + i64 *pI64 = sqlite3DbMallocRaw(pParse->db, sizeof(value)); + if( pI64 ){ + memcpy(pI64, &value, sizeof(value)); } + sqlite3VdbeAddOp4(v, OP_Int64, 0, nMem, 0, (char*)pI64, P4_INT64); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, nMem, 1); } @@ -114917,7 +106835,7 @@ SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){ ** ** Pragmas are of this form: ** -** PRAGMA [schema.]id [= value] +** PRAGMA [database.]id [= value] ** ** The identifier might also be a string. The value is a string, and ** identifier, or a number. If minusFlag is true, then the value is @@ -114929,8 +106847,8 @@ SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){ */ SQLITE_PRIVATE void sqlite3Pragma( Parse *pParse, - Token *pId1, /* First part of [schema.]id field */ - Token *pId2, /* Second part of [schema.]id field, or NULL */ + Token *pId1, /* First part of [database.]id field */ + Token *pId2, /* Second part of [database.]id field, or NULL */ Token *pValue, /* Token for , or NULL */ int minusFlag /* True if a '-' sign preceded */ ){ @@ -114956,7 +106874,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeRunOnlyOnce(v); pParse->nMem = 2; - /* Interpret the [schema.] part of the pragma statement. iDb is the + /* Interpret the [database.] part of the pragma statement. iDb is the ** index of the database this pragma is being applied to in db.aDb[]. */ iDb = sqlite3TwoPartName(pParse, pId1, pId2, &pId); if( iDb<0 ) return; @@ -114978,7 +106896,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } assert( pId2 ); - zDb = pId2->n>0 ? pDb->zDbSName : 0; + zDb = pId2->n>0 ? pDb->zName : 0; if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ goto pragma_out; } @@ -115005,8 +106923,14 @@ SQLITE_PRIVATE void sqlite3Pragma( db->busyHandler.nBusy = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ - returnSingleText(v, "result", aFcntl[0]); - sqlite3_free(aFcntl[0]); + if( aFcntl[0] ){ + int nMem = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_String8, 0, nMem, 0, aFcntl[0], 0); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "result", SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, nMem, 1); + sqlite3_free(aFcntl[0]); + } goto pragma_out; } if( rc!=SQLITE_NOTFOUND ){ @@ -115055,8 +106979,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* - ** PRAGMA [schema.]default_cache_size - ** PRAGMA [schema.]default_cache_size=N + ** PRAGMA [database.]default_cache_size + ** PRAGMA [database.]default_cache_size=N ** ** The first form reports the current persistent setting for the ** page cache size. The value returned is the maximum number of @@ -115083,21 +107007,21 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_Noop, 0, 0, 0}, { OP_ResultRow, 1, 1, 0}, }; - VdbeOp *aOp; + int addr; sqlite3VdbeUsesBtree(v, iDb); if( !zRight ){ - setOneColumnName(v, "cache_size"); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", SQLITE_STATIC); pParse->nMem += 2; - sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(getCacheSize)); - aOp = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize, iLn); - if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; - aOp[0].p1 = iDb; - aOp[1].p1 = iDb; - aOp[6].p1 = SQLITE_DEFAULT_CACHE_SIZE; + addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize,iLn); + sqlite3VdbeChangeP1(v, addr, iDb); + sqlite3VdbeChangeP1(v, addr+1, iDb); + sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE); }else{ int size = sqlite3AbsInt32(sqlite3Atoi(zRight)); sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, size); + sqlite3VdbeAddOp2(v, OP_Integer, size, 1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, 1); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); @@ -115108,8 +107032,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) /* - ** PRAGMA [schema.]page_size - ** PRAGMA [schema.]page_size=N + ** PRAGMA [database.]page_size + ** PRAGMA [database.]page_size=N ** ** The first form reports the current setting for the ** database page size in bytes. The second form sets the @@ -115121,22 +107045,22 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( pBt!=0 ); if( !zRight ){ int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; - returnSingleInt(v, "page_size", size); + returnSingleInt(pParse, "page_size", size); }else{ /* Malloc may fail when setting the page-size, as there is an internal ** buffer that the pager module resizes using sqlite3_realloc(). */ db->nextPagesize = sqlite3Atoi(zRight); if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,-1,0) ){ - sqlite3OomFault(db); + db->mallocFailed = 1; } } break; } /* - ** PRAGMA [schema.]secure_delete - ** PRAGMA [schema.]secure_delete=ON/OFF + ** PRAGMA [database.]secure_delete + ** PRAGMA [database.]secure_delete=ON/OFF ** ** The first form reports the current setting for the ** secure_delete flag. The second form changes the secure_delete @@ -115156,13 +107080,13 @@ SQLITE_PRIVATE void sqlite3Pragma( } } b = sqlite3BtreeSecureDelete(pBt, b); - returnSingleInt(v, "secure_delete", b); + returnSingleInt(pParse, "secure_delete", b); break; } /* - ** PRAGMA [schema.]max_page_count - ** PRAGMA [schema.]max_page_count=N + ** PRAGMA [database.]max_page_count + ** PRAGMA [database.]max_page_count=N ** ** The first form reports the current setting for the ** maximum number of pages in the database file. The @@ -115173,7 +107097,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** change. The only purpose is to provide an easy way to test ** the sqlite3AbsInt32() function. ** - ** PRAGMA [schema.]page_count + ** PRAGMA [database.]page_count ** ** Return the number of pages in the specified database. */ @@ -115194,8 +107118,8 @@ SQLITE_PRIVATE void sqlite3Pragma( } /* - ** PRAGMA [schema.]locking_mode - ** PRAGMA [schema.]locking_mode = (normal|exclusive) + ** PRAGMA [database.]locking_mode + ** PRAGMA [database.]locking_mode = (normal|exclusive) */ case PragTyp_LOCKING_MODE: { const char *zRet = "normal"; @@ -115235,20 +107159,25 @@ SQLITE_PRIVATE void sqlite3Pragma( if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ zRet = "exclusive"; } - returnSingleText(v, "locking_mode", zRet); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "locking_mode", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zRet, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); break; } /* - ** PRAGMA [schema.]journal_mode - ** PRAGMA [schema.]journal_mode = + ** PRAGMA [database.]journal_mode + ** PRAGMA [database.]journal_mode = ** (delete|persist|off|truncate|memory|wal|off) */ case PragTyp_JOURNAL_MODE: { int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ int ii; /* Loop counter */ - setOneColumnName(v, "journal_mode"); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); + if( zRight==0 ){ /* If there is no "=MODE" part of the pragma, do a query for the ** current mode */ @@ -115281,8 +107210,8 @@ SQLITE_PRIVATE void sqlite3Pragma( } /* - ** PRAGMA [schema.]journal_size_limit - ** PRAGMA [schema.]journal_size_limit=N + ** PRAGMA [database.]journal_size_limit + ** PRAGMA [database.]journal_size_limit=N ** ** Get or set the size limit on rollback journal files. */ @@ -115294,15 +107223,15 @@ SQLITE_PRIVATE void sqlite3Pragma( if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); - returnSingleInt(v, "journal_size_limit", iLimit); + returnSingleInt(pParse, "journal_size_limit", iLimit); break; } #endif /* SQLITE_OMIT_PAGER_PRAGMAS */ /* - ** PRAGMA [schema.]auto_vacuum - ** PRAGMA [schema.]auto_vacuum=N + ** PRAGMA [database.]auto_vacuum + ** PRAGMA [database.]auto_vacuum=N ** ** Get or set the value of the database 'auto-vacuum' parameter. ** The value is one of: 0 NONE 1 FULL 2 INCREMENTAL @@ -115312,7 +107241,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Btree *pBt = pDb->pBt; assert( pBt!=0 ); if( !zRight ){ - returnSingleInt(v, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); + returnSingleInt(pParse, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); }else{ int eAuto = getAutoVacuum(zRight); assert( eAuto>=0 && eAuto<=2 ); @@ -115335,18 +107264,16 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, { OP_If, 1, 0, 0}, /* 2 */ { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ - { OP_SetCookie, 0, BTREE_INCR_VACUUM, 0}, /* 4 */ + { OP_Integer, 0, 1, 0}, /* 4 */ + { OP_SetCookie, 0, BTREE_INCR_VACUUM, 1}, /* 5 */ }; - VdbeOp *aOp; - int iAddr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setMeta6)); - aOp = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); - if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; - aOp[0].p1 = iDb; - aOp[1].p1 = iDb; - aOp[2].p2 = iAddr+4; - aOp[4].p1 = iDb; - aOp[4].p3 = eAuto - 1; + int iAddr; + iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); + sqlite3VdbeChangeP1(v, iAddr, iDb); + sqlite3VdbeChangeP1(v, iAddr+1, iDb); + sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4); + sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1); + sqlite3VdbeChangeP1(v, iAddr+5, iDb); sqlite3VdbeUsesBtree(v, iDb); } } @@ -115355,7 +107282,7 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif /* - ** PRAGMA [schema.]incremental_vacuum(N) + ** PRAGMA [database.]incremental_vacuum(N) ** ** Do N steps of incremental vacuuming on a database. */ @@ -115378,8 +107305,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_PAGER_PRAGMAS /* - ** PRAGMA [schema.]cache_size - ** PRAGMA [schema.]cache_size=N + ** PRAGMA [database.]cache_size + ** PRAGMA [database.]cache_size=N ** ** The first form reports the current local setting for the ** page cache size. The second form sets the local @@ -115391,7 +107318,7 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_CACHE_SIZE: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ - returnSingleInt(v, "cache_size", pDb->pSchema->cache_size); + returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); }else{ int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; @@ -115401,50 +107328,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } /* - ** PRAGMA [schema.]cache_spill - ** PRAGMA cache_spill=BOOLEAN - ** PRAGMA [schema.]cache_spill=N - ** - ** The first form reports the current local setting for the - ** page cache spill size. The second form turns cache spill on - ** or off. When turnning cache spill on, the size is set to the - ** current cache_size. The third form sets a spill size that - ** may be different form the cache size. - ** If N is positive then that is the - ** number of pages in the cache. If N is negative, then the - ** number of pages is adjusted so that the cache uses -N kibibytes - ** of memory. - ** - ** If the number of cache_spill pages is less then the number of - ** cache_size pages, no spilling occurs until the page count exceeds - ** the number of cache_size pages. - ** - ** The cache_spill=BOOLEAN setting applies to all attached schemas, - ** not just the schema specified. - */ - case PragTyp_CACHE_SPILL: { - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - if( !zRight ){ - returnSingleInt(v, "cache_spill", - (db->flags & SQLITE_CacheSpill)==0 ? 0 : - sqlite3BtreeSetSpillSize(pDb->pBt,0)); - }else{ - int size = 1; - if( sqlite3GetInt32(zRight, &size) ){ - sqlite3BtreeSetSpillSize(pDb->pBt, size); - } - if( sqlite3GetBoolean(zRight, size!=0) ){ - db->flags |= SQLITE_CacheSpill; - }else{ - db->flags &= ~SQLITE_CacheSpill; - } - setAllPagerFlags(db); - } - break; - } - - /* - ** PRAGMA [schema.]mmap_size(N) + ** PRAGMA [database.]mmap_size(N) ** ** Used to set mapping size limit. The mapping size limit is ** used to limit the aggregate size of all memory mapped regions of the @@ -115479,7 +107363,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = SQLITE_OK; #endif if( rc==SQLITE_OK ){ - returnSingleInt(v, "mmap_size", sz); + returnSingleInt(pParse, "mmap_size", sz); }else if( rc!=SQLITE_NOTFOUND ){ pParse->nErr++; pParse->rc = rc; @@ -115500,7 +107384,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TEMP_STORE: { if( !zRight ){ - returnSingleInt(v, "temp_store", db->temp_store); + returnSingleInt(pParse, "temp_store", db->temp_store); }else{ changeTempStorage(pParse, zRight); } @@ -115519,7 +107403,13 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TEMP_STORE_DIRECTORY: { if( !zRight ){ - returnSingleText(v, "temp_store_directory", sqlite3_temp_directory); + if( sqlite3_temp_directory ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, + "temp_store_directory", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_temp_directory, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -115563,7 +107453,13 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_DATA_STORE_DIRECTORY: { if( !zRight ){ - returnSingleText(v, "data_store_directory", sqlite3_data_directory); + if( sqlite3_data_directory ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, + "data_store_directory", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_data_directory, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -115588,8 +107484,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #if SQLITE_ENABLE_LOCKING_STYLE /* - ** PRAGMA [schema.]lock_proxy_file - ** PRAGMA [schema.]lock_proxy_file = ":auto:"|"lock_file_path" + ** PRAGMA [database.]lock_proxy_file + ** PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path" ** ** Return or set the value of the lock_proxy_file flag. Changing ** the value sets a specific file to be used for database access locks. @@ -115602,7 +107498,14 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3_file *pFile = sqlite3PagerFile(pPager); sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, &proxy_file_path); - returnSingleText(v, "lock_proxy_file", proxy_file_path); + + if( proxy_file_path ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, + "lock_proxy_file", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, proxy_file_path, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } }else{ Pager *pPager = sqlite3BtreePager(pDb->pBt); sqlite3_file *pFile = sqlite3PagerFile(pPager); @@ -115624,8 +107527,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif /* SQLITE_ENABLE_LOCKING_STYLE */ /* - ** PRAGMA [schema.]synchronous - ** PRAGMA [schema.]synchronous=OFF|ON|NORMAL|FULL|EXTRA + ** PRAGMA [database.]synchronous + ** PRAGMA [database.]synchronous=OFF|ON|NORMAL|FULL ** ** Return or set the local value of the synchronous flag. Changing ** the local value does not make changes to the disk file and the @@ -115634,7 +107537,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_SYNCHRONOUS: { if( !zRight ){ - returnSingleInt(v, "synchronous", pDb->safety_level-1); + returnSingleInt(pParse, "synchronous", pDb->safety_level-1); }else{ if( !db->autoCommit ){ sqlite3ErrorMsg(pParse, @@ -115643,7 +107546,6 @@ SQLITE_PRIVATE void sqlite3Pragma( int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; if( iLevel==0 ) iLevel = 1; pDb->safety_level = iLevel; - pDb->bSyncSet = 1; setAllPagerFlags(db); } } @@ -115654,7 +107556,7 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_FLAG_PRAGMAS case PragTyp_FLAG: { if( zRight==0 ){ - returnSingleInt(v, pPragma->zName, (db->flags & pPragma->iArg)!=0 ); + returnSingleInt(pParse, pPragma->zName, (db->flags & pPragma->iArg)!=0 ); }else{ int mask = pPragma->iArg; /* Mask of bits to set or clear. */ if( db->autoCommit==0 ){ @@ -115680,7 +107582,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** compiler (eg. count_changes). So add an opcode to expire all ** compiled SQL statements after modifying a pragma value. */ - sqlite3VdbeAddOp0(v, OP_Expire); + sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); setAllPagerFlags(db); } break; @@ -115702,24 +107604,37 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TABLE_INFO: if( zRight ){ Table *pTab; - pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); + pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ - static const char *azCol[] = { - "cid", "name", "type", "notnull", "dflt_value", "pk" - }; int i, k; int nHidden = 0; Column *pCol; Index *pPk = sqlite3PrimaryKeyIndex(pTab); + sqlite3VdbeSetNumCols(v, 6); pParse->nMem = 6; sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 6, azCol); assert( 6==ArraySize(azCol) ); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "notnull", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "dflt_value", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "pk", SQLITE_STATIC); sqlite3ViewGetColumnNames(pParse, pTab); for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ if( IsHiddenColumn(pCol) ){ nHidden++; continue; } + sqlite3VdbeAddOp2(v, OP_Integer, i-nHidden, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pCol->zName, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + pCol->zType ? pCol->zType : "", 0); + sqlite3VdbeAddOp2(v, OP_Integer, (pCol->notNull ? 1 : 0), 4); + if( pCol->zDflt ){ + sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, (char*)pCol->zDflt, 0); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, 5); + } if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ k = 0; }else if( pPk==0 ){ @@ -115727,14 +107642,7 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} } - assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN ); - sqlite3VdbeMultiLoad(v, 1, "issisi", - i-nHidden, - pCol->zName, - sqlite3ColumnType(pCol,""), - pCol->notNull ? 1 : 0, - pCol->pDflt ? pCol->pDflt->u.zToken : 0, - k); + sqlite3VdbeAddOp2(v, OP_Integer, k, 6); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } } @@ -115742,26 +107650,31 @@ SQLITE_PRIVATE void sqlite3Pragma( break; case PragTyp_STATS: { - static const char *azCol[] = { "table", "index", "width", "height" }; Index *pIdx; HashElem *i; v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 4); pParse->nMem = 4; sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "index", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "width", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "height", SQLITE_STATIC); for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); - sqlite3VdbeMultiLoad(v, 1, "ssii", - pTab->zName, - 0, - pTab->szTabRow, - pTab->nRowLogEst); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, pTab->zName, 0); + sqlite3VdbeAddOp2(v, OP_Null, 0, 2); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->szTabRow), 3); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeMultiLoad(v, 2, "sii", - pIdx->zName, - pIdx->szIdxRow, - pIdx->aiRowLogEst[0]); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } @@ -115773,9 +107686,6 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab; pIdx = sqlite3FindIndex(db, zRight, zDb); if( pIdx ){ - static const char *azCol[] = { - "seqno", "cid", "name", "desc", "coll", "key" - }; int i; int mx; if( pPragma->iArg ){ @@ -115788,18 +107698,29 @@ SQLITE_PRIVATE void sqlite3Pragma( pParse->nMem = 3; } pTab = pIdx->pTable; + sqlite3VdbeSetNumCols(v, pParse->nMem); sqlite3CodeVerifySchema(pParse, iDb); - assert( pParse->nMem<=ArraySize(azCol) ); - setAllColumnNames(v, pParse->nMem, azCol); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); + if( pPragma->iArg ){ + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "desc", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "coll", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "key", SQLITE_STATIC); + } for(i=0; iaiColumn[i]; - sqlite3VdbeMultiLoad(v, 1, "iis", i, cnum, - cnum<0 ? 0 : pTab->aCol[cnum].zName); + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp2(v, OP_Integer, cnum, 2); + if( cnum<0 ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, 3); + }else{ + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pTab->aCol[cnum].zName, 0); + } if( pPragma->iArg ){ - sqlite3VdbeMultiLoad(v, 4, "isi", - pIdx->aSortOrder[i], - pIdx->azColl[i], - inKeyCol); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->aSortOrder[i], 4); + sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, pIdx->azColl[i], 0); + sqlite3VdbeAddOp2(v, OP_Integer, inKeyCol, 6); } sqlite3VdbeAddOp2(v, OP_ResultRow, 1, pParse->nMem); } @@ -115813,21 +107734,22 @@ SQLITE_PRIVATE void sqlite3Pragma( int i; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ - static const char *azCol[] = { - "seq", "name", "unique", "origin", "partial" - }; v = sqlite3GetVdbe(pParse); + sqlite3VdbeSetNumCols(v, 5); pParse->nMem = 5; sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 5, azCol); assert( 5==ArraySize(azCol) ); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "origin", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "partial", SQLITE_STATIC); for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ const char *azOrigin[] = { "c", "u", "pk" }; - sqlite3VdbeMultiLoad(v, 1, "isisi", - i, - pIdx->zName, - IsUniqueIndex(pIdx), - azOrigin[pIdx->idxType], - pIdx->pPartIdxWhere!=0); + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); + sqlite3VdbeAddOp2(v, OP_Integer, IsUniqueIndex(pIdx), 3); + sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, azOrigin[pIdx->idxType], 0); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->pPartIdxWhere!=0, 5); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); } } @@ -115835,31 +107757,35 @@ SQLITE_PRIVATE void sqlite3Pragma( break; case PragTyp_DATABASE_LIST: { - static const char *azCol[] = { "seq", "name", "file" }; int i; + sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; - setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "file", SQLITE_STATIC); for(i=0; inDb; i++){ if( db->aDb[i].pBt==0 ) continue; - assert( db->aDb[i].zDbSName!=0 ); - sqlite3VdbeMultiLoad(v, 1, "iss", - i, - db->aDb[i].zDbSName, - sqlite3BtreeGetFilename(db->aDb[i].pBt)); + assert( db->aDb[i].zName!=0 ); + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, db->aDb[i].zName, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + sqlite3BtreeGetFilename(db->aDb[i].pBt), 0); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } } break; case PragTyp_COLLATION_LIST: { - static const char *azCol[] = { "seq", "name" }; int i = 0; HashElem *p; + sqlite3VdbeSetNumCols(v, 2); pParse->nMem = 2; - setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ CollSeq *pColl = (CollSeq *)sqliteHashData(p); - sqlite3VdbeMultiLoad(v, 1, "is", i++, pColl->zName); + sqlite3VdbeAddOp2(v, OP_Integer, i++, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pColl->zName, 0); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } } @@ -115875,26 +107801,33 @@ SQLITE_PRIVATE void sqlite3Pragma( v = sqlite3GetVdbe(pParse); pFK = pTab->pFKey; if( pFK ){ - static const char *azCol[] = { - "id", "seq", "table", "from", "to", "on_update", "on_delete", - "match" - }; int i = 0; + sqlite3VdbeSetNumCols(v, 8); pParse->nMem = 8; sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 8, azCol); assert( 8==ArraySize(azCol) ); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "from", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "to", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "on_update", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 6, COLNAME_NAME, "on_delete", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 7, COLNAME_NAME, "match", SQLITE_STATIC); while(pFK){ int j; for(j=0; jnCol; j++){ - sqlite3VdbeMultiLoad(v, 1, "iissssss", - i, - j, - pFK->zTo, - pTab->aCol[pFK->aCol[j].iFrom].zName, - pFK->aCol[j].zCol, - actionName(pFK->aAction[1]), /* ON UPDATE */ - actionName(pFK->aAction[0]), /* ON DELETE */ - "NONE"); + char *zCol = pFK->aCol[j].zCol; + char *zOnDelete = (char *)actionName(pFK->aAction[0]); + char *zOnUpdate = (char *)actionName(pFK->aAction[1]); + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp2(v, OP_Integer, j, 2); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pFK->zTo, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + pTab->aCol[pFK->aCol[j].iFrom].zName, 0); + sqlite3VdbeAddOp4(v, zCol ? OP_String8 : OP_Null, 0, 5, 0, zCol, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 6, 0, zOnUpdate, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 7, 0, zOnDelete, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 8, 0, "NONE", 0); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 8); } ++i; @@ -115923,14 +107856,17 @@ SQLITE_PRIVATE void sqlite3Pragma( int addrTop; /* Top of a loop checking foreign keys */ int addrOk; /* Jump here if the key is OK */ int *aiCols; /* child to parent column mapping */ - static const char *azCol[] = { "table", "rowid", "parent", "fkid" }; regResult = pParse->nMem+1; pParse->nMem += 4; regKey = ++pParse->nMem; regRow = ++pParse->nMem; v = sqlite3GetVdbe(pParse); - setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); + sqlite3VdbeSetNumCols(v, 4); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "rowid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "parent", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "fkid", SQLITE_STATIC); sqlite3CodeVerifySchema(pParse, iDb); k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); while( k ){ @@ -115945,7 +107881,8 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); - sqlite3VdbeLoadString(v, regResult, pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult, 0, pTab->zName, + P4_TRANSIENT); for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ pParent = sqlite3FindTable(db, pFK->zTo, zDb); if( pParent==0 ) continue; @@ -115984,11 +107921,13 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); sqlite3ColumnDefault(v, pTab, iKey, regRow); sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, + sqlite3VdbeCurrentAddr(v)+3); VdbeCoverage(v); }else{ sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); } - sqlite3VdbeAddOp3(v, OP_SeekRowid, i, 0, regRow); VdbeCoverage(v); - sqlite3VdbeGoto(v, addrOk); + sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); }else{ for(j=0; jnCol; j++){ @@ -115998,13 +107937,15 @@ SQLITE_PRIVATE void sqlite3Pragma( } if( pParent ){ sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, - sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); + sqlite3IndexAffinityStr(v,pIdx), pFK->nCol); sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); VdbeCoverage(v); } } sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); - sqlite3VdbeMultiLoad(v, regResult+2, "si", pFK->zTo, i-1); + sqlite3VdbeAddOp4(v, OP_String8, 0, regResult+2, 0, + pFK->zTo, P4_TRANSIENT); + sqlite3VdbeAddOp2(v, OP_Integer, i-1, regResult+3); sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); sqlite3VdbeResolveLabel(v, addrOk); sqlite3DbFree(db, aiCols); @@ -116021,7 +107962,7 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_PARSER_TRACE: { if( zRight ){ if( sqlite3GetBoolean(zRight, 0) ){ - sqlite3ParserTrace(stdout, "parser: "); + sqlite3ParserTrace(stderr, "parser: "); }else{ sqlite3ParserTrace(0, 0); } @@ -116052,6 +107993,17 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_INTEGRITY_CHECK: { int i, j, addr, mxErr; + /* Code that appears at the end of the integrity check. If no error + ** messages have been generated, output OK. Otherwise output the + ** error message + */ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList endCode[] = { + { OP_IfNeg, 1, 0, 0}, /* 0 */ + { OP_String8, 0, 3, 0}, /* 1 */ + { OP_ResultRow, 3, 1, 0}, + }; + int isQuick = (sqlite3Tolower(zLeft[0])=='q'); /* If the PRAGMA command was of the form "PRAGMA .integrity_check", @@ -116069,7 +108021,8 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Initialize the VDBE program */ pParse->nMem = 6; - setOneColumnName(v, "integrity_check"); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", SQLITE_STATIC); /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; @@ -116085,10 +108038,7 @@ SQLITE_PRIVATE void sqlite3Pragma( for(i=0; inDb; i++){ HashElem *x; Hash *pTbls; - int *aRoot; int cnt = 0; - int mxIdx = 0; - int nIdx; if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; @@ -116101,39 +108051,35 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Do an integrity check of the B-Tree ** - ** Begin by finding the root pages numbers + ** Begin by filling registers 2, 3, ... with the root pages numbers ** for all tables and indices in the database. */ assert( sqlite3SchemaMutexHeld(db, i, 0) ); pTbls = &db->aDb[i].pSchema->tblHash; - for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; - if( HasRowid(pTab) ) cnt++; - for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } - if( nIdx>mxIdx ) mxIdx = nIdx; - } - aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(cnt+1)); - if( aRoot==0 ) break; - for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ - Table *pTab = sqliteHashData(x); - Index *pIdx; - if( HasRowid(pTab) ) aRoot[cnt++] = pTab->tnum; + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Integer, pTab->tnum, 2+cnt); + VdbeComment((v, "%s", pTab->zName)); + cnt++; + } for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - aRoot[cnt++] = pIdx->tnum; + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->tnum, 2+cnt); + VdbeComment((v, "%s", pIdx->zName)); + cnt++; } } - aRoot[cnt] = 0; /* Make sure sufficient number of registers have been allocated */ - pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); + pParse->nMem = MAX( pParse->nMem, cnt+8 ); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); + sqlite3VdbeAddOp3(v, OP_IntegrityCk, 2, cnt, 1); sqlite3VdbeChangeP5(v, (u8)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName), + sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), P4_DYNAMIC); sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); @@ -116157,14 +108103,13 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeJumpHere(v, addr); sqlite3ExprCacheClear(pParse); - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 1, 0, &iDataCur, &iIdxCur); sqlite3VdbeAddOp2(v, OP_Integer, 0, 7); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ } - assert( pParse->nMem>=8+j ); - assert( sqlite3NoTempsInRange(pParse,1,7+j) ); + pParse->nMem = MAX(pParse->nMem, 8+j); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); /* Verify that all NOT NULL columns really are NOT NULL */ @@ -116199,11 +108144,13 @@ SQLITE_PRIVATE void sqlite3Pragma( jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeLoadString(v, 3, "row "); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, "row ", P4_STATIC); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); - sqlite3VdbeLoadString(v, 4, " missing from index "); + sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + " missing from index ", P4_STATIC); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); + jmp5 = sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + pIdx->zName, P4_TRANSIENT); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); @@ -116218,19 +108165,20 @@ SQLITE_PRIVATE void sqlite3Pragma( int kk; for(kk=0; kknKeyCol; kk++){ int iCol = pIdx->aiColumn[kk]; - assert( iCol!=XN_ROWID && iColnCol ); - if( iCol>=0 && pTab->aCol[iCol].notNull ) continue; + assert( iCol>=0 && iColnCol ); + if( pTab->aCol[iCol].notNull ) continue; sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); VdbeCoverage(v); } jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); - sqlite3VdbeGoto(v, uniqOk); + sqlite3VdbeAddOp2(v, OP_Goto, 0, uniqOk); sqlite3VdbeJumpHere(v, jmp6); sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, pIdx->nKeyCol); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); - sqlite3VdbeGoto(v, jmp5); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + "non-unique entry in index ", P4_STATIC); + sqlite3VdbeAddOp2(v, OP_Goto, 0, jmp5); sqlite3VdbeResolveLabel(v, uniqOk); } sqlite3VdbeJumpHere(v, jmp4); @@ -116239,7 +108187,8 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); #ifndef SQLITE_OMIT_BTREECOUNT - sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, + "wrong # of entries in index ", P4_STATIC); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ if( pPk==pIdx ) continue; addr = sqlite3VdbeCurrentAddr(v); @@ -116249,30 +108198,17 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeLoadString(v, 3, pIdx->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pIdx->zName, P4_TRANSIENT); sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); } #endif /* SQLITE_OMIT_BTREECOUNT */ } } - { - static const int iLn = VDBE_OFFSET_LINENO(2); - static const VdbeOpList endCode[] = { - { OP_AddImm, 1, 0, 0}, /* 0 */ - { OP_If, 1, 4, 0}, /* 1 */ - { OP_String8, 0, 3, 0}, /* 2 */ - { OP_ResultRow, 3, 1, 0}, /* 3 */ - }; - VdbeOp *aOp; - - aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); - if( aOp ){ - aOp[0].p2 = -mxErr; - aOp[2].p4type = P4_STATIC; - aOp[2].p4.z = "ok"; - } - } + addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); + sqlite3VdbeChangeP3(v, addr, -mxErr); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC); } break; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -116318,10 +108254,14 @@ SQLITE_PRIVATE void sqlite3Pragma( const struct EncName *pEnc; if( !zRight ){ /* "PRAGMA encoding" */ if( sqlite3ReadSchema(pParse) ) goto pragma_out; + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "encoding", SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_String8, 0, 1); assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); - returnSingleText(v, "encoding", encnames[ENC(pParse->db)].zName); + sqlite3VdbeChangeP4(v, -1, encnames[ENC(pParse->db)].zName, P4_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); }else{ /* "PRAGMA encoding = XXX" */ /* Only change the value of sqlite.enc if the database handle is not ** initialized. If the main database exists, the new sqlite.enc value @@ -116350,18 +108290,16 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS /* - ** PRAGMA [schema.]schema_version - ** PRAGMA [schema.]schema_version = + ** PRAGMA [database.]schema_version + ** PRAGMA [database.]schema_version = ** - ** PRAGMA [schema.]user_version - ** PRAGMA [schema.]user_version = + ** PRAGMA [database.]user_version + ** PRAGMA [database.]user_version = ** - ** PRAGMA [schema.]freelist_count + ** PRAGMA [database.]freelist_count = ** - ** PRAGMA [schema.]data_version - ** - ** PRAGMA [schema.]application_id - ** PRAGMA [schema.]application_id = + ** PRAGMA [database.]application_id + ** PRAGMA [database.]application_id = ** ** The pragma's schema_version and user_version are used to set or get ** the value of the schema-version and user-version, respectively. Both @@ -116388,16 +108326,14 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Write the specified cookie value */ static const VdbeOpList setCookie[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ - { OP_SetCookie, 0, 0, 0}, /* 1 */ + { OP_Integer, 0, 1, 0}, /* 1 */ + { OP_SetCookie, 0, 0, 1}, /* 2 */ }; - VdbeOp *aOp; - sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setCookie)); - aOp = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); - if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; - aOp[0].p1 = iDb; - aOp[1].p1 = iDb; - aOp[1].p2 = iCookie; - aOp[1].p3 = sqlite3Atoi(zRight); + int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); + sqlite3VdbeChangeP1(v, addr, iDb); + sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight)); + sqlite3VdbeChangeP1(v, addr+2, iDb); + sqlite3VdbeChangeP2(v, addr+2, iCookie); }else{ /* Read the specified cookie value */ static const VdbeOpList readCookie[] = { @@ -116405,16 +108341,12 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_ReadCookie, 0, 1, 0}, /* 1 */ { OP_ResultRow, 1, 1, 0} }; - VdbeOp *aOp; - sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(readCookie)); - aOp = sqlite3VdbeAddOpList(v, ArraySize(readCookie),readCookie,0); - if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; - aOp[0].p1 = iDb; - aOp[1].p1 = iDb; - aOp[1].p3 = iCookie; + int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie, 0); + sqlite3VdbeChangeP1(v, addr, iDb); + sqlite3VdbeChangeP1(v, addr+1, iDb); + sqlite3VdbeChangeP3(v, addr+1, iCookie); sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); - sqlite3VdbeReusable(v); } } break; @@ -116430,25 +108362,24 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_COMPILE_OPTIONS: { int i = 0; const char *zOpt; + sqlite3VdbeSetNumCols(v, 1); pParse->nMem = 1; - setOneColumnName(v, "compile_option"); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "compile_option", SQLITE_STATIC); while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ - sqlite3VdbeLoadString(v, 1, zOpt); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zOpt, 0); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } - sqlite3VdbeReusable(v); } break; #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ #ifndef SQLITE_OMIT_WAL /* - ** PRAGMA [schema.]wal_checkpoint = passive|full|restart|truncate + ** PRAGMA [database.]wal_checkpoint = passive|full|restart|truncate ** ** Checkpoint the database. */ case PragTyp_WAL_CHECKPOINT: { - static const char *azCol[] = { "busy", "log", "checkpointed" }; int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); int eMode = SQLITE_CHECKPOINT_PASSIVE; if( zRight ){ @@ -116460,8 +108391,12 @@ SQLITE_PRIVATE void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_TRUNCATE; } } - setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); + sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "log", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "checkpointed", SQLITE_STATIC); + sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } @@ -116479,7 +108414,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight ){ sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); } - returnSingleInt(v, "wal_autocheckpoint", + returnSingleInt(pParse, "wal_autocheckpoint", db->xWalCallback==sqlite3WalDefaultHook ? SQLITE_PTR_TO_INT(db->pWalArg) : 0); } @@ -116512,7 +108447,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight ){ sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); } - returnSingleInt(v, "timeout", db->busyTimeout); + returnSingleInt(pParse, "timeout", db->busyTimeout); break; } @@ -116532,7 +108467,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_soft_heap_limit64(N); } - returnSingleInt(v, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); + returnSingleInt(pParse, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); break; } @@ -116551,7 +108486,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ){ sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); } - returnSingleInt(v, "threads", + returnSingleInt(pParse, "threads", sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); break; } @@ -116564,23 +108499,25 @@ SQLITE_PRIVATE void sqlite3Pragma( static const char *const azLockName[] = { "unlocked", "shared", "reserved", "pending", "exclusive" }; - static const char *azCol[] = { "database", "status" }; int i; - setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); + sqlite3VdbeSetNumCols(v, 2); pParse->nMem = 2; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "database", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); for(i=0; inDb; i++){ Btree *pBt; const char *zState = "unknown"; int j; - if( db->aDb[i].zDbSName==0 ) continue; + if( db->aDb[i].zName==0 ) continue; + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); pBt = db->aDb[i].pBt; if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; - }else if( sqlite3_file_control(db, i ? db->aDb[i].zDbSName : 0, + }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ zState = azLockName[j]; } - sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zDbSName, zState); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, zState, P4_STATIC); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } break; @@ -116656,7 +108593,6 @@ SQLITE_PRIVATE void sqlite3Pragma( ** interface, and routines that contribute to loading the database schema ** from disk. */ -/* #include "sqliteInt.h" */ /* ** Fill the InitData structure with an error message that indicates @@ -116669,14 +108605,15 @@ static void corruptSchema( ){ sqlite3 *db = pData->db; if( !db->mallocFailed && (db->flags & SQLITE_RecoveryMode)==0 ){ - char *z; if( zObj==0 ) zObj = "?"; - z = sqlite3MPrintf(db, "malformed database schema (%s)", zObj); - if( zExtra ) z = sqlite3MPrintf(db, "%z - %s", z, zExtra); - sqlite3DbFree(db, *pData->pzErrMsg); - *pData->pzErrMsg = z; + sqlite3SetString(pData->pzErrMsg, db, + "malformed database schema (%s)", zObj); + if( zExtra ){ + *pData->pzErrMsg = sqlite3MAppendf(db, *pData->pzErrMsg, + "%s - %s", *pData->pzErrMsg, zExtra); + } } - pData->rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_CORRUPT_BKPT; + pData->rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_CORRUPT_BKPT; } /* @@ -116716,7 +108653,6 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char ** structures that describe the table, index, or view. */ int rc; - u8 saved_iDb = db->init.iDb; sqlite3_stmt *pStmt; TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ @@ -116727,15 +108663,14 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); rc = db->errCode; assert( (rc&0xFF)==(rcp&0xFF) ); - db->init.iDb = saved_iDb; - assert( saved_iDb==0 || (db->flags & SQLITE_Vacuum)!=0 ); + db->init.iDb = 0; if( SQLITE_OK!=rc ){ if( db->init.orphanTrigger ){ assert( iDb==1 ); }else{ pData->rc = rc; if( rc==SQLITE_NOMEM ){ - sqlite3OomFault(db); + db->mallocFailed = 1; }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ corruptSchema(pData, argv[0], sqlite3_errmsg(db)); } @@ -116752,7 +108687,7 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char ** to do here is record the root page number for that index. */ Index *pIndex; - pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zDbSName); + pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zName); if( pIndex==0 ){ /* This can occur if there exists an index on a TEMP table which ** has the same name as another index on a permanent index. Since @@ -116781,27 +108716,61 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ #ifndef SQLITE_OMIT_DEPRECATED int size; #endif + Table *pTab; Db *pDb; char const *azArg[4]; int meta[5]; InitData initData; - const char *zMasterName; + char const *zMasterSchema; + char const *zMasterName; int openedTransaction = 0; + /* + ** The master database table has a structure like this + */ + static const char master_schema[] = + "CREATE TABLE sqlite_master(\n" + " type text,\n" + " name text,\n" + " tbl_name text,\n" + " rootpage integer,\n" + " sql text\n" + ")" + ; +#ifndef SQLITE_OMIT_TEMPDB + static const char temp_master_schema[] = + "CREATE TEMP TABLE sqlite_temp_master(\n" + " type text,\n" + " name text,\n" + " tbl_name text,\n" + " rootpage integer,\n" + " sql text\n" + ")" + ; +#else + #define temp_master_schema 0 +#endif + assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pSchema ); assert( sqlite3_mutex_held(db->mutex) ); assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); - /* Construct the in-memory representation schema tables (sqlite_master or - ** sqlite_temp_master) by invoking the parser directly. The appropriate - ** table name will be inserted automatically by the parser so we can just - ** use the abbreviation "x" here. The parser will also automatically tag - ** the schema table as read-only. */ - azArg[0] = zMasterName = SCHEMA_TABLE(iDb); + /* zMasterSchema and zInitScript are set to point at the master schema + ** and initialisation script appropriate for the database being + ** initialized. zMasterName is the name of the master table. + */ + if( !OMIT_TEMPDB && iDb==1 ){ + zMasterSchema = temp_master_schema; + }else{ + zMasterSchema = master_schema; + } + zMasterName = SCHEMA_TABLE(iDb); + + /* Construct the schema tables. */ + azArg[0] = zMasterName; azArg[1] = "1"; - azArg[2] = "CREATE TABLE x(type text,name text,tbl_name text," - "rootpage integer,sql text)"; + azArg[2] = zMasterSchema; azArg[3] = 0; initData.db = db; initData.iDb = iDb; @@ -116812,6 +108781,10 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ rc = initData.rc; goto error_out; } + pTab = sqlite3FindTable(db, zMasterName, db->aDb[iDb].zName); + if( ALWAYS(pTab) ){ + pTab->tabFlags |= TF_Readonly; + } /* Create a cursor to hold the database open */ @@ -116830,7 +108803,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ if( !sqlite3BtreeIsInReadTrans(pDb->pBt) ){ rc = sqlite3BtreeBeginTrans(pDb->pBt, 0); if( rc!=SQLITE_OK ){ - sqlite3SetString(pzErrMsg, db, sqlite3ErrStr(rc)); + sqlite3SetString(pzErrMsg, db, "%s", sqlite3ErrStr(rc)); goto initone_error_out; } openedTransaction = 1; @@ -116930,8 +108903,8 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ { char *zSql; zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM \"%w\".%s ORDER BY rowid", - db->aDb[iDb].zDbSName, zMasterName); + "SELECT name, rootpage, sql FROM '%q'.%s ORDER BY rowid", + db->aDb[iDb].zName, zMasterName); #ifndef SQLITE_OMIT_AUTHORIZATION { sqlite3_xauth xAuth; @@ -116952,7 +108925,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ #endif } if( db->mallocFailed ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; sqlite3ResetAllSchemasOfConnection(db); } if( rc==SQLITE_OK || (db->flags&SQLITE_RecoveryMode)){ @@ -116980,7 +108953,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ error_out: if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - sqlite3OomFault(db); + db->mallocFailed = 1; } return rc; } @@ -117078,7 +109051,7 @@ static void schemaIsValid(Parse *pParse){ if( !sqlite3BtreeIsInReadTrans(pBt) ){ rc = sqlite3BtreeBeginTrans(pBt, 0); if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - sqlite3OomFault(db); + db->mallocFailed = 1; } if( rc!=SQLITE_OK ) return; openedTransaction = 1; @@ -117141,11 +109114,6 @@ SQLITE_PRIVATE void sqlite3ParserReset(Parse *pParse){ sqlite3 *db = pParse->db; sqlite3DbFree(db, pParse->aLabel); sqlite3ExprListDelete(db, pParse->pConstExpr); - if( db ){ - assert( db->lookaside.bDisable >= pParse->disableLookaside ); - db->lookaside.bDisable -= pParse->disableLookaside; - } - pParse->disableLookaside = 0; } } @@ -117161,16 +109129,20 @@ static int sqlite3Prepare( sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const char **pzTail /* OUT: End of parsed string */ ){ + Parse *pParse; /* Parsing context */ char *zErrMsg = 0; /* Error message */ int rc = SQLITE_OK; /* Result code */ int i; /* Loop counter */ - Parse sParse; /* Parsing context */ - memset(&sParse, 0, PARSE_HDR_SZ); - memset(PARSE_TAIL(&sParse), 0, PARSE_TAIL_SZ); - sParse.pReprepare = pReprepare; + /* Allocate the parsing context */ + pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); + if( pParse==0 ){ + rc = SQLITE_NOMEM; + goto end_prepare; + } + pParse->pReprepare = pReprepare; assert( ppStmt && *ppStmt==0 ); - /* assert( !db->mallocFailed ); // not true with SQLITE_USE_ALLOCA */ + assert( !db->mallocFailed ); assert( sqlite3_mutex_held(db->mutex) ); /* Check to verify that it is possible to get a read lock on all @@ -117202,7 +109174,7 @@ static int sqlite3Prepare( assert( sqlite3BtreeHoldsMutex(pBt) ); rc = sqlite3BtreeSchemaLocked(pBt); if( rc ){ - const char *zDb = db->aDb[i].zDbSName; + const char *zDb = db->aDb[i].zName; sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); testcase( db->flags & SQLITE_ReadUncommitted ); goto end_prepare; @@ -117212,7 +109184,8 @@ static int sqlite3Prepare( sqlite3VtabUnlockList(db); - sParse.db = db; + pParse->db = db; + pParse->nQueryLoop = 0; /* Logarithmic, so 0 really means 1 */ if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; @@ -117225,61 +109198,64 @@ static int sqlite3Prepare( } zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); if( zSqlCopy ){ - sqlite3RunParser(&sParse, zSqlCopy, &zErrMsg); - sParse.zTail = &zSql[sParse.zTail-zSqlCopy]; + sqlite3RunParser(pParse, zSqlCopy, &zErrMsg); sqlite3DbFree(db, zSqlCopy); + pParse->zTail = &zSql[pParse->zTail-zSqlCopy]; }else{ - sParse.zTail = &zSql[nBytes]; + pParse->zTail = &zSql[nBytes]; } }else{ - sqlite3RunParser(&sParse, zSql, &zErrMsg); + sqlite3RunParser(pParse, zSql, &zErrMsg); } - assert( 0==sParse.nQueryLoop ); + assert( 0==pParse->nQueryLoop ); - if( sParse.rc==SQLITE_DONE ) sParse.rc = SQLITE_OK; - if( sParse.checkSchema ){ - schemaIsValid(&sParse); + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM; + } + if( pParse->rc==SQLITE_DONE ) pParse->rc = SQLITE_OK; + if( pParse->checkSchema ){ + schemaIsValid(pParse); } if( db->mallocFailed ){ - sParse.rc = SQLITE_NOMEM_BKPT; + pParse->rc = SQLITE_NOMEM; } if( pzTail ){ - *pzTail = sParse.zTail; + *pzTail = pParse->zTail; } - rc = sParse.rc; + rc = pParse->rc; #ifndef SQLITE_OMIT_EXPLAIN - if( rc==SQLITE_OK && sParse.pVdbe && sParse.explain ){ + if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){ static const char * const azColName[] = { "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", "selectid", "order", "from", "detail" }; int iFirst, mx; - if( sParse.explain==2 ){ - sqlite3VdbeSetNumCols(sParse.pVdbe, 4); + if( pParse->explain==2 ){ + sqlite3VdbeSetNumCols(pParse->pVdbe, 4); iFirst = 8; mx = 12; }else{ - sqlite3VdbeSetNumCols(sParse.pVdbe, 8); + sqlite3VdbeSetNumCols(pParse->pVdbe, 8); iFirst = 0; mx = 8; } for(i=iFirst; ipVdbe, i-iFirst, COLNAME_NAME, azColName[i], SQLITE_STATIC); } } #endif if( db->init.busy==0 ){ - Vdbe *pVdbe = sParse.pVdbe; - sqlite3VdbeSetSql(pVdbe, zSql, (int)(sParse.zTail-zSql), saveSqlFlag); + Vdbe *pVdbe = pParse->pVdbe; + sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); } - if( sParse.pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ - sqlite3VdbeFinalize(sParse.pVdbe); + if( pParse->pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ + sqlite3VdbeFinalize(pParse->pVdbe); assert(!(*ppStmt)); }else{ - *ppStmt = (sqlite3_stmt*)sParse.pVdbe; + *ppStmt = (sqlite3_stmt*)pParse->pVdbe; } if( zErrMsg ){ @@ -117290,15 +109266,16 @@ static int sqlite3Prepare( } /* Delete any TriggerPrg structures allocated while parsing this statement. */ - while( sParse.pTriggerPrg ){ - TriggerPrg *pT = sParse.pTriggerPrg; - sParse.pTriggerPrg = pT->pNext; + while( pParse->pTriggerPrg ){ + TriggerPrg *pT = pParse->pTriggerPrg; + pParse->pTriggerPrg = pT->pNext; sqlite3DbFree(db, pT); } end_prepare: - sqlite3ParserReset(&sParse); + sqlite3ParserReset(pParse); + sqlite3StackFree(db, pParse); rc = sqlite3ApiExit(db, rc); assert( (rc&db->errMask)==rc ); return rc; @@ -117356,7 +109333,7 @@ SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ rc = sqlite3LockAndPrepare(db, zSql, -1, 0, p, &pNew, 0); if( rc ){ if( rc==SQLITE_NOMEM ){ - sqlite3OomFault(db); + db->mallocFailed = 1; } assert( pNew==0 ); return rc; @@ -117379,7 +109356,7 @@ SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ ** and the statement is automatically recompiled if an schema change ** occurs. */ -SQLITE_API int sqlite3_prepare( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare( sqlite3 *db, /* Database handle. */ const char *zSql, /* UTF-8 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ @@ -117391,7 +109368,7 @@ SQLITE_API int sqlite3_prepare( assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ return rc; } -SQLITE_API int sqlite3_prepare_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare_v2( sqlite3 *db, /* Database handle. */ const char *zSql, /* UTF-8 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ @@ -117467,7 +109444,7 @@ static int sqlite3Prepare16( ** and the statement is automatically recompiled if an schema change ** occurs. */ -SQLITE_API int sqlite3_prepare16( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare16( sqlite3 *db, /* Database handle. */ const void *zSql, /* UTF-16 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ @@ -117479,7 +109456,7 @@ SQLITE_API int sqlite3_prepare16( assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ return rc; } -SQLITE_API int sqlite3_prepare16_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_prepare16_v2( sqlite3 *db, /* Database handle. */ const void *zSql, /* UTF-16 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ @@ -117510,7 +109487,6 @@ SQLITE_API int sqlite3_prepare16_v2( ** This file contains C code routines that are called by the parser ** to handle SELECT statements in SQLite. */ -/* #include "sqliteInt.h" */ /* ** Trace output macros @@ -117519,8 +109495,7 @@ SQLITE_API int sqlite3_prepare16_v2( /***/ int sqlite3SelectTrace = 0; # define SELECTTRACE(K,P,S,X) \ if(sqlite3SelectTrace&(K)) \ - sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",\ - (S)->zSelName,(S)),\ + sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",(S)->zSelName,(S)),\ sqlite3DebugPrintf X #else # define SELECTTRACE(K,P,S,X) @@ -117552,9 +109527,7 @@ struct SortCtx { int regReturn; /* Register holding block-output return address */ int labelBkOut; /* Start label for the block-output subroutine */ int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ - int labelDone; /* Jump here when done, ex: LIMIT reached */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ - u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */ }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -117573,7 +109546,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3ExprListDelete(db, p->pOrderBy); sqlite3ExprDelete(db, p->pLimit); sqlite3ExprDelete(db, p->pOffset); - if( p->pWith ) sqlite3WithDelete(db, p->pWith); + sqlite3WithDelete(db, p->pWith); if( bFree ) sqlite3DbFree(db, p); p = pPrior; bFree = 1; @@ -117586,7 +109559,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ pDest->eDest = (u8)eDest; pDest->iSDParm = iParm; - pDest->zAffSdst = 0; + pDest->affSdst = 0; pDest->iSdst = 0; pDest->nSdst = 0; } @@ -117604,44 +109577,36 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( ExprList *pGroupBy, /* the GROUP BY clause */ Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ - u32 selFlags, /* Flag parameters, such as SF_Distinct */ + u16 selFlags, /* Flag parameters, such as SF_Distinct */ Expr *pLimit, /* LIMIT value. NULL means not used */ Expr *pOffset /* OFFSET value. NULL means no offset */ ){ Select *pNew; Select standin; sqlite3 *db = pParse->db; - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); if( pNew==0 ){ assert( db->mallocFailed ); pNew = &standin; + memset(pNew, 0, sizeof(*pNew)); } if( pEList==0 ){ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0)); + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ALL,0)); } pNew->pEList = pEList; - pNew->op = TK_SELECT; - pNew->selFlags = selFlags; - pNew->iLimit = 0; - pNew->iOffset = 0; -#if SELECTTRACE_ENABLED - pNew->zSelName[0] = 0; -#endif - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; - pNew->nSelectRow = 0; if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc)); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; - pNew->pPrior = 0; - pNew->pNext = 0; + pNew->selFlags = selFlags; + pNew->op = TK_SELECT; pNew->pLimit = pLimit; pNew->pOffset = pOffset; - pNew->pWith = 0; assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || db->mallocFailed!=0 ); + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; if( db->mallocFailed ) { clearSelect(db, pNew, pNew!=&standin); pNew = 0; @@ -117668,7 +109633,7 @@ SQLITE_PRIVATE void sqlite3SelectSetName(Select *p, const char *zName){ ** Delete the given Select structure and all of its substructures. */ SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ - if( p ) clearSelect(db, p, 1); + clearSelect(db, p, 1); } /* @@ -117874,12 +109839,6 @@ static void setJoinExpr(Expr *p, int iTable){ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(p, EP_NoReduce); p->iRightJoinTable = (i16)iTable; - if( p->op==TK_FUNCTION && p->x.pList ){ - int i; - for(i=0; ix.pList->nExpr; i++){ - setJoinExpr(p->x.pList->a[i].pExpr, iTable); - } - } setJoinExpr(p->pLeft, iTable); p = p->pRight; } @@ -117914,12 +109873,12 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ int isOuter; if( NEVER(pLeftTab==0 || pRightTab==0) ) continue; - isOuter = (pRight->fg.jointype & JT_OUTER)!=0; + isOuter = (pRight->jointype & JT_OUTER)!=0; /* When the NATURAL keyword is present, add WHERE clause terms for ** every column that the two tables have in common. */ - if( pRight->fg.jointype & JT_NATURAL ){ + if( pRight->jointype & JT_NATURAL ){ if( pRight->pOn || pRight->pUsing ){ sqlite3ErrorMsg(pParse, "a NATURAL join may not have " "an ON or USING clause", 0); @@ -118004,7 +109963,6 @@ static void pushOntoSorter( SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ int regData, /* First register holding data to be sorted */ - int regOrigData, /* First register holding data before packing */ int nData, /* Number of elements in the data array */ int nPrefixReg /* No. of reg prior to regData available for use */ ){ @@ -118016,10 +109974,8 @@ static void pushOntoSorter( int regRecord = ++pParse->nMem; /* Assembled sorter record */ int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ - int iLimit; /* LIMIT counter */ assert( bSeq==0 || bSeq==1 ); - assert( nData==1 || regData==regOrigData ); if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nExpr - bSeq; @@ -118027,17 +109983,14 @@ static void pushOntoSorter( regBase = pParse->nMem + 1; pParse->nMem += nBase; } - assert( pSelect->iOffset==0 || pSelect->iLimit!=0 ); - iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; - pSort->labelDone = sqlite3VdbeMakeLabel(v); - sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, - SQLITE_ECEL_DUP|SQLITE_ECEL_REF); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); if( bSeq ){ sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } if( nPrefixReg==0 ){ sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ @@ -118072,10 +110025,6 @@ static void pushOntoSorter( pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); - if( iLimit ){ - sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone); - VdbeCoverage(v); - } sqlite3VdbeJumpHere(v, addrFirst); sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); @@ -118086,32 +110035,17 @@ static void pushOntoSorter( op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - if( iLimit ){ + if( pSelect->iLimit ){ int addr; - int r1 = 0; - /* Fill the sorter until it contains LIMIT+OFFSET entries. (The iLimit - ** register is initialized with value of LIMIT+OFFSET.) After the sorter - ** fills up, delete the least entry in the sorter after each insert. - ** Thus we never hold more than the LIMIT+OFFSET rows in memory at once */ - addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); - sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); - if( pSort->bOrderedInnerLoop ){ - r1 = ++pParse->nMem; - sqlite3VdbeAddOp3(v, OP_Column, pSort->iECursor, nExpr, r1); - VdbeComment((v, "seq")); + int iLimit; + if( pSelect->iOffset ){ + iLimit = pSelect->iOffset+1; + }else{ + iLimit = pSelect->iLimit; } + addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, -1); VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); - if( pSort->bOrderedInnerLoop ){ - /* If the inner loop is driven by an index such that values from - ** the same iteration of the inner loop are in sorted order, then - ** immediately jump to the next iteration of an inner loop if the - ** entry from the current iteration does not fit into the top - ** LIMIT+OFFSET entries of the sorter. */ - int iBrk = sqlite3VdbeCurrentAddr(v) + 2; - sqlite3VdbeAddOp3(v, OP_Eq, regBase+nExpr, iBrk, r1); - sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - VdbeCoverage(v); - } sqlite3VdbeJumpHere(v, addr); } } @@ -118125,8 +110059,11 @@ static void codeOffset( int iContinue /* Jump here to skip the current record */ ){ if( iOffset>0 ){ - sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v); - VdbeComment((v, "OFFSET")); + int addr; + addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); + VdbeComment((v, "skip OFFSET records")); + sqlite3VdbeJumpHere(v, addr); } } @@ -118157,6 +110094,30 @@ static void codeDistinct( sqlite3ReleaseTempReg(pParse, r1); } +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate an error message when a SELECT is used within a subexpression +** (example: "a IN (SELECT * FROM table)") but it has more than 1 result +** column. We do this in a subroutine because the error used to occur +** in multiple places. (The error only occurs in one place now, but we +** retain the subroutine to minimize code disruption.) +*/ +static int checkForMultiColumnSelectError( + Parse *pParse, /* Parse context. */ + SelectDest *pDest, /* Destination of SELECT results */ + int nExpr /* Number of result columns returned by SELECT */ +){ + int eDest = pDest->eDest; + if( nExpr>1 && (eDest==SRT_Mem || eDest==SRT_Set) ){ + sqlite3ErrorMsg(pParse, "only a single result allowed for " + "a SELECT that is part of an expression"); + return 1; + }else{ + return 0; + } +} +#endif + /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. @@ -118226,13 +110187,8 @@ static void selectInnerLoop( /* If the destination is an EXISTS(...) expression, the actual ** values returned by the SELECT are not required. */ - u8 ecelFlags; - if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){ - ecelFlags = SQLITE_ECEL_DUP; - }else{ - ecelFlags = 0; - } - sqlite3ExprCodeExprList(pParse, pEList, regResult, 0, ecelFlags); + sqlite3ExprCodeExprList(pParse, pEList, regResult, + (eDest==SRT_Output||eDest==SRT_Coroutine)?SQLITE_ECEL_DUP:0); } /* If the DISTINCT keyword was present on the SELECT statement @@ -118287,8 +110243,7 @@ static void selectInnerLoop( default: { assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); - codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, - regResult); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, regResult); break; } } @@ -118330,8 +110285,6 @@ static void selectInnerLoop( int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); - testcase( eDest==SRT_Fifo ); - testcase( eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ @@ -118341,14 +110294,13 @@ static void selectInnerLoop( ** current row to the index and proceed with writing it to the ** output table as well. */ int addr = sqlite3VdbeCurrentAddr(v) + 4; - sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); - VdbeCoverage(v); + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); assert( pSort==0 ); } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1+nPrefixReg,regResult,1,nPrefixReg); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, 1, nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -118366,19 +110318,19 @@ static void selectInnerLoop( ** item into the set table with bogus data. */ case SRT_Set: { + assert( nResultCol==1 ); + pDest->affSdst = + sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pSort ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter( - pParse, pSort, p, regResult, regResult, nResultCol, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); - assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, - r1, pDest->zAffSdst, nResultCol); - sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); + sqlite3ExprCacheAffinityChange(pParse, regResult, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); } @@ -118394,14 +110346,13 @@ static void selectInnerLoop( } /* If this is a scalar select that is part of an expression, then - ** store the results in the appropriate memory cell or array of - ** memory cells and break out of the scan loop. + ** store the results in the appropriate memory cell and break out + ** of the scan loop. */ case SRT_Mem: { - assert( nResultCol==pDest->nSdst ); + assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter( - pParse, pSort, p, regResult, regResult, nResultCol, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ @@ -118415,8 +110366,7 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, regResult, nResultCol, - nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, nResultCol, nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ @@ -118502,8 +110452,8 @@ static void selectInnerLoop( ** X extra columns. */ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ - int nExtra = (N+X)*(sizeof(CollSeq*)+1); - KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); + KeyInfo *p = sqlite3DbMallocZero(0, + sizeof(KeyInfo) + (N+X)*(sizeof(CollSeq*)+1)); if( p ){ p->aSortOrder = (u8*)&p->aColl[N+X]; p->nField = (u16)N; @@ -118511,9 +110461,8 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ p->enc = ENC(db); p->db = db; p->nRef = 1; - memset(&p[1], 0, nExtra); }else{ - sqlite3OomFault(db); + db->mallocFailed = 1; } return p; } @@ -118525,7 +110474,7 @@ SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo *p){ if( p ){ assert( p->nRef>0 ); p->nRef--; - if( p->nRef==0 ) sqlite3DbFree(p->db, p); + if( p->nRef==0 ) sqlite3DbFree(0, p); } } @@ -118591,6 +110540,7 @@ static KeyInfo *keyInfoFromExprList( return pInfo; } +#ifndef SQLITE_OMIT_COMPOUND_SELECT /* ** Name of the connection operator, used for error messages. */ @@ -118604,6 +110554,7 @@ static const char *selectOpName(int id){ } return z; } +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ #ifndef SQLITE_OMIT_EXPLAIN /* @@ -118690,7 +110641,7 @@ static void generateSortTail( SelectDest *pDest /* Write the sorted results here */ ){ Vdbe *v = pParse->pVdbe; /* The prepared statement */ - int addrBreak = pSort->labelDone; /* Jump here to exit loop */ + int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; int addrOnce = 0; @@ -118709,28 +110660,27 @@ static void generateSortTail( struct ExprList_item *aOutEx = p->pEList->a; #endif - assert( addrBreak<0 ); if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); - sqlite3VdbeGoto(v, addrBreak); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } iTab = pSort->iECursor; - if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){ + if( eDest==SRT_Output || eDest==SRT_Coroutine ){ regRowid = 0; regRow = pDest->iSdst; nSortData = nColumn; }else{ regRowid = sqlite3GetTempReg(pParse); - regRow = sqlite3GetTempRange(pParse, nColumn); - nSortData = nColumn; + regRow = sqlite3GetTempReg(pParse); + nSortData = 1; } nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; if( pSort->labelBkOut ){ - addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); } sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); @@ -118750,7 +110700,10 @@ static void generateSortTail( VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan)); } switch( eDest ){ + case SRT_Table: case SRT_EphemTab: { + testcase( eDest==SRT_Table ); + testcase( eDest==SRT_EphemTab ); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -118758,14 +110711,16 @@ static void generateSortTail( } #ifndef SQLITE_OMIT_SUBQUERY case SRT_Set: { - assert( nColumn==sqlite3Strlen30(pDest->zAffSdst) ); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, nColumn, regRowid, - pDest->zAffSdst, nColumn); - sqlite3ExprCacheAffinityChange(pParse, regRow, nColumn); + assert( nColumn==1 ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, + &pDest->affSdst, 1); + sqlite3ExprCacheAffinityChange(pParse, regRow, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); break; } case SRT_Mem: { + assert( nColumn==1 ); + sqlite3ExprCodeMove(pParse, regRow, iParm, 1); /* The LIMIT clause will terminate the loop for us */ break; } @@ -118784,11 +110739,7 @@ static void generateSortTail( } } if( regRowid ){ - if( eDest==SRT_Set ){ - sqlite3ReleaseTempRange(pParse, regRow, nColumn); - }else{ - sqlite3ReleaseTempReg(pParse, regRow); - } + sqlite3ReleaseTempReg(pParse, regRow); sqlite3ReleaseTempReg(pParse, regRowid); } /* The bottom of the loop @@ -118829,30 +110780,30 @@ static void generateSortTail( */ #ifdef SQLITE_ENABLE_COLUMN_METADATA # define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,C,D,E,F) -#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ -# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F) -#endif static const char *columnTypeImpl( NameContext *pNC, Expr *pExpr, -#ifdef SQLITE_ENABLE_COLUMN_METADATA const char **pzOrigDb, const char **pzOrigTab, const char **pzOrigCol, -#endif u8 *pEstWidth ){ - char const *zType = 0; - int j; - u8 estWidth = 1; -#ifdef SQLITE_ENABLE_COLUMN_METADATA char const *zOrigDb = 0; char const *zOrigTab = 0; char const *zOrigCol = 0; -#endif +#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ +# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F) +static const char *columnTypeImpl( + NameContext *pNC, + Expr *pExpr, + u8 *pEstWidth +){ +#endif /* !defined(SQLITE_ENABLE_COLUMN_METADATA) */ + char const *zType = 0; + int j; + u8 estWidth = 1; - assert( pExpr!=0 ); - assert( pNC->pSrcList!=0 ); + if( NEVER(pExpr==0) || pNC->pSrcList==0 ) return 0; switch( pExpr->op ){ case TK_AGG_COLUMN: case TK_COLUMN: { @@ -118903,13 +110854,10 @@ static const char *columnTypeImpl( ** of the SELECT statement. Return the declaration type and origin ** data for the result-set column of the sub-select. */ - if( iCol>=0 && ALWAYS(iColpEList->nExpr) ){ + if( iCol>=0 && iColpEList->nExpr ){ /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see ** test case misc2.2.2) - it always evaluates to NULL. - ** - ** The ALWAYS() is because iCol>=pS->pEList->nExpr will have been - ** caught already by name resolution. */ NameContext sNC; Expr *p = pS->pEList->a[iCol].pExpr; @@ -118928,20 +110876,20 @@ static const char *columnTypeImpl( zType = "INTEGER"; zOrigCol = "rowid"; }else{ + zType = pTab->aCol[iCol].zType; zOrigCol = pTab->aCol[iCol].zName; - zType = sqlite3ColumnType(&pTab->aCol[iCol],0); estWidth = pTab->aCol[iCol].szEst; } zOrigTab = pTab->zName; if( pNC->pParse ){ int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema); - zOrigDb = pNC->pParse->db->aDb[iDb].zDbSName; + zOrigDb = pNC->pParse->db->aDb[iDb].zName; } #else if( iCol<0 ){ zType = "INTEGER"; }else{ - zType = sqlite3ColumnType(&pTab->aCol[iCol],0); + zType = pTab->aCol[iCol].zType; estWidth = pTab->aCol[iCol].szEst; } #endif @@ -119040,9 +110988,7 @@ static void generateColumnNames( } #endif - if( pParse->colNamesSet || db->mallocFailed ) return; - assert( v!=0 ); - assert( pTabList!=0 ); + if( pParse->colNamesSet || NEVER(v==0) || db->mallocFailed ) return; pParse->colNamesSet = 1; fullNames = (db->flags & SQLITE_FullColNames)!=0; shortNames = (db->flags & SQLITE_ShortColNames)!=0; @@ -119054,7 +111000,7 @@ static void generateColumnNames( if( pEList->a[i].zName ){ char *zName = pEList->a[i].zName; sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); - }else if( p->op==TK_COLUMN || p->op==TK_AGG_COLUMN ){ + }else if( (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) && pTabList ){ Table *pTab; char *zCol; int iCol = p->iColumn; @@ -119102,7 +111048,7 @@ static void generateColumnNames( ** Return SQLITE_OK on success. If a memory allocation error occurs, ** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. */ -SQLITE_PRIVATE int sqlite3ColumnsFromExprList( +static int selectColumnsFromExprList( Parse *pParse, /* Parsing context */ ExprList *pEList, /* Expr list from which to derive column names */ i16 *pnCol, /* Write the number of columns here */ @@ -119110,15 +111056,13 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( ){ sqlite3 *db = pParse->db; /* Database connection */ int i, j; /* Loop counters */ - u32 cnt; /* Index added to make the name unique */ + int cnt; /* Index added to make the name unique */ Column *aCol, *pCol; /* For looping over result columns */ int nCol; /* Number of columns in the result set */ Expr *p; /* Expression for a single result column */ char *zName; /* Column name */ int nName; /* Size of name in zName[] */ - Hash ht; /* Hash table of column names */ - sqlite3HashInit(&ht); if( pEList ){ nCol = pEList->nExpr; aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); @@ -119127,16 +111071,16 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( nCol = 0; aCol = 0; } - assert( nCol==(i16)nCol ); *pnCol = nCol; *paCol = aCol; - for(i=0, pCol=aCol; imallocFailed; i++, pCol++){ + for(i=0, pCol=aCol; ia[i].pExpr); if( (zName = pEList->a[i].zName)!=0 ){ /* If the column contains an "AS " phrase, use as the name */ + zName = sqlite3DbStrDup(db, zName); }else{ Expr *pColExpr = p; /* The expression that is the result column name */ Table *pTab; /* Table associated with this expression */ @@ -119149,37 +111093,41 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( int iCol = pColExpr->iColumn; pTab = pColExpr->pTab; if( iCol<0 ) iCol = pTab->iPKey; - zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid"; + zName = sqlite3MPrintf(db, "%s", + iCol>=0 ? pTab->aCol[iCol].zName : "rowid"); }else if( pColExpr->op==TK_ID ){ assert( !ExprHasProperty(pColExpr, EP_IntValue) ); - zName = pColExpr->u.zToken; + zName = sqlite3MPrintf(db, "%s", pColExpr->u.zToken); }else{ /* Use the original text of the column expression as its name */ - zName = pEList->a[i].zSpan; + zName = sqlite3MPrintf(db, "%s", pEList->a[i].zSpan); } } - zName = sqlite3MPrintf(db, "%s", zName); + if( db->mallocFailed ){ + sqlite3DbFree(db, zName); + break; + } /* Make sure the column name is unique. If the name is not unique, ** append an integer to the name so that it becomes unique. */ - cnt = 0; - while( zName && sqlite3HashFind(&ht, zName)!=0 ){ - nName = sqlite3Strlen30(zName); - if( nName>0 ){ - for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){} - if( zName[j]==':' ) nName = j; + nName = sqlite3Strlen30(zName); + for(j=cnt=0; j1 && sqlite3Isdigit(zName[k]); k--){} + if( k>=0 && zName[k]==':' ) nName = k; + zName[nName] = 0; + zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt); + sqlite3DbFree(db, zName); + zName = zNewName; + j = -1; + if( zName==0 ) break; } - zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); - if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt); } pCol->zName = zName; - sqlite3ColumnPropertiesFromName(0, pCol); - if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){ - sqlite3OomFault(db); - } } - sqlite3HashClear(&ht); if( db->mallocFailed ){ for(j=0; jpSrc; a = pSelect->pEList->a; for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ - const char *zType; - int n, m; p = a[i].pExpr; - zType = columnType(&sNC, p, 0, 0, 0, &pCol->szEst); + if( pCol->zType==0 ){ + pCol->zType = sqlite3DbStrDup(db, columnType(&sNC, p,0,0,0, &pCol->szEst)); + } szAll += pCol->szEst; pCol->affinity = sqlite3ExprAffinity(p); - if( zType && (m = sqlite3Strlen30(zType))>0 ){ - n = sqlite3Strlen30(pCol->zName); - pCol->zName = sqlite3DbReallocOrFree(db, pCol->zName, n+m+2); - if( pCol->zName ){ - memcpy(&pCol->zName[n+1], zType, m+1); - pCol->colFlags |= COLFLAG_HASTYPE; - } - } - if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_BLOB; + if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_NONE; pColl = sqlite3ExprCollSeq(pParse, p); if( pColl && pCol->zColl==0 ){ pCol->zColl = sqlite3DbStrDup(db, pColl->zName); @@ -119270,12 +111210,12 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ } /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside ** is disabled */ - assert( db->lookaside.bDisable ); + assert( db->lookaside.bEnabled==0 ); pTab->nRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect); + selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); + selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -119288,19 +111228,19 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ ** Get a VDBE for the given parser context. Create a new one if necessary. ** If an error occurs, return NULL and leave a message in pParse. */ -static SQLITE_NOINLINE Vdbe *allocVdbe(Parse *pParse){ - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(pParse); - if( v ) sqlite3VdbeAddOp2(v, OP_Init, 0, 1); - if( pParse->pToplevel==0 - && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) - ){ - pParse->okConstFactor = 1; - } - return v; -} SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ Vdbe *v = pParse->pVdbe; - return v ? v : allocVdbe(pParse); + if( v==0 ){ + v = pParse->pVdbe = sqlite3VdbeCreate(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Init); + if( pParse->pToplevel==0 + && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) + ){ + pParse->okConstFactor = 1; + } + + } + return v; } @@ -119331,7 +111271,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ Vdbe *v = 0; int iLimit = 0; int iOffset; - int n; + int addr1, n; if( p->iLimit ) return; /* @@ -119350,10 +111290,9 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ - sqlite3VdbeGoto(v, iBreak); - }else if( n>=0 && p->nSelectRow>sqlite3LogEst((u64)n) ){ - p->nSelectRow = sqlite3LogEst((u64)n); - p->selFlags |= SF_FixedLimit; + sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + }else if( n>=0 && p->nSelectRow>(u64)n ){ + p->nSelectRow = n; } }else{ sqlite3ExprCode(pParse, p->pLimit, iLimit); @@ -119367,8 +111306,14 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3ExprCode(pParse, p->pOffset, iOffset); sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); VdbeComment((v, "OFFSET counter")); - sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); VdbeComment((v, "LIMIT+OFFSET")); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); + sqlite3VdbeJumpHere(v, addr1); } } } @@ -119390,10 +111335,7 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ pRet = 0; } assert( iCol>=0 ); - /* iCol must be less than p->pEList->nExpr. Otherwise an error would - ** have been thrown during name resolution and we would not have gotten - ** this far */ - if( pRet==0 && ALWAYS(iColpEList->nExpr) ){ + if( pRet==0 && iColpEList->nExpr ){ pRet = sqlite3ExprCollSeq(pParse, p->pEList->a[iCol].pExpr); } return pRet; @@ -119448,7 +111390,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ ** ** ** There is exactly one reference to the recursive-table in the FROM clause -** of recursive-query, marked with the SrcList->a[].fg.isRecursive flag. +** of recursive-query, marked with the SrcList->a[].isRecursive flag. ** ** The setup-query runs once to generate an initial set of rows that go ** into a Queue table. Rows are extracted from the Queue table one by @@ -119513,7 +111455,7 @@ static void generateWithRecursiveQuery( /* Locate the cursor number of the Current table */ for(i=0; ALWAYS(inSrc); i++){ - if( pSrc->a[i].fg.isRecursive ){ + if( pSrc->a[i].isRecursive ){ iCurrent = pSrc->a[i].iCursor; break; } @@ -119583,17 +111525,13 @@ static void generateWithRecursiveQuery( /* Execute the recursive SELECT taking the single row in Current as ** the value for the recursive-table. Store the results in the Queue. */ - if( p->selFlags & SF_Aggregate ){ - sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported"); - }else{ - p->pPrior = 0; - sqlite3Select(pParse, p, &destQueue); - assert( p->pPrior==0 ); - p->pPrior = pSetup; - } + p->pPrior = 0; + sqlite3Select(pParse, p, &destQueue); + assert( p->pPrior==0 ); + p->pPrior = pSetup; /* Keep running the loop until the Queue is empty */ - sqlite3VdbeGoto(v, addrTop); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); sqlite3VdbeResolveLabel(v, addrBreak); end_of_recursive_query: @@ -119612,6 +111550,19 @@ static int multiSelectOrderBy( SelectDest *pDest /* What to do with query results */ ); +/* +** Error message for when two or more terms of a compound select have different +** size result sets. +*/ +static void selectWrongNumTermsError(Parse *pParse, Select *p){ + if( p->selFlags & SF_Values ){ + sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); + }else{ + sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", selectOpName(p->op)); + } +} + /* ** Handle the special case of a compound-select that originates from a ** VALUES clause. By handling this as a special case, we avoid deep @@ -119629,6 +111580,7 @@ static int multiSelectValues( SelectDest *pDest /* What to do with query results */ ){ Select *pPrior; + int nExpr = p->pEList->nExpr; int nRow = 1; int rc = 0; assert( p->selFlags & SF_MultiValue ); @@ -119637,7 +111589,10 @@ static int multiSelectValues( assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) ); assert( p->pLimit==0 ); assert( p->pOffset==0 ); - assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr ); + if( p->pEList->nExpr!=nExpr ){ + selectWrongNumTermsError(pParse, p); + return 1; + } if( p->pPrior==0 ) break; assert( p->pPrior->pNext==p ); p = p->pPrior; @@ -119731,6 +111686,7 @@ static int multiSelect( if( dest.eDest==SRT_EphemTab ){ assert( p->pEList ); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iSDParm, p->pEList->nExpr); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); dest.eDest = SRT_Table; } @@ -119745,7 +111701,11 @@ static int multiSelect( ** in their result sets. */ assert( p->pEList && pPrior->pEList ); - assert( p->pEList->nExpr==pPrior->pEList->nExpr ); + if( p->pEList->nExpr!=pPrior->pEList->nExpr ){ + selectWrongNumTermsError(pParse, p); + rc = 1; + goto multi_select_end; + } #ifndef SQLITE_OMIT_CTE if( p->selFlags & SF_Recursive ){ @@ -119783,22 +111743,18 @@ static int multiSelect( if( p->iLimit ){ addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); VdbeComment((v, "Jump ahead if LIMIT reached")); - if( p->iOffset ){ - sqlite3VdbeAddOp3(v, OP_OffsetLimit, - p->iLimit, p->iOffset+1, p->iOffset); - } } explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &dest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; p->pPrior = pPrior; - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + p->nSelectRow += pPrior->nSelectRow; if( pPrior->pLimit && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) - && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) + && nLimit>0 && p->nSelectRow > (u64)nLimit ){ - p->nSelectRow = sqlite3LogEst((u64)nLimit); + p->nSelectRow = nLimit; } if( addr ){ sqlite3VdbeJumpHere(v, addr); @@ -119870,9 +111826,7 @@ static int multiSelect( pDelete = p->pPrior; p->pPrior = pPrior; p->pOrderBy = 0; - if( p->op==TK_UNION ){ - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); - } + if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow; sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->pOffset = pOffset; @@ -119889,7 +111843,7 @@ static int multiSelect( if( dest.eDest==SRT_Output ){ Select *pFirst = p; while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); + generateColumnNames(pParse, 0, pFirst->pEList); } iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); @@ -119964,7 +111918,7 @@ static int multiSelect( if( dest.eDest==SRT_Output ){ Select *pFirst = p; while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); + generateColumnNames(pParse, 0, pFirst->pEList); } iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); @@ -120007,7 +111961,7 @@ static int multiSelect( nCol = p->pEList->nExpr; pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); if( !pKeyInfo ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; goto multi_select_end; } for(i=0, apColl=pKeyInfo->aColl; iselFlags & SF_Values ){ - sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); - }else{ - sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" - " do not have the same number of result columns", selectOpName(p->op)); - } -} - /* ** Code an output subroutine for a coroutine implementation of a ** SELECT statment. @@ -120096,12 +112037,12 @@ static int generateOutputSubroutine( /* Suppress duplicates for UNION, EXCEPT, and INTERSECT */ if( regPrev ){ - int addr1, addr2; - addr1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); - addr2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, + int j1, j2; + j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); + j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - sqlite3VdbeAddOp3(v, OP_Jump, addr2+2, iContinue, addr2+2); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j1); sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } @@ -120111,14 +112052,15 @@ static int generateOutputSubroutine( */ codeOffset(v, p->iOffset, iContinue); - assert( pDest->eDest!=SRT_Exists ); - assert( pDest->eDest!=SRT_Table ); switch( pDest->eDest ){ /* Store the result as data using a unique key. */ + case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); int r2 = sqlite3GetTempReg(pParse); + testcase( pDest->eDest==SRT_Table ); + testcase( pDest->eDest==SRT_EphemTab ); sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); @@ -120129,20 +112071,33 @@ static int generateOutputSubroutine( } #ifndef SQLITE_OMIT_SUBQUERY - /* If we are creating a set for an "expr IN (SELECT ...)". + /* If we are creating a set for an "expr IN (SELECT ...)" construct, + ** then there should be a single item on the stack. Write this + ** item into the set table with bogus data. */ case SRT_Set: { int r1; - testcase( pIn->nSdst>1 ); + assert( pIn->nSdst==1 || pParse->nErr>0 ); + pDest->affSdst = + sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, - r1, pDest->zAffSdst, pIn->nSdst); - sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &pDest->affSdst,1); + sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iSDParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; } +#if 0 /* Never occurs on an ORDER BY query */ + /* If any row exist in the result set, record that fact and abort. + */ + case SRT_Exists: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iSDParm); + /* The LIMIT clause will terminate the loop for us */ + break; + } +#endif + /* If this is a scalar select that is part of an expression, then ** store the results in the appropriate memory cell and break out ** of the scan loop. @@ -120315,7 +112270,7 @@ static int multiSelectOrderBy( int savedOffset; /* Saved value of p->iOffset */ int labelCmpr; /* Label for the start of the merge algorithm */ int labelEnd; /* Label for the end of the overall SELECT stmt */ - int addr1; /* Jump instructions that get retargetted */ + int j1; /* Jump instructions that get retargetted */ int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ KeyInfo *pKeyMerge; /* Comparison information for merging rows */ @@ -120359,7 +112314,7 @@ static int multiSelectOrderBy( } if( j==nOrderBy ){ Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); - if( pNew==0 ) return SQLITE_NOMEM_BKPT; + if( pNew==0 ) return SQLITE_NOMEM; pNew->flags |= EP_IntValue; pNew->u.iValue = i; pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); @@ -120375,13 +112330,14 @@ static int multiSelectOrderBy( ** to the right and the left are evaluated, they use the correct ** collation. */ - aPermute = sqlite3DbMallocRawNN(db, sizeof(int)*(nOrderBy + 1)); + aPermute = sqlite3DbMallocRaw(db, sizeof(int)*nOrderBy); if( aPermute ){ struct ExprList_item *pItem; - aPermute[0] = nOrderBy; - for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){ + for(i=0, pItem=pOrderBy->a; iu.x.iOrderByCol>0 ); - assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); + /* assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ) is also true + ** but only for well-formed SELECT statements. */ + testcase( pItem->u.x.iOrderByCol > p->pEList->nExpr ); aPermute[i] = pItem->u.x.iOrderByCol - 1; } pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); @@ -120452,19 +112408,19 @@ static int multiSelectOrderBy( ** left of the compound operator - the "A" select. */ addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; - addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); + j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); VdbeComment((v, "left SELECT")); pPrior->iLimit = regLimitA; explainSetInteger(iSub1, pParse->iNextSelectId); sqlite3Select(pParse, pPrior, &destA); - sqlite3VdbeEndCoroutine(v, regAddrA); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrA); + sqlite3VdbeJumpHere(v, j1); /* Generate a coroutine to evaluate the SELECT statement on ** the right - the "B" select */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; - addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); + j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); VdbeComment((v, "right SELECT")); savedLimit = p->iLimit; savedOffset = p->iOffset; @@ -120474,7 +112430,7 @@ static int multiSelectOrderBy( sqlite3Select(pParse, p, &destB); p->iLimit = savedLimit; p->iOffset = savedOffset; - sqlite3VdbeEndCoroutine(v, regAddrB); + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrB); /* Generate a subroutine that outputs the current row of the A ** select as the next output row of the compound select. @@ -120505,8 +112461,8 @@ static int multiSelectOrderBy( addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); VdbeCoverage(v); - sqlite3VdbeGoto(v, addrEofA); - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); + p->nSelectRow += pPrior->nSelectRow; } /* Generate a subroutine to run when the results from select B @@ -120519,7 +112475,7 @@ static int multiSelectOrderBy( VdbeNoopComment((v, "eof-B subroutine")); addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); - sqlite3VdbeGoto(v, addrEofB); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofB); } /* Generate code to handle the case of AB @@ -120551,11 +112507,11 @@ static int multiSelectOrderBy( sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); } sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); - sqlite3VdbeGoto(v, labelCmpr); + sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); /* This code runs once to initialize everything. */ - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeJumpHere(v, j1); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); @@ -120577,7 +112533,7 @@ static int multiSelectOrderBy( if( pDest->eDest==SRT_Output ){ Select *pFirst = pPrior; while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); + generateColumnNames(pParse, 0, pFirst->pEList); } /* Reassembly the compound query so that it will be freed correctly @@ -120598,7 +112554,7 @@ static int multiSelectOrderBy( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Forward Declarations */ static void substExprList(sqlite3*, ExprList*, int, ExprList*); -static void substSelect(sqlite3*, Select *, int, ExprList*, int); +static void substSelect(sqlite3*, Select *, int, ExprList *); /* ** Scan through the expression pExpr. Replace every reference to @@ -120635,7 +112591,7 @@ static Expr *substExpr( pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - substSelect(db, pExpr->x.pSelect, iTable, pEList, 1); + substSelect(db, pExpr->x.pSelect, iTable, pEList); }else{ substExprList(db, pExpr->x.pList, iTable, pEList); } @@ -120658,28 +112614,25 @@ static void substSelect( sqlite3 *db, /* Report malloc errors here */ Select *p, /* SELECT statement in which to make substitutions */ int iTable, /* Table to be replaced */ - ExprList *pEList, /* Substitute values */ - int doPrior /* Do substitutes on p->pPrior too */ + ExprList *pEList /* Substitute values */ ){ SrcList *pSrc; struct SrcList_item *pItem; int i; if( !p ) return; - do{ - substExprList(db, p->pEList, iTable, pEList); - substExprList(db, p->pGroupBy, iTable, pEList); - substExprList(db, p->pOrderBy, iTable, pEList); - p->pHaving = substExpr(db, p->pHaving, iTable, pEList); - p->pWhere = substExpr(db, p->pWhere, iTable, pEList); - pSrc = p->pSrc; - assert( pSrc!=0 ); + substExprList(db, p->pEList, iTable, pEList); + substExprList(db, p->pGroupBy, iTable, pEList); + substExprList(db, p->pOrderBy, iTable, pEList); + p->pHaving = substExpr(db, p->pHaving, iTable, pEList); + p->pWhere = substExpr(db, p->pWhere, iTable, pEList); + substSelect(db, p->pPrior, iTable, pEList); + pSrc = p->pSrc; + assert( pSrc ); /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */ + if( ALWAYS(pSrc) ){ for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(db, pItem->pSelect, iTable, pEList, 1); - if( pItem->fg.isTabFunc ){ - substExprList(db, pItem->u1.pFuncArg, iTable, pEList); - } + substSelect(db, pItem->pSelect, iTable, pEList); } - }while( doPrior && (p = p->pPrior)!=0 ); + } } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -120743,8 +112696,8 @@ static void substSelect( ** ** (**) Restriction (10) was removed from the code on 2005-02-05 but we ** accidently carried the comment forward until 2014-09-15. Original -** text: "The subquery does not use aggregates or the outer query -** does not use LIMIT." +** text: "The subquery does not use aggregates or the outer query does not +** use LIMIT." ** ** (11) The subquery and the outer query do not both have ORDER BY clauses. ** @@ -120831,7 +112784,7 @@ static int flattenSubquery( int subqueryIsAgg /* True if the subquery uses aggregate functions */ ){ const char *zSavedAuthContext = pParse->zAuthContext; - Select *pParent; /* Current UNION ALL term of the other query */ + Select *pParent; Select *pSub; /* The inner query or "subquery" */ Select *pSub1; /* Pointer to the rightmost select in sub-query */ SrcList *pSrc; /* The FROM clause of the outer query */ @@ -120934,7 +112887,7 @@ static int flattenSubquery( ** is fraught with danger. Best to avoid the whole thing. If the ** subquery is the right term of a LEFT JOIN, then do not flatten. */ - if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ + if( (pSubitem->jointype & JT_OUTER)!=0 ){ return 0; } @@ -120954,10 +112907,10 @@ static int flattenSubquery( testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); assert( pSub->pSrc!=0 ); - assert( pSub->pEList->nExpr==pSub1->pEList->nExpr ); if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 || (pSub1->pPrior && pSub1->op!=TK_ALL) || pSub1->pSrc->nSrc<1 + || pSub->pEList->nExpr!=pSub1->pEList->nExpr ){ return 0; } @@ -121105,7 +113058,7 @@ static int flattenSubquery( if( pSrc ){ assert( pParent==p ); /* First time through the loop */ - jointype = pSubitem->fg.jointype; + jointype = pSubitem->jointype; }else{ assert( pParent!=p ); /* 2nd and subsequent times through the loop */ pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); @@ -121126,9 +113079,9 @@ static int flattenSubquery( ** ** The outer query has 3 slots in its FROM clause. One slot of the ** outer query (the middle slot) is used by the subquery. The next - ** block of code will expand the outer query FROM clause to 4 slots. - ** The middle slot is expanded to two slots in order to make space - ** for the two elements in the FROM clause of the subquery. + ** block of code will expand the out query to 4 slots. The middle + ** slot is expanded to two slots in order to make space for the + ** two elements in the FROM clause of the subquery. */ if( nSubSrc>1 ){ pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); @@ -121142,11 +113095,10 @@ static int flattenSubquery( */ for(i=0; ia[i+iFrom].pUsing); - assert( pSrc->a[i+iFrom].fg.isTabFunc==0 ); pSrc->a[i+iFrom] = pSubSrc->a[i]; memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSrc->a[iFrom].fg.jointype = jointype; + pSrc->a[iFrom].jointype = jointype; /* Now begin substituting subquery result set expressions for ** references to the iParent in the outer query. @@ -121168,6 +113120,11 @@ static int flattenSubquery( pList->a[i].zName = zName; } } + substExprList(db, pParent->pEList, iParent, pSub->pEList); + if( isAgg ){ + substExprList(db, pParent->pGroupBy, iParent, pSub->pEList); + pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); + } if( pSub->pOrderBy ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th @@ -121187,21 +113144,27 @@ static int flattenSubquery( assert( pSub->pPrior==0 ); pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; + }else if( pParent->pOrderBy ){ + substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); + } + if( pSub->pWhere ){ + pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); + }else{ + pWhere = 0; } - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); if( subqueryIsAgg ){ assert( pParent->pHaving==0 ); pParent->pHaving = pParent->pWhere; pParent->pWhere = pWhere; - pParent->pHaving = sqlite3ExprAnd(db, - sqlite3ExprDup(db, pSub->pHaving, 0), pParent->pHaving - ); + pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); + pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, + sqlite3ExprDup(db, pSub->pHaving, 0)); assert( pParent->pGroupBy==0 ); pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); }else{ - pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); + pParent->pWhere = substExpr(db, pParent->pWhere, iParent, pSub->pEList); + pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); } - substSelect(db, pParent, iParent, pSub->pEList, 0); /* The flattened query is distinct if either the inner or the ** outer query is distinct. @@ -121227,7 +113190,7 @@ static int flattenSubquery( #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After flattening:\n")); + sqlite3DebugPrintf("After flattening:\n"); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -121236,83 +113199,6 @@ static int flattenSubquery( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ - - -#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) -/* -** Make copies of relevant WHERE clause terms of the outer query into -** the WHERE clause of subquery. Example: -** -** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1) WHERE x=5 AND y=10; -** -** Transformed into: -** -** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1 WHERE a=5 AND c-d=10) -** WHERE x=5 AND y=10; -** -** The hope is that the terms added to the inner query will make it more -** efficient. -** -** Do not attempt this optimization if: -** -** (1) The inner query is an aggregate. (In that case, we'd really want -** to copy the outer WHERE-clause terms onto the HAVING clause of the -** inner query. But they probably won't help there so do not bother.) -** -** (2) The inner query is the recursive part of a common table expression. -** -** (3) The inner query has a LIMIT clause (since the changes to the WHERE -** close would change the meaning of the LIMIT). -** -** (4) The inner query is the right operand of a LEFT JOIN. (The caller -** enforces this restriction since this routine does not have enough -** information to know.) -** -** (5) The WHERE clause expression originates in the ON or USING clause -** of a LEFT JOIN. -** -** Return 0 if no changes are made and non-zero if one or more WHERE clause -** terms are duplicated into the subquery. -*/ -static int pushDownWhereTerms( - sqlite3 *db, /* The database connection (for malloc()) */ - Select *pSubq, /* The subquery whose WHERE clause is to be augmented */ - Expr *pWhere, /* The WHERE clause of the outer query */ - int iCursor /* Cursor number of the subquery */ -){ - Expr *pNew; - int nChng = 0; - Select *pX; /* For looping over compound SELECTs in pSubq */ - if( pWhere==0 ) return 0; - for(pX=pSubq; pX; pX=pX->pPrior){ - if( (pX->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ - testcase( pX->selFlags & SF_Aggregate ); - testcase( pX->selFlags & SF_Recursive ); - testcase( pX!=pSubq ); - return 0; /* restrictions (1) and (2) */ - } - } - if( pSubq->pLimit!=0 ){ - return 0; /* restriction (3) */ - } - while( pWhere->op==TK_AND ){ - nChng += pushDownWhereTerms(db, pSubq, pWhere->pRight, iCursor); - pWhere = pWhere->pLeft; - } - if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */ - if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ - nChng++; - while( pSubq ){ - pNew = sqlite3ExprDup(db, pWhere, 0); - pNew = substExpr(db, pNew, iCursor, pSubq->pEList); - pSubq->pWhere = sqlite3ExprAnd(db, pSubq->pWhere, pNew); - pSubq = pSubq->pPrior; - } - } - return nChng; -} -#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ - /* ** Based on the contents of the AggInfo structure indicated by the first ** argument, this function checks if the following are true: @@ -121396,20 +113282,20 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ ** pFrom->pIndex and return SQLITE_OK. */ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ - if( pFrom->pTab && pFrom->fg.isIndexedBy ){ + if( pFrom->pTab && pFrom->zIndex ){ Table *pTab = pFrom->pTab; - char *zIndexedBy = pFrom->u1.zIndexedBy; + char *zIndex = pFrom->zIndex; Index *pIdx; for(pIdx=pTab->pIndex; - pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); + pIdx && sqlite3StrICmp(pIdx->zName, zIndex); pIdx=pIdx->pNext ); if( !pIdx ){ - sqlite3ErrorMsg(pParse, "no such index: %s", zIndexedBy, 0); + sqlite3ErrorMsg(pParse, "no such index: %s", zIndex, 0); pParse->checkSchema = 1; return SQLITE_ERROR; } - pFrom->pIBIndex = pIdx; + pFrom->pIndex = pIdx; } return SQLITE_OK; } @@ -121465,7 +113351,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ if( pNewSrc==0 ) return WRC_Abort; *pNew = *p; p->pSrc = pNewSrc; - p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0)); + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0)); p->op = TK_SELECT; p->pWhere = 0; pNew->pGroupBy = 0; @@ -121484,19 +113370,6 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ return WRC_Continue; } -/* -** Check to see if the FROM clause term pFrom has table-valued function -** arguments. If it does, leave an error message in pParse and return -** non-zero, since pFrom is not allowed to be a table-valued function. -*/ -static int cannotBeFunction(Parse *pParse, struct SrcList_item *pFrom){ - if( pFrom->fg.isTabFunc ){ - sqlite3ErrorMsg(pParse, "'%s' is not a function", pFrom->zName); - return 1; - } - return 0; -} - #ifndef SQLITE_OMIT_CTE /* ** Argument pWith (which may be NULL) points to a linked list of nested @@ -121509,7 +113382,7 @@ static int cannotBeFunction(Parse *pParse, struct SrcList_item *pFrom){ ** object that the returned CTE belongs to. */ static struct Cte *searchWith( - With *pWith, /* Current innermost WITH clause */ + With *pWith, /* Current outermost WITH clause */ struct SrcList_item *pItem, /* FROM clause element to resolve */ With **ppContext /* OUT: WITH clause return value belongs to */ ){ @@ -121540,12 +113413,11 @@ static struct Cte *searchWith( ** statement with which it is associated. */ SQLITE_PRIVATE void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ - assert( bFree==0 || (pParse->pWith==0 && pParse->pWithToFree==0) ); + assert( bFree==0 || pParse->pWith==0 ); if( pWith ){ - assert( pParse->pWith!=pWith ); pWith->pOuter = pParse->pWith; pParse->pWith = pWith; - if( bFree ) pParse->pWithToFree = pWith; + pParse->bFreeWith = bFree; } } @@ -121584,15 +113456,14 @@ static int withExpand( int bMayRecursive; /* True if compound joined by UNION [ALL] */ With *pSavedWith; /* Initial value of pParse->pWith */ - /* If pCte->zCteErr is non-NULL at this point, then this is an illegal + /* If pCte->zErr is non-NULL at this point, then this is an illegal ** recursive reference to CTE pCte. Leave an error in pParse and return - ** early. If pCte->zCteErr is NULL, then this is not a recursive reference. + ** early. If pCte->zErr is NULL, then this is not a recursive reference. ** In this case, proceed. */ - if( pCte->zCteErr ){ - sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName); + if( pCte->zErr ){ + sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); return SQLITE_ERROR; } - if( cannotBeFunction(pParse, pFrom) ) return SQLITE_ERROR; assert( pFrom->pTab==0 ); pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); @@ -121601,9 +113472,9 @@ static int withExpand( pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; + pTab->tabFlags |= TF_Ephemeral; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); - if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; + if( db->mallocFailed ) return SQLITE_NOMEM; assert( pFrom->pSelect ); /* Check if this is a recursive CTE. */ @@ -121619,7 +113490,7 @@ static int withExpand( && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ pItem->pTab = pTab; - pItem->fg.isRecursive = 1; + pItem->isRecursive = 1; pTab->nRef++; pSel->selFlags |= SF_Recursive; } @@ -121635,11 +113506,10 @@ static int withExpand( } assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); - pCte->zCteErr = "circular reference: %s"; + pCte->zErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); - pParse->pWith = pWith; for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); pEList = pLeft->pEList; @@ -121654,16 +113524,16 @@ static int withExpand( pEList = pCte->pCols; } - sqlite3ColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); if( bMayRecursive ){ if( pSel->selFlags & SF_Recursive ){ - pCte->zCteErr = "multiple recursive references: %s"; + pCte->zErr = "multiple recursive references: %s"; }else{ - pCte->zCteErr = "recursive reference in a subquery: %s"; + pCte->zErr = "recursive reference in a subquery: %s"; } sqlite3WalkSelect(pWalker, pSel); } - pCte->zCteErr = 0; + pCte->zErr = 0; pParse->pWith = pSavedWith; } @@ -121750,9 +113620,17 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); - if( pFrom->fg.isRecursive ) continue; - assert( pFrom->pTab==0 ); + assert( pFrom->isRecursive==0 || pFrom->pTab ); + if( pFrom->isRecursive ) continue; + if( pFrom->pTab!=0 ){ + /* This statement has already been prepared. There is no need + ** to go further. */ + assert( i==0 ); +#ifndef SQLITE_OMIT_CTE + selectPopWith(pWalker, p); +#endif + return WRC_Prune; + } #ifndef SQLITE_OMIT_CTE if( withExpand(pWalker, pFrom) ) return WRC_Abort; if( pFrom->pTab ) {} else @@ -121769,7 +113647,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pTab->nRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } - sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); + selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; @@ -121786,20 +113664,14 @@ static int selectExpander(Walker *pWalker, Select *p){ return WRC_Abort; } pTab->nRef++; - if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){ - return WRC_Abort; - } #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) - if( IsVirtual(pTab) || pTab->pSelect ){ - i16 nCol; + if( pTab->pSelect || IsVirtual(pTab) ){ + /* We reach here if the named table is a really a view */ if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); sqlite3SelectSetName(pFrom->pSelect, pTab->zName); - nCol = pTab->nCol; - pTab->nCol = -1; sqlite3WalkSelect(pWalker, pFrom->pSelect); - pTab->nCol = nCol; } #endif } @@ -121819,20 +113691,19 @@ static int selectExpander(Walker *pWalker, Select *p){ /* For every "*" that occurs in the column list, insert the names of ** all columns in all tables. And for every TABLE.* insert the names ** of all columns in TABLE. The parser inserted a special expression - ** with the TK_ASTERISK operator for each "*" that it found in the column - ** list. The following code just has to locate the TK_ASTERISK - ** expressions and expand each one to the list of all columns in - ** all tables. + ** with the TK_ALL operator for each "*" that it found in the column list. + ** The following code just has to locate the TK_ALL expressions and expand + ** each one to the list of all columns in all tables. ** ** The first loop just checks to see if there are any "*" operators ** that need expanding. */ for(k=0; knExpr; k++){ pE = pEList->a[k].pExpr; - if( pE->op==TK_ASTERISK ) break; + if( pE->op==TK_ALL ) break; assert( pE->op!=TK_DOT || pE->pRight!=0 ); assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); - if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break; + if( pE->op==TK_DOT && pE->pRight->op==TK_ALL ) break; } if( knExpr ){ /* @@ -121846,13 +113717,18 @@ static int selectExpander(Walker *pWalker, Select *p){ int longNames = (flags & SQLITE_FullColNames)!=0 && (flags & SQLITE_ShortColNames)==0; + /* When processing FROM-clause subqueries, it is always the case + ** that full_column_names=OFF and short_column_names=ON. The + ** sqlite3ResultSetOfSelect() routine makes it so. */ + assert( (p->selFlags & SF_NestedFrom)==0 + || ((flags & SQLITE_FullColNames)==0 && + (flags & SQLITE_ShortColNames)!=0) ); + for(k=0; knExpr; k++){ pE = a[k].pExpr; pRight = pE->pRight; assert( pE->op!=TK_DOT || pRight!=0 ); - if( pE->op!=TK_ASTERISK - && (pE->op!=TK_DOT || pRight->op!=TK_ASTERISK) - ){ + if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pRight->op!=TK_ALL) ){ /* This particular expression does not need to be expanded. */ pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); @@ -121889,7 +113765,7 @@ static int selectExpander(Walker *pWalker, Select *p){ continue; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - zSchemaName = iDb>=0 ? db->aDb[iDb].zDbSName : "*"; + zSchemaName = iDb>=0 ? db->aDb[iDb].zName : "*"; } for(j=0; jnCol; j++){ char *zName = pTab->aCol[j].zName; @@ -121904,19 +113780,18 @@ static int selectExpander(Walker *pWalker, Select *p){ continue; } - /* If a column is marked as 'hidden', omit it from the expanded - ** result-set list unless the SELECT has the SF_IncludeHidden - ** bit set. + /* If a column is marked as 'hidden' (currently only possible + ** for virtual tables), do not include it in the expanded + ** result-set list. */ - if( (p->selFlags & SF_IncludeHidden)==0 - && IsHiddenColumn(&pTab->aCol[j]) - ){ + if( IsHiddenColumn(&pTab->aCol[j]) ){ + assert(IsVirtual(pTab)); continue; } tableSeen = 1; if( i>0 && zTName==0 ){ - if( (pFrom->fg.jointype & JT_NATURAL)!=0 + if( (pFrom->jointype & JT_NATURAL)!=0 && tableAndColumnIndex(pTabList, i, zName, 0, 0) ){ /* In a NATURAL join, omit the join columns from the @@ -121948,7 +113823,8 @@ static int selectExpander(Walker *pWalker, Select *p){ pExpr = pRight; } pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); - sqlite3TokenInit(&sColname, zColname); + sColname.z = zColname; + sColname.n = sqlite3Strlen30(zColname); sqlite3ExprListSetName(pParse, pNew, &sColname, 0); if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; @@ -121980,7 +113856,6 @@ static int selectExpander(Walker *pWalker, Select *p){ #if SQLITE_MAX_COLUMN if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many columns in result set"); - return WRC_Abort; } #endif return WRC_Continue; @@ -121995,7 +113870,7 @@ static int selectExpander(Walker *pWalker, Select *p){ ** Walker.xSelectCallback is set to do something useful for every ** subquery in the parser tree. */ -SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ +static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ UNUSED_PARAMETER2(NotUsed, NotUsed2); return WRC_Continue; } @@ -122016,7 +113891,7 @@ SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; memset(&w, 0, sizeof(w)); - w.xExprCallback = sqlite3ExprWalkNoop; + w.xExprCallback = exprWalkNoop; w.pParse = pParse; if( pParse->hasCompound ){ w.xSelectCallback = convertCompoundSelectToSubquery; @@ -122051,19 +113926,19 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ struct SrcList_item *pFrom; assert( p->selFlags & SF_Resolved ); - assert( (p->selFlags & SF_HasTypeInfo)==0 ); - p->selFlags |= SF_HasTypeInfo; - pParse = pWalker->pParse; - pTabList = p->pSrc; - for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - assert( pTab!=0 ); - if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ - /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - if( pSel ){ - while( pSel->pPrior ) pSel = pSel->pPrior; - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel); + if( (p->selFlags & SF_HasTypeInfo)==0 ){ + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab, pSel); + } } } } @@ -122083,7 +113958,7 @@ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ Walker w; memset(&w, 0, sizeof(w)); w.xSelectCallback2 = selectAddSubqueryTypeInfo; - w.xExprCallback = sqlite3ExprWalkNoop; + w.xExprCallback = exprWalkNoop; w.pParse = pParse; sqlite3WalkSelect(&w, pSelect); #endif @@ -122202,7 +114077,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); - sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); + sqlite3ExprCodeExprList(pParse, pList, regAgg, SQLITE_ECEL_DUP); }else{ nArg = 0; regAgg = 0; @@ -122227,7 +114102,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_AggStep0, 0, regAgg, pF->iMem, + sqlite3VdbeAddOp4(v, OP_AggStep, 0, regAgg, pF->iMem, (void*)pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); @@ -122310,7 +114185,7 @@ SQLITE_PRIVATE int sqlite3Select( WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */ Vdbe *v; /* The virtual machine under construction */ int isAgg; /* True for select lists like "count(*)" */ - ExprList *pEList = 0; /* List of columns to extract. */ + ExprList *pEList; /* List of columns to extract. */ SrcList *pTabList; /* List of tables to select from */ Expr *pWhere; /* The WHERE clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ @@ -122360,11 +114235,12 @@ SQLITE_PRIVATE int sqlite3Select( memset(&sSort, 0, sizeof(sSort)); sSort.pOrderBy = p->pOrderBy; pTabList = p->pSrc; + pEList = p->pEList; if( pParse->nErr || db->mallocFailed ){ goto select_end; } - assert( p->pEList!=0 ); isAgg = (p->selFlags & SF_Aggregate)!=0; + assert( pEList!=0 ); #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x100 ){ SELECTTRACE(0x100,pParse,p, ("after name resolution:\n")); @@ -122372,68 +114248,30 @@ SQLITE_PRIVATE int sqlite3Select( } #endif - /* Try to flatten subqueries in the FROM clause up into the main query - */ -#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) - for(i=0; !p->pPrior && inSrc; i++){ - struct SrcList_item *pItem = &pTabList->a[i]; - Select *pSub = pItem->pSelect; - int isAggSub; - Table *pTab = pItem->pTab; - if( pSub==0 ) continue; - - /* Catch mismatch in the declared columns of a view and the number of - ** columns in the SELECT on the RHS */ - if( pTab->nCol!=pSub->pEList->nExpr ){ - sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d", - pTab->nCol, pTab->zName, pSub->pEList->nExpr); - goto select_end; - } - isAggSub = (pSub->selFlags & SF_Aggregate)!=0; - if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ - /* This subquery can be absorbed into its parent. */ - if( isAggSub ){ - isAgg = 1; - p->selFlags |= SF_Aggregate; - } - i = -1; - } - pTabList = p->pSrc; - if( db->mallocFailed ) goto select_end; - if( !IgnorableOrderby(pDest) ){ - sSort.pOrderBy = p->pOrderBy; - } - } -#endif - - /* Get a pointer the VDBE under construction, allocating a new VDBE if one - ** does not already exist */ + /* Begin generating code. + */ v = sqlite3GetVdbe(pParse); if( v==0 ) goto select_end; -#ifndef SQLITE_OMIT_COMPOUND_SELECT - /* Handle compound SELECT statements using the separate multiSelect() - ** procedure. + /* If writing to memory or generating a set + ** only a single column may be output. */ - if( p->pPrior ){ - rc = multiSelect(pParse, p, pDest); - explainSetInteger(pParse->iSelectId, iRestoreSelectId); -#if SELECTTRACE_ENABLED - SELECTTRACE(1,pParse,p,("end compound-select processing\n")); - pParse->nSelectIndent--; -#endif - return rc; +#ifndef SQLITE_OMIT_SUBQUERY + if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ + goto select_end; } #endif /* Generate code for all sub-queries in the FROM clause */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) - for(i=0; inSrc; i++){ + for(i=0; !p->pPrior && inSrc; i++){ struct SrcList_item *pItem = &pTabList->a[i]; SelectDest dest; Select *pSub = pItem->pSelect; + int isAggSub; + if( pSub==0 ) continue; /* Sometimes the code for a subquery will be generated more than @@ -122443,7 +114281,7 @@ SQLITE_PRIVATE int sqlite3Select( ** is sufficient, though the subroutine to manifest the view does need ** to be invoked again. */ if( pItem->addrFillSub ){ - if( pItem->fg.viaCoroutine==0 ){ + if( pItem->viaCoroutine==0 ){ sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); } continue; @@ -122458,39 +114296,16 @@ SQLITE_PRIVATE int sqlite3Select( */ pParse->nHeight += sqlite3SelectExprHeight(p); - /* Make copies of constant WHERE-clause terms in the outer query down - ** inside the subquery. This can help the subquery to run more efficiently. - */ - if( (pItem->fg.jointype & JT_OUTER)==0 - && pushDownWhereTerms(db, pSub, p->pWhere, pItem->iCursor) - ){ -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After WHERE-clause push-down:\n")); - sqlite3TreeViewSelect(0, p, 0); + isAggSub = (pSub->selFlags & SF_Aggregate)!=0; + if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ + /* This subquery can be absorbed into its parent. */ + if( isAggSub ){ + isAgg = 1; + p->selFlags |= SF_Aggregate; } -#endif - } - - /* Generate code to implement the subquery - ** - ** The subquery is implemented as a co-routine if all of these are true: - ** (1) The subquery is guaranteed to be the outer loop (so that it - ** does not need to be computed more than once) - ** (2) The ALL keyword after SELECT is omitted. (Applications are - ** allowed to say "SELECT ALL" instead of just "SELECT" to disable - ** the use of co-routines.) - ** (3) Co-routines are not disabled using sqlite3_test_control() - ** with SQLITE_TESTCTRL_OPTIMIZATIONS. - ** - ** TODO: Are there other reasons beside (1) to use a co-routine - ** implementation? - */ - if( i==0 - && (pTabList->nSrc==1 - || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ - && (p->selFlags & SF_All)==0 /* (2) */ - && OptimizationEnabled(db, SQLITE_SubqCoroutine) /* (3) */ + i = -1; + }else if( pTabList->nSrc==1 + && OptimizationEnabled(db, SQLITE_SubqCoroutine) ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. @@ -122503,10 +114318,10 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; - pItem->fg.viaCoroutine = 1; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + pItem->viaCoroutine = 1; pItem->regResult = dest.iSdst; - sqlite3VdbeEndCoroutine(v, pItem->regReturn); + sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); }else{ @@ -122522,11 +114337,11 @@ SQLITE_PRIVATE int sqlite3Select( pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); pItem->addrFillSub = topAddr+1; - if( pItem->fg.isCorrelated==0 ){ + if( pItem->isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ - onceAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + onceAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); }else{ VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); @@ -122534,30 +114349,40 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); sqlite3VdbeChangeP1(v, topAddr, retAddr); sqlite3ClearTempRegCache(pParse); } - if( db->mallocFailed ) goto select_end; + if( /*pParse->nErr ||*/ db->mallocFailed ){ + goto select_end; + } pParse->nHeight -= sqlite3SelectExprHeight(p); + pTabList = p->pSrc; + if( !IgnorableOrderby(pDest) ){ + sSort.pOrderBy = p->pOrderBy; + } } -#endif - - /* Various elements of the SELECT copied into local variables for - ** convenience */ pEList = p->pEList; +#endif pWhere = p->pWhere; pGroupBy = p->pGroupBy; pHaving = p->pHaving; sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; +#ifndef SQLITE_OMIT_COMPOUND_SELECT + /* If there is are a sequence of queries, do the earlier ones first. + */ + if( p->pPrior ){ + rc = multiSelect(pParse, p, pDest); + explainSetInteger(pParse->iSelectId, iRestoreSelectId); #if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); - sqlite3TreeViewSelect(0, p, 0); + SELECTTRACE(1,pParse,p,("end compound-select processing\n")); + pParse->nSelectIndent--; +#endif + return rc; } #endif @@ -122577,30 +114402,23 @@ SQLITE_PRIVATE int sqlite3Select( ** BY and DISTINCT, and an index or separate temp-table for the other. */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct - && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 + && sqlite3ExprListCompare(sSort.pOrderBy, p->pEList, -1)==0 ){ p->selFlags &= ~SF_Distinct; - pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); + p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); + pGroupBy = p->pGroupBy; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ assert( sDistinct.isTnct ); - -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); - sqlite3TreeViewSelect(0, p, 0); - } -#endif } - /* If there is an ORDER BY clause, then create an ephemeral index to - ** do the sorting. But this sorting ephemeral index might end up - ** being unused if the data can be extracted in pre-sorted order. - ** If that is the case, then the OP_OpenEphemeral instruction will be - ** changed to an OP_Noop once we figure out that the sorting index is - ** not needed. The sSort.addrSortIndex variable is used to facilitate - ** that change. + /* If there is an ORDER BY clause, then this sorting + ** index might end up being unused if the data can be + ** extracted in pre-sorted order. If that is the case, then the + ** OP_OpenEphemeral instruction will be changed to an OP_Noop once + ** we figure out that the sorting index is not needed. The addrSortIndex + ** variable is used to facilitate that change. */ if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; @@ -122624,21 +114442,21 @@ SQLITE_PRIVATE int sqlite3Select( /* Set the limiter. */ iEnd = sqlite3VdbeMakeLabel(v); - p->nSelectRow = 320; /* 4 billion rows */ + p->nSelectRow = LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ - sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); + sqlite3VdbeGetOp(v, sSort.addrSortIndex)->opcode = OP_SorterOpen; sSort.sortFlags |= SORTFLAG_UseSorter; } - /* Open an ephemeral index to use for the distinct set. + /* Open a virtual index to use for the distinct set. */ if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList,0,0), - P4_KEYINFO); + sDistinct.tabTnct, 0, 0, + (char*)keyInfoFromExprList(pParse, p->pEList,0,0), + P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ @@ -122648,12 +114466,10 @@ SQLITE_PRIVATE int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); - assert( WHERE_USE_LIMIT==SF_FixedLimit ); - wctrlFlags |= p->selFlags & SF_FixedLimit; /* Begin the database scan. */ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, - p->pEList, wctrlFlags, p->nSelectRow); + p->pEList, wctrlFlags, 0); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); @@ -122663,7 +114479,6 @@ SQLITE_PRIVATE int sqlite3Select( } if( sSort.pOrderBy ){ sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); - sSort.bOrderedInnerLoop = sqlite3WhereOrderedInnerLoop(pWInfo); if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ sSort.pOrderBy = 0; } @@ -122714,17 +114529,16 @@ SQLITE_PRIVATE int sqlite3Select( for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ pItem->u.x.iAlias = 0; } - assert( 66==sqlite3LogEst(100) ); - if( p->nSelectRow>66 ) p->nSelectRow = 66; + if( p->nSelectRow>100 ) p->nSelectRow = 100; }else{ - assert( 0==sqlite3LogEst(1) ); - p->nSelectRow = 0; + p->nSelectRow = 1; } + /* If there is both a GROUP BY and an ORDER BY clause and they are ** identical, then it may be possible to disable the ORDER BY clause ** on the grounds that the GROUP BY will cause elements to come out - ** in the correct order. It also may not - the GROUP BY might use a + ** in the correct order. It also may not - the GROUP BY may use a ** database index that causes rows to be grouped together as required ** but not actually sorted. Either way, record the fact that the ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp @@ -122767,7 +114581,7 @@ SQLITE_PRIVATE int sqlite3Select( */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - int addr1; /* A-vs-B comparision jump */ + int j1; /* A-vs-B comparision jump */ int addrOutputRow; /* Start of subroutine that outputs a result row */ int regOutputRow; /* Return address register for output subroutine */ int addrSetAbort; /* Set the abort flag and return */ @@ -122848,14 +114662,19 @@ SQLITE_PRIVATE int sqlite3Select( } regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); + sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); j = nGroupBy; for(i=0; iiSorterColumn>=j ){ int r1 = j + regBase; - sqlite3ExprCodeGetColumnToReg(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1); + int r2; + + r2 = sqlite3ExprCodeGetColumn(pParse, + pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); + if( r1!=r2 ){ + sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); + } j++; } } @@ -122897,8 +114716,7 @@ SQLITE_PRIVATE int sqlite3Select( addrTopOfLoop = sqlite3VdbeCurrentAddr(v); sqlite3ExprCacheClear(pParse); if( groupBySort ){ - sqlite3VdbeAddOp3(v, OP_SorterData, sAggInfo.sortingIdx, - sortOut, sortPTab); + sqlite3VdbeAddOp3(v, OP_SorterData, sAggInfo.sortingIdx, sortOut,sortPTab); } for(j=0; jnExpr; j++){ if( groupBySort ){ @@ -122910,8 +114728,8 @@ SQLITE_PRIVATE int sqlite3Select( } sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - addr1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_Jump, addr1+1, 0, addr1+1); VdbeCoverage(v); + j1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); VdbeCoverage(v); /* Generate code that runs whenever the GROUP BY changes. ** Changes in the GROUP BY are detected by the previous code @@ -122933,7 +114751,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Update the aggregate accumulators based on the content of ** the current row */ - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeJumpHere(v, j1); updateAccumulator(pParse, &sAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); @@ -122955,7 +114773,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Jump over the subroutines */ - sqlite3VdbeGoto(v, addrEnd); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEnd); /* Generate a subroutine that outputs a single row of the result ** set. This subroutine first looks at the iUseFlag. If iUseFlag @@ -122970,8 +114788,7 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); sqlite3VdbeResolveLabel(v, addrOutputRow); addrOutputRow = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); - VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeCoverage(v); VdbeComment((v, "Groupby result generator entry point")); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); @@ -123090,8 +114907,7 @@ SQLITE_PRIVATE int sqlite3Select( if( flag ){ pMinMax = sqlite3ExprListDup(db, pMinMax, 0); pDel = pMinMax; - assert( db->mallocFailed || pMinMax!=0 ); - if( !db->mallocFailed ){ + if( pMinMax && !db->mallocFailed ){ pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; pMinMax->a[0].pExpr->op = TK_COLUMN; } @@ -123110,7 +114926,7 @@ SQLITE_PRIVATE int sqlite3Select( updateAccumulator(pParse, &sAggInfo); assert( pMinMax==0 || pMinMax->nExpr==1 ); if( sqlite3WhereIsOrdered(pWInfo)>0 ){ - sqlite3VdbeGoto(v, sqlite3WhereBreakLabel(pWInfo)); + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); } @@ -123136,8 +114952,7 @@ SQLITE_PRIVATE int sqlite3Select( ** and send them to the callback one by one. */ if( sSort.pOrderBy ){ - explainTempTable(pParse, - sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + explainTempTable(pParse, sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } @@ -123170,6 +114985,100 @@ SQLITE_PRIVATE int sqlite3Select( return rc; } +#ifdef SQLITE_DEBUG +/* +** Generate a human-readable description of a the Select object. +*/ +SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ + int n = 0; + pView = sqlite3TreeViewPush(pView, moreToFollow); + sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p)", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p + ); + if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pWhere ) n++; + if( p->pGroupBy ) n++; + if( p->pHaving ) n++; + if( p->pOrderBy ) n++; + if( p->pLimit ) n++; + if( p->pOffset ) n++; + if( p->pPrior ) n++; + sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); + if( p->pSrc && p->pSrc->nSrc ){ + int i; + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "FROM"); + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); + }else if( pItem->zName ){ + sqlite3XPrintf(&x, 0, " %s", pItem->zName); + } + if( pItem->pTab ){ + sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName); + } + if( pItem->zAlias ){ + sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); + } + if( pItem->jointype & JT_LEFT ){ + sqlite3XPrintf(&x, 0, " LEFT-JOIN"); + } + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); + } + if( p->pWhere ){ + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); + sqlite3TreeViewExpr(pView, p->pWhere, 0); + sqlite3TreeViewPop(pView); + } + if( p->pGroupBy ){ + sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); + } + if( p->pHaving ){ + sqlite3TreeViewItem(pView, "HAVING", (n--)>0); + sqlite3TreeViewExpr(pView, p->pHaving, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOrderBy ){ + sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); + } + if( p->pLimit ){ + sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); + sqlite3TreeViewExpr(pView, p->pLimit, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOffset ){ + sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewExpr(pView, p->pOffset, 0); + sqlite3TreeViewPop(pView); + } + if( p->pPrior ){ + const char *zOp = "UNION"; + switch( p->op ){ + case TK_ALL: zOp = "UNION ALL"; break; + case TK_INTERSECT: zOp = "INTERSECT"; break; + case TK_EXCEPT: zOp = "EXCEPT"; break; + } + sqlite3TreeViewItem(pView, zOp, (n--)>0); + sqlite3TreeViewSelect(pView, p->pPrior, 0); + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); +} +#endif /* SQLITE_DEBUG */ + /************** End of select.c **********************************************/ /************** Begin file table.c *******************************************/ /* @@ -123190,7 +115099,6 @@ SQLITE_PRIVATE int sqlite3Select( ** These routines are in a separate files so that they will not be linked ** if they are not used. */ -/* #include "sqliteInt.h" */ /* #include */ /* #include */ @@ -123275,7 +115183,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ return 0; malloc_failed: - p->rc = SQLITE_NOMEM_BKPT; + p->rc = SQLITE_NOMEM; return 1; } @@ -123289,7 +115197,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ ** Instead, the entire table should be passed to sqlite3_free_table() when ** the calling procedure is finished using it. */ -SQLITE_API int sqlite3_get_table( +SQLITE_API int SQLITE_STDCALL sqlite3_get_table( sqlite3 *db, /* The database on which the SQL executes */ const char *zSql, /* The SQL to be executed */ char ***pazResult, /* Write the result table here */ @@ -123316,7 +115224,7 @@ SQLITE_API int sqlite3_get_table( res.azResult = sqlite3_malloc64(sizeof(char*)*res.nAlloc ); if( res.azResult==0 ){ db->errCode = SQLITE_NOMEM; - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } res.azResult[0] = 0; rc = sqlite3_exec(db, zSql, sqlite3_get_table_cb, &res, pzErrMsg); @@ -123345,7 +115253,7 @@ SQLITE_API int sqlite3_get_table( if( azNew==0 ){ sqlite3_free_table(&res.azResult[1]); db->errCode = SQLITE_NOMEM; - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } res.azResult = azNew; } @@ -123358,7 +115266,7 @@ SQLITE_API int sqlite3_get_table( /* ** This routine frees the space the sqlite3_get_table() malloced. */ -SQLITE_API void sqlite3_free_table( +SQLITE_API void SQLITE_STDCALL sqlite3_free_table( char **azResult /* Result returned from sqlite3_get_table() */ ){ if( azResult ){ @@ -123387,7 +115295,6 @@ SQLITE_API void sqlite3_free_table( ************************************************************************* ** This file contains the implementation for TRIGGERs */ -/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_TRIGGER /* @@ -123473,6 +115380,7 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( int iDb; /* The database to store the trigger in */ Token *pName; /* The unqualified db name */ DbFixer sFix; /* State vector for the DB fixer */ + int iTabDb; /* Index of the database holding pTab */ assert( pName1!=0 ); /* pName1->z might be NULL, but not pName1 itself */ assert( pName2!=0 ); @@ -123585,13 +115493,13 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( " trigger on table: %S", pTableName, 0); goto trigger_cleanup; } + iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); #ifndef SQLITE_OMIT_AUTHORIZATION { - int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); int code = SQLITE_CREATE_TRIGGER; - const char *zDb = db->aDb[iTabDb].zDbSName; - const char *zDbTrig = isTemp ? db->aDb[1].zDbSName : zDb; + const char *zDb = db->aDb[iTabDb].zName; + const char *zDbTrig = isTemp ? db->aDb[1].zName : zDb; if( iTabDb==1 || isTemp ) code = SQLITE_CREATE_TEMP_TRIGGER; if( sqlite3AuthCheck(pParse, code, zName, pTab->zName, zDbTrig) ){ goto trigger_cleanup; @@ -123663,7 +115571,8 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( pStepList->pTrig = pTrig; pStepList = pStepList->pNext; } - sqlite3TokenInit(&nameToken, pTrig->zName); + nameToken.z = pTrig->zName; + nameToken.n = sqlite3Strlen30(nameToken.z); sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken); if( sqlite3FixTriggerStep(&sFix, pTrig->step_list) || sqlite3FixExpr(&sFix, pTrig->pWhen) @@ -123685,7 +115594,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n); sqlite3NestedParse(pParse, "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')", - db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), zName, + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName, pTrig->table, z); sqlite3DbFree(db, z); sqlite3ChangeCookie(pParse, iDb); @@ -123699,7 +115608,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pTrig = sqlite3HashInsert(pHash, zName, pTrig); if( pTrig ){ - sqlite3OomFault(db); + db->mallocFailed = 1; }else if( pLink->pSchema==pLink->pTabSchema ){ Table *pTab; pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table); @@ -123874,7 +115783,7 @@ SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr) assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDb && sqlite3StrICmp(db->aDb[j].zDbSName, zDb) ) continue; + if( zDb && sqlite3StrICmp(db->aDb[j].zName, zDb) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName); if( pTrigger ) break; @@ -123920,7 +115829,7 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ #ifndef SQLITE_OMIT_AUTHORIZATION { int code = SQLITE_DROP_TRIGGER; - const char *zDb = db->aDb[iDb].zDbSName; + const char *zDb = db->aDb[iDb].zName; const char *zTab = SCHEMA_TABLE(iDb); if( iDb==1 ) code = SQLITE_DROP_TEMP_TRIGGER; if( sqlite3AuthCheck(pParse, code, pTrigger->zName, pTable->zName, zDb) || @@ -123934,12 +115843,31 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ */ assert( pTable!=0 ); if( (v = sqlite3GetVdbe(pParse))!=0 ){ - sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE name=%Q AND type='trigger'", - db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), pTrigger->zName - ); + int base; + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList dropTrigger[] = { + { OP_Rewind, 0, ADDR(9), 0}, + { OP_String8, 0, 1, 0}, /* 1 */ + { OP_Column, 0, 1, 2}, + { OP_Ne, 2, ADDR(8), 1}, + { OP_String8, 0, 1, 0}, /* 4: "trigger" */ + { OP_Column, 0, 0, 2}, + { OP_Ne, 2, ADDR(8), 1}, + { OP_Delete, 0, 0, 0}, + { OP_Next, 0, ADDR(1), 0}, /* 8 */ + }; + + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3OpenMasterTable(pParse, iDb); + base = sqlite3VdbeAddOpList(v, ArraySize(dropTrigger), dropTrigger, iLn); + sqlite3VdbeChangeP4(v, base+1, pTrigger->zName, P4_TRANSIENT); + sqlite3VdbeChangeP4(v, base+4, "trigger", P4_STATIC); sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddOp2(v, OP_Close, 0, 0); sqlite3VdbeAddOp4(v, OP_DropTrigger, iDb, 0, 0, pTrigger->zName, 0); + if( pParse->nMem<3 ){ + pParse->nMem = 3; + } } } @@ -124039,10 +115967,8 @@ static SrcList *targetSrcList( pSrc->a[pSrc->nSrc-1].zName = sqlite3DbStrDup(db, pStep->zTarget); iDb = sqlite3SchemaToIndex(db, pStep->pTrig->pSchema); if( iDb==0 || iDb>=2 ){ - const char *zDb; assert( iDbnDb ); - zDb = db->aDb[iDb].zDbSName; - pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, zDb); + pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); } } return pSrc; @@ -124256,6 +116182,7 @@ static TriggerPrg *codeRowTrigger( } pProgram->nMem = pSubParse->nMem; pProgram->nCsr = pSubParse->nTab; + pProgram->nOnce = pSubParse->nOnce; pProgram->token = (void *)pTrigger; pPrg->aColmask[0] = pSubParse->oldmask; pPrg->aColmask[1] = pSubParse->newmask; @@ -124328,8 +116255,8 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( if( pPrg ){ int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); - sqlite3VdbeAddOp4(v, OP_Program, reg, ignoreJump, ++pParse->nMem, - (const char *)pPrg->pProgram, P4_SUBPROGRAM); + sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem); + sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM); VdbeComment( (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf))); @@ -124491,7 +116418,6 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask( ** This file contains C code routines that are called by the parser ** to handle UPDATE statements. */ -/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_VIRTUALTABLE /* Forward declaration */ @@ -124613,9 +116539,9 @@ SQLITE_PRIVATE void sqlite3Update( /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ - int regOldRowid = 0; /* The old rowid */ - int regNewRowid = 0; /* The new rowid */ - int regNew = 0; /* Content of the NEW.* table in triggers */ + int regOldRowid; /* The old rowid */ + int regNewRowid; /* The new rowid */ + int regNew; /* Content of the NEW.* table in triggers */ int regOld = 0; /* Content of OLD.* table in triggers */ int regRowSet = 0; /* Rowset of rows to be updated */ int regKey = 0; /* composite PRIMARY KEY value */ @@ -124676,7 +116602,7 @@ SQLITE_PRIVATE void sqlite3Update( /* Allocate space for aXRef[], aRegIdx[], and aToOpen[]. ** Initialize aXRef[] and aToOpen[] to their default values. */ - aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); + aXRef = sqlite3DbMallocRaw(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); if( aXRef==0 ) goto update_cleanup; aRegIdx = aXRef+pTab->nCol; aToOpen = (u8*)(aRegIdx+nIdx); @@ -124728,7 +116654,7 @@ SQLITE_PRIVATE void sqlite3Update( int rc; rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, j<0 ? "ROWID" : pTab->aCol[j].zName, - db->aDb[iDb].zDbSName); + db->aDb[iDb].zName); if( rc==SQLITE_DENY ){ goto update_cleanup; }else if( rc==SQLITE_IGNORE ){ @@ -124742,20 +116668,16 @@ SQLITE_PRIVATE void sqlite3Update( assert( chngPk==0 || chngPk==1 ); chngKey = chngRowid + chngPk; - /* The SET expressions are not actually used inside the WHERE loop. - ** So reset the colUsed mask. Unless this is a virtual table. In that - ** case, set all bits of the colUsed mask (to ensure that the virtual - ** table implementation makes all columns available). + /* The SET expressions are not actually used inside the WHERE loop. + ** So reset the colUsed mask */ - pTabList->a[0].colUsed = IsVirtual(pTab) ? ALLBITS : 0; + pTabList->a[0].colUsed = 0; hasFK = sqlite3FkRequired(pParse, pTab, aXRef, chngKey); /* There is one entry in the aRegIdx[] array for each index on the table ** being updated. Fill in aRegIdx[] with a register number that will hold - ** the key for accessing each index. - ** - ** FIXME: Be smarter about omitting indexes that use expressions. + ** the key for accessing each index. */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; @@ -124764,8 +116686,7 @@ SQLITE_PRIVATE void sqlite3Update( }else{ reg = 0; for(i=0; inKeyCol; i++){ - i16 iIdxCol = pIdx->aiColumn[i]; - if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){ + if( aXRef[pIdx->aiColumn[i]]>=0 ){ reg = ++pParse->nMem; break; } @@ -124781,20 +116702,29 @@ SQLITE_PRIVATE void sqlite3Update( if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); sqlite3BeginWriteOperation(pParse, 1, iDb); +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Virtual tables must be handled separately */ + if( IsVirtual(pTab) ){ + updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, + pWhere, onError); + pWhere = 0; + pTabList = 0; + goto update_cleanup; + } +#endif + /* Allocate required registers. */ - if( !IsVirtual(pTab) ){ - regRowSet = ++pParse->nMem; - regOldRowid = regNewRowid = ++pParse->nMem; - if( chngPk || pTrigger || hasFK ){ - regOld = pParse->nMem + 1; - pParse->nMem += pTab->nCol; - } - if( chngKey || pTrigger || hasFK ){ - regNewRowid = ++pParse->nMem; - } - regNew = pParse->nMem + 1; + regRowSet = ++pParse->nMem; + regOldRowid = regNewRowid = ++pParse->nMem; + if( chngPk || pTrigger || hasFK ){ + regOld = pParse->nMem + 1; pParse->nMem += pTab->nCol; } + if( chngKey || pTrigger || hasFK ){ + regNewRowid = ++pParse->nMem; + } + regNew = pParse->nMem + 1; + pParse->nMem += pTab->nCol; /* Start the view context. */ if( isView ){ @@ -124817,22 +116747,12 @@ SQLITE_PRIVATE void sqlite3Update( goto update_cleanup; } -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Virtual tables must be handled separately */ - if( IsVirtual(pTab) ){ - updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, - pWhere, onError); - goto update_cleanup; - } -#endif - /* Begin the database scan */ if( HasRowid(pTab) ){ sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); pWInfo = sqlite3WhereBegin( - pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED | WHERE_SEEK_TABLE, iIdxCur + pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, iIdxCur ); if( pWInfo==0 ) goto update_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); @@ -124866,7 +116786,6 @@ SQLITE_PRIVATE void sqlite3Update( if( pWInfo==0 ) goto update_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); for(i=0; iaiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i], iPk+i); } @@ -124876,7 +116795,7 @@ SQLITE_PRIVATE void sqlite3Update( regKey = iPk; }else{ sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, - sqlite3IndexAffinityStr(db, pPk), nPk); + sqlite3IndexAffinityStr(v, pPk), nPk); sqlite3VdbeAddOp2(v, OP_IdxInsert, iEph, regKey); } sqlite3WhereEnd(pWInfo); @@ -124911,7 +116830,7 @@ SQLITE_PRIVATE void sqlite3Update( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; } - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen, + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iBaseCur, aToOpen, 0, 0); } @@ -124989,6 +116908,7 @@ SQLITE_PRIVATE void sqlite3Update( newmask = sqlite3TriggerColmask( pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError ); + /*sqlite3VdbeAddOp3(v, OP_Null, 0, regNew, regNew+pTab->nCol-1);*/ for(i=0; inCol; i++){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); @@ -125004,7 +116924,7 @@ SQLITE_PRIVATE void sqlite3Update( */ testcase( i==31 ); testcase( i==32 ); - sqlite3ExprCodeGetColumnToReg(pParse, pTab, i, iDataCur, regNew+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); } @@ -125046,14 +116966,13 @@ SQLITE_PRIVATE void sqlite3Update( } if( !isView ){ - int addr1 = 0; /* Address of jump instruction */ + int j1 = 0; /* Address of jump instruction */ int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Do constraint checks. */ assert( regOldRowid>0 ); sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, - regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace, - aXRef); + regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace); /* Do FK constraint checks. */ if( hasFK ){ @@ -125063,39 +116982,20 @@ SQLITE_PRIVATE void sqlite3Update( /* Delete the index entries associated with the current record. */ if( bReplace || chngKey ){ if( pPk ){ - addr1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); + j1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); }else{ - addr1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); + j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); } VdbeCoverageNeverTaken(v); } - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1); - - /* If changing the rowid value, or if there are foreign key constraints - ** to process, delete the old record. Otherwise, add a noop OP_Delete - ** to invoke the pre-update hook. - ** - ** That (regNew==regnewRowid+1) is true is also important for the - ** pre-update hook. If the caller invokes preupdate_new(), the returned - ** value is copied from memory cell (regNewRowid+1+iCol), where iCol - ** is the column index supplied by the user. - */ - assert( regNew==regNewRowid+1 ); -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK - sqlite3VdbeAddOp3(v, OP_Delete, iDataCur, - OPFLAG_ISUPDATE | ((hasFK || chngKey || pPk!=0) ? 0 : OPFLAG_ISNOOP), - regNewRowid - ); - if( !pParse->nested ){ - sqlite3VdbeChangeP4(v, -1, (char*)pTab, P4_TABLE); - } -#else + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx); + + /* If changing the record number, delete the old record. */ if( hasFK || chngKey || pPk!=0 ){ sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } -#endif if( bReplace || chngKey ){ - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeJumpHere(v, j1); } if( hasFK ){ @@ -125132,7 +117032,7 @@ SQLITE_PRIVATE void sqlite3Update( sqlite3VdbeResolveLabel(v, labelContinue); sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); }else{ - sqlite3VdbeGoto(v, labelContinue); + sqlite3VdbeAddOp2(v, OP_Goto, 0, labelContinue); } sqlite3VdbeResolveLabel(v, labelBreak); @@ -125186,23 +117086,21 @@ SQLITE_PRIVATE void sqlite3Update( /* ** Generate code for an UPDATE of a virtual table. ** -** There are two possible strategies - the default and the special -** "onepass" strategy. Onepass is only used if the virtual table -** implementation indicates that pWhere may match at most one row. -** -** The default strategy is to create an ephemeral table that contains +** The strategy is that we create an ephemeral table that contains ** for each row to be changed: ** ** (A) The original rowid of that row. -** (B) The revised rowid for the row. +** (B) The revised rowid for the row. (note1) ** (C) The content of every column in the row. ** -** Then loop through the contents of this ephemeral table executing a -** VUpdate for each row. When finished, drop the ephemeral table. +** Then we loop over this ephemeral table and for each row in +** the ephemeral table call VUpdate. ** -** The "onepass" strategy does not use an ephemeral table. Instead, it -** stores the same values (A, B and C above) in a register array and -** makes a single invocation of VUpdate. +** When finished, drop the ephemeral table. +** +** (note1) Actually, if we know in advance that (A) is always the same +** as (B) we only store (A), then duplicate (A) when pulling +** it out of the ephemeral table before calling VUpdate. */ static void updateVirtualTable( Parse *pParse, /* The parsing context */ @@ -125215,96 +117113,68 @@ static void updateVirtualTable( int onError /* ON CONFLICT strategy */ ){ Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ + ExprList *pEList = 0; /* The result set of the SELECT statement */ + Select *pSelect = 0; /* The SELECT statement */ + Expr *pExpr; /* Temporary expression */ int ephemTab; /* Table holding the result of the SELECT */ int i; /* Loop counter */ + int addr; /* Address of top of loop */ + int iReg; /* First register in set passed to OP_VUpdate */ sqlite3 *db = pParse->db; /* Database connection */ const char *pVTab = (const char*)sqlite3GetVTable(db, pTab); - WhereInfo *pWInfo; - int nArg = 2 + pTab->nCol; /* Number of arguments to VUpdate */ - int regArg; /* First register in VUpdate arg array */ - int regRec; /* Register in which to assemble record */ - int regRowid; /* Register for ephem table rowid */ - int iCsr = pSrc->a[0].iCursor; /* Cursor used for virtual table scan */ - int aDummy[2]; /* Unused arg for sqlite3WhereOkOnePass() */ - int bOnePass; /* True to use onepass strategy */ - int addr; /* Address of OP_OpenEphemeral */ - - /* Allocate nArg registers to martial the arguments to VUpdate. Then - ** create and open the ephemeral table in which the records created from - ** these arguments will be temporarily stored. */ - assert( v ); - ephemTab = pParse->nTab++; - addr= sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, nArg); - regArg = pParse->nMem + 1; - pParse->nMem += nArg; - regRec = ++pParse->nMem; - regRowid = ++pParse->nMem; - - /* Start scanning the virtual table */ - pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0,0,WHERE_ONEPASS_DESIRED,0); - if( pWInfo==0 ) return; - - /* Populate the argument registers. */ - sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); + SelectDest dest; + + /* Construct the SELECT statement that will find the new values for + ** all updated rows. + */ + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, "_rowid_")); if( pRowid ){ - sqlite3ExprCode(pParse, pRowid, regArg+1); - }else{ - sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); + pEList = sqlite3ExprListAppend(pParse, pEList, + sqlite3ExprDup(db, pRowid, 0)); } + assert( pTab->iPKey<0 ); for(i=0; inCol; i++){ if( aXRef[i]>=0 ){ - sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); + pExpr = sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0); }else{ - sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); + pExpr = sqlite3Expr(db, TK_ID, pTab->aCol[i].zName); } + pEList = sqlite3ExprListAppend(pParse, pEList, pExpr); } + pSelect = sqlite3SelectNew(pParse, pEList, pSrc, pWhere, 0, 0, 0, 0, 0, 0); + + /* Create the ephemeral table into which the update results will + ** be stored. + */ + assert( v ); + ephemTab = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, pTab->nCol+1+(pRowid!=0)); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); - bOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy); - - if( bOnePass ){ - /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded - ** above. Also, if this is a top-level parse (not a trigger), clear the - ** multi-write flag so that the VM does not open a statement journal */ - sqlite3VdbeChangeToNoop(v, addr); - if( sqlite3IsToplevel(pParse) ){ - pParse->isMultiWrite = 0; - } - }else{ - /* Create a record from the argument register contents and insert it into - ** the ephemeral table. */ - sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec); - sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, ephemTab, regRec, regRowid); - } - - - if( bOnePass==0 ){ - /* End the virtual table scan */ - sqlite3WhereEnd(pWInfo); - - /* Begin scannning through the ephemeral table. */ - addr = sqlite3VdbeAddOp1(v, OP_Rewind, ephemTab); VdbeCoverage(v); + /* fill the ephemeral table + */ + sqlite3SelectDestInit(&dest, SRT_Table, ephemTab); + sqlite3Select(pParse, pSelect, &dest); - /* Extract arguments from the current row of the ephemeral table and - ** invoke the VUpdate method. */ - for(i=0; inMem; + pParse->nMem += pTab->nCol+1; + addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, ephemTab, 0, iReg); + sqlite3VdbeAddOp3(v, OP_Column, ephemTab, (pRowid?1:0), iReg+1); + for(i=0; inCol; i++){ + sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i+1+(pRowid!=0), iReg+2+i); } sqlite3VtabMakeWritable(pParse, pTab); - sqlite3VdbeAddOp4(v, OP_VUpdate, 0, nArg, regArg, pVTab, P4_VTAB); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, pTab->nCol+2, iReg, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); + sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); - /* End of the ephemeral table scan. Or, if using the onepass strategy, - ** jump to here if the scan visited zero rows. */ - if( bOnePass==0 ){ - sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); - }else{ - sqlite3WhereEnd(pWInfo); - } + /* Cleanup */ + sqlite3SelectDelete(db, pSelect); } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -125326,56 +117196,59 @@ static void updateVirtualTable( ** Most of the code in this file may be omitted by defining the ** SQLITE_OMIT_VACUUM macro. */ -/* #include "sqliteInt.h" */ -/* #include "vdbeInt.h" */ #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) +/* +** Finalize a prepared statement. If there was an error, store the +** text of the error message in *pzErrMsg. Return the result code. +*/ +static int vacuumFinalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ + int rc; + rc = sqlite3VdbeFinalize((Vdbe*)pStmt); + if( rc ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + } + return rc; +} /* -** Execute zSql on database db. -** -** If zSql returns rows, then each row will have exactly one -** column. (This will only happen if zSql begins with "SELECT".) -** Take each row of result and call execSql() again recursively. -** -** The execSqlF() routine does the same thing, except it accepts -** a format string as its third argument +** Execute zSql on database db. Return an error code. */ static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ sqlite3_stmt *pStmt; - int rc; - - /* printf("SQL: [%s]\n", zSql); fflush(stdout); */ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ - const char *zSubSql = (const char*)sqlite3_column_text(pStmt,0); - assert( sqlite3_strnicmp(zSql,"SELECT",6)==0 ); - if( zSubSql ){ - assert( zSubSql[0]!='S' ); - rc = execSql(db, pzErrMsg, zSubSql); - if( rc!=SQLITE_OK ) break; - } + VVA_ONLY( int rc; ) + if( !zSql ){ + return SQLITE_NOMEM; } - assert( rc!=SQLITE_ROW ); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; - if( rc ){ + if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + return sqlite3_errcode(db); } - (void)sqlite3_finalize(pStmt); - return rc; + VVA_ONLY( rc = ) sqlite3_step(pStmt); + assert( rc!=SQLITE_ROW || (db->flags&SQLITE_CountRows) ); + return vacuumFinalize(db, pStmt, pzErrMsg); } -static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){ - char *z; - va_list ap; + +/* +** Execute zSql on database db. The statement returns exactly +** one column. Execute this as SQL on the same database. +*/ +static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; int rc; - va_start(ap, zSql); - z = sqlite3VMPrintf(db, zSql, ap); - va_end(ap); - if( z==0 ) return SQLITE_NOMEM; - rc = execSql(db, pzErrMsg, z); - sqlite3DbFree(db, z); - return rc; + + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + rc = execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); + if( rc!=SQLITE_OK ){ + vacuumFinalize(db, pStmt, pzErrMsg); + return rc; + } + } + + return vacuumFinalize(db, pStmt, pzErrMsg); } /* @@ -125408,12 +117281,11 @@ static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){ ** transient would cause the database file to appear to be deleted ** following reboot. */ -SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm){ +SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse){ Vdbe *v = sqlite3GetVdbe(pParse); - int iDb = pNm ? sqlite3TwoPartName(pParse, pNm, pNm, &pNm) : 0; - if( v && (iDb>=2 || iDb==0) ){ - sqlite3VdbeAddOp1(v, OP_Vacuum, iDb); - sqlite3VdbeUsesBtree(v, iDb); + if( v ){ + sqlite3VdbeAddOp2(v, OP_Vacuum, 0, 0); + sqlite3VdbeUsesBtree(v, 0); } return; } @@ -125421,19 +117293,19 @@ SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm){ /* ** This routine implements the OP_Vacuum opcode of the VDBE. */ -SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ +SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ int rc = SQLITE_OK; /* Return code from service routines */ Btree *pMain; /* The database being vacuumed */ Btree *pTemp; /* The temporary database we vacuum into */ + char *zSql = 0; /* SQL statements */ int saved_flags; /* Saved value of the db->flags */ int saved_nChange; /* Saved value of db->nChange */ int saved_nTotalChange; /* Saved value of db->nTotalChange */ - u8 saved_mTrace; /* Saved trace settings */ + void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ Db *pDb = 0; /* Database to detach at end of vacuum */ int isMemDb; /* True if vacuuming a :memory: database */ int nRes; /* Bytes of reserved space at the end of each page */ int nDb; /* Number of attached databases */ - const char *zDbMain; /* Schema name of database to vacuum */ if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); @@ -125450,14 +117322,12 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ saved_flags = db->flags; saved_nChange = db->nChange; saved_nTotalChange = db->nTotalChange; - saved_mTrace = db->mTrace; - db->flags |= (SQLITE_WriteSchema | SQLITE_IgnoreChecks - | SQLITE_PreferBuiltin | SQLITE_Vacuum); - db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_CountRows); - db->mTrace = 0; - - zDbMain = db->aDb[iDb].zDbSName; - pMain = db->aDb[iDb].pBt; + saved_xTrace = db->xTrace; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; + db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); + db->xTrace = 0; + + pMain = db->aDb[0].pBt; isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain)); /* Attach the temporary database as 'vacuum_db'. The synchronous pragma @@ -125475,12 +117345,18 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ ** to write the journal header file. */ nDb = db->nDb; - rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db"); + if( sqlite3TempInMemory(db) ){ + zSql = "ATTACH ':memory:' AS vacuum_db;"; + }else{ + zSql = "ATTACH '' AS vacuum_db;"; + } + rc = execSql(db, pzErrMsg, zSql); + if( db->nDb>nDb ){ + pDb = &db->aDb[db->nDb-1]; + assert( strcmp(pDb->zName,"vacuum_db")==0 ); + } if( rc!=SQLITE_OK ) goto end_of_vacuum; - assert( (db->nDb-1)==nDb ); - pDb = &db->aDb[nDb]; - assert( strcmp(pDb->zDbSName,"vacuum_db")==0 ); - pTemp = pDb->pBt; + pTemp = db->aDb[db->nDb-1].pBt; /* The call to execSql() to attach the temp database has left the file ** locked (as there was more than one active statement when the transaction @@ -125501,15 +117377,14 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ } #endif - sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); - sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); - sqlite3BtreeSetPagerFlags(pTemp, PAGER_SYNCHRONOUS_OFF|PAGER_CACHESPILL); + rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Begin a transaction and take an exclusive lock on the main database ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, ** to ensure that we do not try to change the page-size on a WAL database. */ - rc = execSql(db, pzErrMsg, "BEGIN"); + rc = execSql(db, pzErrMsg, "BEGIN;"); if( rc!=SQLITE_OK ) goto end_of_vacuum; rc = sqlite3BtreeBeginTrans(pMain, 2); if( rc!=SQLITE_OK ) goto end_of_vacuum; @@ -125524,7 +117399,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ || (!isMemDb && sqlite3BtreeSetPageSize(pTemp, db->nextPagesize, nRes, 0)) || NEVER(db->mallocFailed) ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; goto end_of_vacuum; } @@ -125536,48 +117411,64 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ /* Query the schema of the main database. Create a mirror schema ** in the temporary database. */ - db->init.iDb = nDb; /* force new CREATE statements into vacuum_db */ - rc = execSqlF(db, pzErrMsg, - "SELECT sql FROM \"%w\".sqlite_master" - " WHERE type='table'AND name<>'sqlite_sequence'" - " AND coalesce(rootpage,1)>0", - zDbMain + rc = execExecSql(db, pzErrMsg, + "SELECT 'CREATE TABLE vacuum_db.' || substr(sql,14) " + " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" + " AND coalesce(rootpage,1)>0" ); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execSqlF(db, pzErrMsg, - "SELECT sql FROM \"%w\".sqlite_master" - " WHERE type='index' AND length(sql)>10", - zDbMain - ); + rc = execExecSql(db, pzErrMsg, + "SELECT 'CREATE INDEX vacuum_db.' || substr(sql,14)" + " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %' "); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = execExecSql(db, pzErrMsg, + "SELECT 'CREATE UNIQUE INDEX vacuum_db.' || substr(sql,21) " + " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %'"); if( rc!=SQLITE_OK ) goto end_of_vacuum; - db->init.iDb = 0; /* Loop through the tables in the main database. For each, do ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy ** the contents to the temporary database. */ - rc = execSqlF(db, pzErrMsg, - "SELECT'INSERT INTO vacuum_db.'||quote(name)" - "||' SELECT*FROM\"%w\".'||quote(name)" - "FROM vacuum_db.sqlite_master " - "WHERE type='table'AND coalesce(rootpage,1)>0", - zDbMain + assert( (db->flags & SQLITE_Vacuum)==0 ); + db->flags |= SQLITE_Vacuum; + rc = execExecSql(db, pzErrMsg, + "SELECT 'INSERT INTO vacuum_db.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';'" + "FROM main.sqlite_master " + "WHERE type = 'table' AND name!='sqlite_sequence' " + " AND coalesce(rootpage,1)>0" ); assert( (db->flags & SQLITE_Vacuum)!=0 ); db->flags &= ~SQLITE_Vacuum; if( rc!=SQLITE_OK ) goto end_of_vacuum; + /* Copy over the sequence table + */ + rc = execExecSql(db, pzErrMsg, + "SELECT 'DELETE FROM vacuum_db.' || quote(name) || ';' " + "FROM vacuum_db.sqlite_master WHERE name='sqlite_sequence' " + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = execExecSql(db, pzErrMsg, + "SELECT 'INSERT INTO vacuum_db.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';' " + "FROM vacuum_db.sqlite_master WHERE name=='sqlite_sequence';" + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Copy the triggers, views, and virtual tables from the main database ** over to the temporary database. None of these objects has any ** associated storage, so all we have to do is copy their entries ** from the SQLITE_MASTER table. */ - rc = execSqlF(db, pzErrMsg, - "INSERT INTO vacuum_db.sqlite_master" - " SELECT*FROM \"%w\".sqlite_master" - " WHERE type IN('view','trigger')" - " OR(type='table'AND rootpage=0)", - zDbMain + rc = execSql(db, pzErrMsg, + "INSERT INTO vacuum_db.sqlite_master " + " SELECT type, name, tbl_name, rootpage, sql" + " FROM main.sqlite_master" + " WHERE type='view' OR type='trigger'" + " OR (type='table' AND rootpage=0)" ); if( rc ) goto end_of_vacuum; @@ -125631,11 +117522,10 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ end_of_vacuum: /* Restore the original value of db->flags */ - db->init.iDb = 0; db->flags = saved_flags; db->nChange = saved_nChange; db->nTotalChange = saved_nTotalChange; - db->mTrace = saved_mTrace; + db->xTrace = saved_xTrace; sqlite3BtreeSetPageSize(pMain, -1, -1, 1); /* Currently there is an SQL level transaction open on the vacuum @@ -125678,7 +117568,6 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ ** This file contains code used to help implement virtual tables. */ #ifndef SQLITE_OMIT_VIRTUALTABLE -/* #include "sqliteInt.h" */ /* ** Before a virtual table xCreate() or xConnect() method is invoked, the @@ -125715,7 +117604,7 @@ static int createModule( rc = SQLITE_MISUSE_BKPT; }else{ Module *pMod; - pMod = (Module *)sqlite3DbMallocRawNN(db, sizeof(Module) + nName + 1); + pMod = (Module *)sqlite3DbMallocRaw(db, sizeof(Module) + nName + 1); if( pMod ){ Module *pDel; char *zCopy = (char *)(&pMod[1]); @@ -125724,11 +117613,10 @@ static int createModule( pMod->pModule = pModule; pMod->pAux = pAux; pMod->xDestroy = xDestroy; - pMod->pEpoTab = 0; pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); assert( pDel==0 || pDel==pMod ); if( pDel ){ - sqlite3OomFault(db); + db->mallocFailed = 1; sqlite3DbFree(db, pDel); } } @@ -125744,7 +117632,7 @@ static int createModule( /* ** External API function used to create a new virtual-table module. */ -SQLITE_API int sqlite3_create_module( +SQLITE_API int SQLITE_STDCALL sqlite3_create_module( sqlite3 *db, /* Database in which module is registered */ const char *zName, /* Name assigned to this module */ const sqlite3_module *pModule, /* The definition of the module */ @@ -125759,7 +117647,7 @@ SQLITE_API int sqlite3_create_module( /* ** External API function used to create a new virtual-table module. */ -SQLITE_API int sqlite3_create_module_v2( +SQLITE_API int SQLITE_STDCALL sqlite3_create_module_v2( sqlite3 *db, /* Database in which module is registered */ const char *zName, /* Name assigned to this module */ const sqlite3_module *pModule, /* The definition of the module */ @@ -125952,17 +117840,23 @@ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ ** deleted. */ static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ - int nBytes = sizeof(char *)*(2+pTable->nModuleArg); + int i = pTable->nModuleArg++; + int nBytes = sizeof(char *)*(1+pTable->nModuleArg); char **azModuleArg; azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); if( azModuleArg==0 ){ + int j; + for(j=0; jazModuleArg[j]); + } sqlite3DbFree(db, zArg); + sqlite3DbFree(db, pTable->azModuleArg); + pTable->nModuleArg = 0; }else{ - int i = pTable->nModuleArg++; azModuleArg[i] = zArg; azModuleArg[i+1] = 0; - pTable->azModuleArg = azModuleArg; } + pTable->azModuleArg = azModuleArg; } /* @@ -126010,7 +117904,7 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( */ if( pTable->azModuleArg ){ sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, - pTable->azModuleArg[0], pParse->db->aDb[iDb].zDbSName); + pTable->azModuleArg[0], pParse->db->aDb[iDb].zName); } #endif } @@ -126074,7 +117968,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ "UPDATE %Q.%s " "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q " "WHERE rowid=#%d", - db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pTab->zName, pTab->zName, zStmt, @@ -126084,12 +117978,12 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ v = sqlite3GetVdbe(pParse); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp0(v, OP_Expire); + sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); iReg = ++pParse->nMem; - sqlite3VdbeLoadString(v, iReg, pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, iReg, 0, pTab->zName, 0); sqlite3VdbeAddOp2(v, OP_VCreate, iDb, iReg); } @@ -126105,7 +117999,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ assert( sqlite3SchemaMutexHeld(db, 0, pSchema) ); pOld = sqlite3HashInsert(&pSchema->tblHash, zName, pTab); if( pOld ){ - sqlite3OomFault(db); + db->mallocFailed = 1; assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ return; } @@ -126172,19 +118066,19 @@ static int vtabCallConstructor( zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); if( !zModuleName ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } pVTable = sqlite3DbMallocZero(db, sizeof(VTable)); if( !pVTable ){ sqlite3DbFree(db, zModuleName); - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } pVTable->db = db; pVTable->pMod = pMod; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - pTab->azModuleArg[1] = db->aDb[iDb].zDbSName; + pTab->azModuleArg[1] = db->aDb[iDb].zName; /* Invoke the virtual table constructor */ assert( &db->pVtabCtx ); @@ -126196,7 +118090,7 @@ static int vtabCallConstructor( db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); db->pVtabCtx = sCtx.pPrior; - if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; assert( sCtx.pTab==pTab ); if( SQLITE_OK!=rc ){ @@ -126230,16 +118124,22 @@ static int vtabCallConstructor( pTab->pVTable = pVTable; for(iCol=0; iColnCol; iCol++){ - char *zType = sqlite3ColumnType(&pTab->aCol[iCol], ""); + char *zType = pTab->aCol[iCol].zType; int nType; int i = 0; + if( !zType ){ + pTab->tabFlags |= oooHidden; + continue; + } nType = sqlite3Strlen30(zType); - for(i=0; inVTrans + ARRAY_INCR); aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes); if( !aVTrans ){ - return SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM; } memset(&aVTrans[db->nVTrans], 0, sizeof(sqlite3_vtab *)*ARRAY_INCR); db->aVTrans = aVTrans; @@ -126338,7 +118238,7 @@ static void addToVTrans(sqlite3 *db, VTable *pVTab){ ** This function is invoked by the vdbe to call the xCreate method ** of the virtual table named zTab in database iDb. ** -** If an error occurs, *pzErr is set to point to an English language +** If an error occurs, *pzErr is set to point an an English language ** description of the error and an SQLITE_XXX error code is returned. ** In this case the caller must call sqlite3DbFree(db, ) on *pzErr. */ @@ -126348,7 +118248,7 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, Module *pMod; const char *zMod; - pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName); + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); assert( pTab && (pTab->tabFlags & TF_Virtual)!=0 && !pTab->pVTable ); /* Locate the required virtual table module */ @@ -126359,7 +118259,7 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, ** invoke it now. If the module has not been registered, return an ** error. Otherwise, do nothing. */ - if( pMod==0 || pMod->pModule->xCreate==0 || pMod->pModule->xDestroy==0 ){ + if( !pMod ){ *pzErr = sqlite3MPrintf(db, "no such module: %s", zMod); rc = SQLITE_ERROR; }else{ @@ -126383,7 +118283,7 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, ** valid to call this function from within the xCreate() or xConnect() of a ** virtual table module. */ -SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ +SQLITE_API int SQLITE_STDCALL sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ VtabCtx *pCtx; Parse *pParse; int rc = SQLITE_OK; @@ -126407,7 +118307,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); if( pParse==0 ){ - rc = SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM; }else{ pParse->declareVtab = 1; pParse->db = db; @@ -126420,24 +118320,10 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ && (pParse->pNewTable->tabFlags & TF_Virtual)==0 ){ if( !pTab->aCol ){ - Table *pNew = pParse->pNewTable; - Index *pIdx; - pTab->aCol = pNew->aCol; - pTab->nCol = pNew->nCol; - pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid); - pNew->nCol = 0; - pNew->aCol = 0; - assert( pTab->pIndex==0 ); - if( !HasRowid(pNew) && pCtx->pVTable->pMod->pModule->xUpdate!=0 ){ - rc = SQLITE_ERROR; - } - pIdx = pNew->pIndex; - if( pIdx ){ - assert( pIdx->pNext==0 ); - pTab->pIndex = pIdx; - pNew->pIndex = 0; - pIdx->pTable = pTab; - } + pTab->aCol = pParse->pNewTable->aCol; + pTab->nCol = pParse->pNewTable->nCol; + pParse->pNewTable->nCol = 0; + pParse->pNewTable->aCol = 0; } pCtx->bDeclared = 1; }else{ @@ -126472,10 +118358,9 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab int rc = SQLITE_OK; Table *pTab; - pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName); - if( pTab!=0 && ALWAYS(pTab->pVTable!=0) ){ + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); + if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){ VTable *p; - int (*xDestroy)(sqlite3_vtab *); for(p=pTab->pVTable; p; p=p->pNext){ assert( p->pVtab ); if( p->pVtab->nRef>0 ){ @@ -126483,9 +118368,7 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab } } p = vtabDisconnectAll(db, pTab); - xDestroy = p->pMod->pModule->xDestroy; - assert( xDestroy!=0 ); /* Checked before the virtual table is created */ - rc = xDestroy(p->pVtab); + rc = p->pMod->pModule->xDestroy(p->pVtab); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ assert( pTab->pVTable==p && p->pNext==0 ); @@ -126509,10 +118392,8 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab static void callFinaliser(sqlite3 *db, int offset){ int i; if( db->aVTrans ){ - VTable **aVTrans = db->aVTrans; - db->aVTrans = 0; for(i=0; inVTrans; i++){ - VTable *pVTab = aVTrans[i]; + VTable *pVTab = db->aVTrans[i]; sqlite3_vtab *p = pVTab->pVtab; if( p ){ int (*x)(sqlite3_vtab *); @@ -126522,8 +118403,9 @@ static void callFinaliser(sqlite3 *db, int offset){ pVTab->iSavepoint = 0; sqlite3VtabUnlock(pVTab); } - sqlite3DbFree(db, aVTrans); + sqlite3DbFree(db, db->aVTrans); db->nVTrans = 0; + db->aVTrans = 0; } } @@ -126611,12 +118493,7 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ if( rc==SQLITE_OK ){ rc = pModule->xBegin(pVTab->pVtab); if( rc==SQLITE_OK ){ - int iSvpt = db->nStatement + db->nSavepoint; addToVTrans(db, pVTab); - if( iSvpt && pModule->xSavepoint ){ - pVTab->iSavepoint = iSvpt; - rc = pModule->xSavepoint(pVTab->pVtab, iSvpt-1); - } } } } @@ -126693,7 +118570,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( Table *pTab; sqlite3_vtab *pVtab; sqlite3_module *pMod; - void (*xSFunc)(sqlite3_context*,int,sqlite3_value**) = 0; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**) = 0; void *pArg = 0; FuncDef *pNew; int rc = 0; @@ -126721,7 +118598,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( for(z=(unsigned char*)zLowerName; *z; z++){ *z = sqlite3UpperToLower[*z]; } - rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xSFunc, &pArg); + rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xFunc, &pArg); sqlite3DbFree(db, zLowerName); } if( rc==0 ){ @@ -126736,9 +118613,9 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( return pDef; } *pNew = *pDef; - pNew->zName = (const char*)&pNew[1]; - memcpy((char*)&pNew[1], pDef->zName, sqlite3Strlen30(pDef->zName)+1); - pNew->xSFunc = xSFunc; + pNew->zName = (char *)&pNew[1]; + memcpy(pNew->zName, pDef->zName, sqlite3Strlen30(pDef->zName)+1); + pNew->xFunc = xFunc; pNew->pUserData = pArg; pNew->funcFlags |= SQLITE_FUNC_EPHEM; return pNew; @@ -126765,71 +118642,7 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ pToplevel->apVtabLock = apVtabLock; pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; }else{ - sqlite3OomFault(pToplevel->db); - } -} - -/* -** Check to see if virtual table module pMod can be have an eponymous -** virtual table instance. If it can, create one if one does not already -** exist. Return non-zero if the eponymous virtual table instance exists -** when this routine returns, and return zero if it does not exist. -** -** An eponymous virtual table instance is one that is named after its -** module, and more importantly, does not require a CREATE VIRTUAL TABLE -** statement in order to come into existance. Eponymous virtual table -** instances always exist. They cannot be DROP-ed. -** -** Any virtual table module for which xConnect and xCreate are the same -** method can have an eponymous virtual table instance. -*/ -SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ - const sqlite3_module *pModule = pMod->pModule; - Table *pTab; - char *zErr = 0; - int rc; - sqlite3 *db = pParse->db; - if( pMod->pEpoTab ) return 1; - if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0; - pTab = sqlite3DbMallocZero(db, sizeof(Table)); - if( pTab==0 ) return 0; - pTab->zName = sqlite3DbStrDup(db, pMod->zName); - if( pTab->zName==0 ){ - sqlite3DbFree(db, pTab); - return 0; - } - pMod->pEpoTab = pTab; - pTab->nRef = 1; - pTab->pSchema = db->aDb[0].pSchema; - pTab->tabFlags |= TF_Virtual; - pTab->nModuleArg = 0; - pTab->iPKey = -1; - addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); - addModuleArgument(db, pTab, 0); - addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); - rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); - if( rc ){ - sqlite3ErrorMsg(pParse, "%s", zErr); - sqlite3DbFree(db, zErr); - sqlite3VtabEponymousTableClear(db, pMod); - return 0; - } - return 1; -} - -/* -** Erase the eponymous virtual table instance associated with -** virtual table module pMod, if it exists. -*/ -SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ - Table *pTab = pMod->pEpoTab; - if( pTab!=0 ){ - /* Mark the table as Ephemeral prior to deleting it, so that the - ** sqlite3DeleteTable() routine will know that it is not stored in - ** the schema. */ - pTab->tabFlags |= TF_Ephemeral; - sqlite3DeleteTable(db, pTab); - pMod->pEpoTab = 0; + pToplevel->db->mallocFailed = 1; } } @@ -126840,7 +118653,7 @@ SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ ** The results of this routine are undefined unless it is called from ** within an xUpdate method. */ -SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *db){ +SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *db){ static const unsigned char aMap[] = { SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE }; @@ -126858,7 +118671,7 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *db){ ** the SQLite core with additional information about the behavior ** of the virtual table being implemented. */ -SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ +SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3 *db, int op, ...){ va_list ap; int rc = SQLITE_OK; @@ -126892,9 +118705,9 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ #endif /* SQLITE_OMIT_VIRTUALTABLE */ /************** End of vtab.c ************************************************/ -/************** Begin file wherecode.c ***************************************/ +/************** Begin file where.c *******************************************/ /* -** 2015-06-06 +** 2001 September 15 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -126905,15 +118718,13 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** ************************************************************************* ** This module contains C code that generates VDBE code used to process -** the WHERE clause of SQL statements. -** -** This file was split off from where.c on 2015-06-06 in order to reduce the -** size of where.c and make it easier to edit. This file contains the routines -** that actually generate the bulk of the WHERE loop code. The original where.c -** file retains the code that does query planning and analysis. +** the WHERE clause of SQL statements. This module is responsible for +** generating the code that loops through a table looking for applicable +** rows. Indices are selected and used to speed the search when doing +** so is applicable. Because this module is responsible for selecting +** indices, you might also think of this module as the "query optimizer". */ -/* #include "sqliteInt.h" */ -/************** Include whereInt.h in the middle of wherecode.c **************/ +/************** Include whereInt.h in the middle of where.c ******************/ /************** Begin file whereInt.h ****************************************/ /* ** 2013-11-12 @@ -126936,7 +118747,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** Trace output macros */ #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -/***/ int sqlite3WhereTrace; +/***/ int sqlite3WhereTrace = 0; #endif #if defined(SQLITE_DEBUG) \ && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) @@ -126986,10 +118797,8 @@ struct WhereLevel { int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ -#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS - u32 iLikeRepCntr; /* LIKE range processing counter register (times 2) */ + int iLikeRepCntr; /* LIKE range processing counter register */ int addrLikeRep; /* LIKE range processing address */ -#endif u8 iFrom; /* Which entry in the FROM clause */ u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ int p1, p2; /* Operands of the opcode used to ends the loop */ @@ -127039,8 +118848,6 @@ struct WhereLoop { union { struct { /* Information for internal btree tables */ u16 nEq; /* Number of equality constraints */ - u16 nBtm; /* Size of BTM vector */ - u16 nTop; /* Size of TOP vector */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ @@ -127082,6 +118889,10 @@ struct WhereOrSet { WhereOrCost a[N_OR_COST]; /* Set of best costs */ }; + +/* Forward declaration of methods */ +static int whereLoopResize(sqlite3*, WhereLoop*, int); + /* ** Each instance of this object holds a sequence of WhereLoop objects ** that implement some or all of a query plan. @@ -127163,20 +118974,18 @@ struct WherePath { */ struct WhereTerm { Expr *pExpr; /* Pointer to the subexpression that is this term */ - WhereClause *pWC; /* The clause this term is part of */ - LogEst truthProb; /* Probability of truth for this expression */ - u16 wtFlags; /* TERM_xxx bit flags. See below */ - u16 eOperator; /* A WO_xx value describing */ - u8 nChild; /* Number of children that must disable us */ - u8 eMatchOp; /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */ int iParent; /* Disable pWC->a[iParent] when this term disabled */ int leftCursor; /* Cursor number of X in "X " */ - int iField; /* Field in (?,?,?) IN (SELECT...) vector */ union { int leftColumn; /* Column number of X in "X " */ WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ } u; + LogEst truthProb; /* Probability of truth for this expression */ + u16 eOperator; /* A WO_xx value describing */ + u16 wtFlags; /* TERM_xxx bit flags. See below */ + u8 nChild; /* Number of children that must disable us */ + WhereClause *pWC; /* The clause this term is part of */ Bitmask prereqRight; /* Bitmask of tables used by pExpr->pRight */ Bitmask prereqAll; /* Bitmask of tables referenced by pExpr */ }; @@ -127199,7 +119008,6 @@ struct WhereTerm { #define TERM_LIKEOPT 0x100 /* Virtual terms from the LIKE optimization */ #define TERM_LIKECOND 0x200 /* Conditionally this LIKE operator term */ #define TERM_LIKE 0x400 /* The original LIKE operator */ -#define TERM_IS 0x800 /* Term.pExpr is an IS operator */ /* ** An instance of the WhereScan object is used as an iterator for locating @@ -127208,15 +119016,13 @@ struct WhereTerm { struct WhereScan { WhereClause *pOrigWC; /* Original, innermost WhereClause */ WhereClause *pWC; /* WhereClause currently being scanned */ - const char *zCollName; /* Required collating sequence, if not NULL */ - Expr *pIdxExpr; /* Search for this index expression */ + char *zCollName; /* Required collating sequence, if not NULL */ char idxaff; /* Must match this affinity, if zCollName!=NULL */ unsigned char nEquiv; /* Number of entries in aEquiv[] */ unsigned char iEquiv; /* Next unused slot in aEquiv[] */ u32 opMask; /* Acceptable operators */ int k; /* Resume scanning at this->pWC->a[this->k] */ - int aiCur[11]; /* Cursors in the equivalence class */ - i16 aiColumn[11]; /* Corresponding column number in the eq-class */ + int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ }; /* @@ -127293,11 +119099,6 @@ struct WhereMaskSet { int ix[BMS]; /* Cursor assigned to each bit */ }; -/* -** Initialize a WhereMaskSet object -*/ -#define initMaskSet(P) (P)->n=0 - /* ** This object is a convenience wrapper holding all information needed ** to construct WhereLoop objects for a particular query. @@ -127328,119 +119129,48 @@ struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ - ExprList *pDistinctSet; /* DISTINCT over all these values */ - LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ - int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ - int iContinue; /* Jump here to continue with next record */ - int iBreak; /* Jump here to break out of the loop */ - int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + ExprList *pResultSet; /* Result set. DISTINCT operates on these */ + WhereLoop *pLoops; /* List of all WhereLoop objects */ + Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ + LogEst nRowOut; /* Estimated number of output rows */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ - u8 nLevel; /* Number of nested loop */ i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ u8 sorted; /* True if really sorted (not just grouped) */ - u8 eOnePass; /* ONEPASS_OFF, or _SINGLE, or _MULTI */ + u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ - u8 eDistinct; /* One of the WHERE_DISTINCT_* values */ - u8 bOrderedInnerLoop; /* True if only the inner-most loop is ordered */ + u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ + u8 nLevel; /* Number of nested loop */ int iTop; /* The very beginning of the WHERE loop */ - WhereLoop *pLoops; /* List of all WhereLoop objects */ - Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ - LogEst nRowOut; /* Estimated number of output rows */ - WhereClause sWC; /* Decomposition of the WHERE clause */ + int iContinue; /* Jump here to continue with next record */ + int iBreak; /* Jump here to break out of the loop */ + int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ + WhereClause sWC; /* Decomposition of the WHERE clause */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; -/* -** Private interfaces - callable only by other where.c routines. -** -** where.c: -*/ -SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet*,int); -#ifdef WHERETRACE_ENABLED -SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC); -#endif -SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( - WhereClause *pWC, /* The WHERE clause to be searched */ - int iCur, /* Cursor number of LHS */ - int iColumn, /* Column number of LHS */ - Bitmask notReady, /* RHS must not overlap with this mask */ - u32 op, /* Mask of WO_xx values describing operator */ - Index *pIdx /* Must be compatible with this index, if not NULL */ -); - -/* wherecode.c: */ -#ifndef SQLITE_OMIT_EXPLAIN -SQLITE_PRIVATE int sqlite3WhereExplainOneScan( - Parse *pParse, /* Parse context */ - SrcList *pTabList, /* Table list this loop refers to */ - WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ - int iLevel, /* Value for "level" column of output */ - int iFrom, /* Value for "from" column of output */ - u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ -); -#else -# define sqlite3WhereExplainOneScan(u,v,w,x,y,z) 0 -#endif /* SQLITE_OMIT_EXPLAIN */ -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -SQLITE_PRIVATE void sqlite3WhereAddScanStatus( - Vdbe *v, /* Vdbe to add scanstatus entry to */ - SrcList *pSrclist, /* FROM clause pLvl reads data from */ - WhereLevel *pLvl, /* Level to add scanstatus() entry for */ - int addrExplain /* Address of OP_Explain (or 0) */ -); -#else -# define sqlite3WhereAddScanStatus(a, b, c, d) ((void)d) -#endif -SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( - WhereInfo *pWInfo, /* Complete information about the WHERE clause */ - int iLevel, /* Which level of pWInfo->a[] should be coded */ - Bitmask notReady /* Which tables are currently available */ -); - -/* whereexpr.c: */ -SQLITE_PRIVATE void sqlite3WhereClauseInit(WhereClause*,WhereInfo*); -SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause*); -SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause*,Expr*,u8); -SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); -SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); -SQLITE_PRIVATE void sqlite3WhereExprAnalyze(SrcList*, WhereClause*); -SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); - - - - - /* ** Bitmasks for the operators on WhereTerm objects. These are all ** operators that are of interest to the query planner. An ** OR-ed combination of these values can be used when searching for ** particular WhereTerms within a WhereClause. -** -** Value constraints: -** WO_EQ == SQLITE_INDEX_CONSTRAINT_EQ -** WO_LT == SQLITE_INDEX_CONSTRAINT_LT -** WO_LE == SQLITE_INDEX_CONSTRAINT_LE -** WO_GT == SQLITE_INDEX_CONSTRAINT_GT -** WO_GE == SQLITE_INDEX_CONSTRAINT_GE -** WO_MATCH == SQLITE_INDEX_CONSTRAINT_MATCH */ -#define WO_IN 0x0001 -#define WO_EQ 0x0002 +#define WO_IN 0x001 +#define WO_EQ 0x002 #define WO_LT (WO_EQ<<(TK_LT-TK_EQ)) #define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) #define WO_GT (WO_EQ<<(TK_GT-TK_EQ)) #define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) -#define WO_MATCH 0x0040 -#define WO_IS 0x0080 -#define WO_ISNULL 0x0100 -#define WO_OR 0x0200 /* Two or more OR-connected terms */ -#define WO_AND 0x0400 /* Two or more AND-connected terms */ -#define WO_EQUIV 0x0800 /* Of the form A==B, both columns */ -#define WO_NOOP 0x1000 /* This term does not restrict search space */ +#define WO_MATCH 0x040 +#define WO_ISNULL 0x080 +#define WO_OR 0x100 /* Two or more OR-connected terms */ +#define WO_AND 0x200 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ +#define WO_NOOP 0x800 /* This term does not restrict search space */ -#define WO_ALL 0x1fff /* Mask of all possible WO_* values */ -#define WO_SINGLE 0x01ff /* Mask of all non-compound WO_* values */ +#define WO_ALL 0xfff /* Mask of all possible WO_* values */ +#define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ /* ** These are definitions of bits in the WhereLoop.wsFlags field. @@ -127468,36220 +119198,11739 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereC #define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ /************** End of whereInt.h ********************************************/ -/************** Continuing where we left off in wherecode.c ******************/ - -#ifndef SQLITE_OMIT_EXPLAIN +/************** Continuing where we left off in where.c **********************/ /* -** Return the name of the i-th column of the pIdx index. +** Return the estimated number of output rows from a WHERE clause */ -static const char *explainIndexColumnName(Index *pIdx, int i){ - i = pIdx->aiColumn[i]; - if( i==XN_EXPR ) return ""; - if( i==XN_ROWID ) return "rowid"; - return pIdx->pTable->aCol[i].zName; +SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ + return sqlite3LogEstToInt(pWInfo->nRowOut); } /* -** This routine is a helper for explainIndexRange() below -** -** pStr holds the text of an expression that we are building up one term -** at a time. This routine adds a new term to the end of the expression. -** Terms are separated by AND so add the "AND" text for second and subsequent -** terms only. +** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this +** WHERE clause returns outputs for DISTINCT processing. */ -static void explainAppendTerm( - StrAccum *pStr, /* The text expression being built */ - Index *pIdx, /* Index to read column names from */ - int nTerm, /* Number of terms */ - int iTerm, /* Zero-based index of first term. */ - int bAnd, /* Non-zero to append " AND " */ - const char *zOp /* Name of the operator */ -){ - int i; - - assert( nTerm>=1 ); - if( bAnd ) sqlite3StrAccumAppend(pStr, " AND ", 5); +SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ + return pWInfo->eDistinct; +} - if( nTerm>1 ) sqlite3StrAccumAppend(pStr, "(", 1); - for(i=0; i1 ) sqlite3StrAccumAppend(pStr, ")", 1); +/* +** Return TRUE if the WHERE clause returns rows in ORDER BY order. +** Return FALSE if the output needs to be sorted. +*/ +SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ + return pWInfo->nOBSat; +} - sqlite3StrAccumAppend(pStr, zOp, 1); +/* +** Return the VDBE address or label to jump to in order to continue +** immediately with the next row of a WHERE clause. +*/ +SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ + assert( pWInfo->iContinue!=0 ); + return pWInfo->iContinue; +} - if( nTerm>1 ) sqlite3StrAccumAppend(pStr, "(", 1); - for(i=0; i1 ) sqlite3StrAccumAppend(pStr, ")", 1); +/* +** Return the VDBE address or label to jump to in order to break +** out of a WHERE loop. +*/ +SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ + return pWInfo->iBreak; } /* -** Argument pLevel describes a strategy for scanning table pTab. This -** function appends text to pStr that describes the subset of table -** rows scanned by the strategy in the form of an SQL expression. -** -** For example, if the query: -** -** SELECT * FROM t1 WHERE a=1 AND b>2; +** Return TRUE if an UPDATE or DELETE statement can operate directly on +** the rowids returned by a WHERE clause. Return FALSE if doing an +** UPDATE or DELETE might change subsequent WHERE clause results. ** -** is run and there is an index on (a, b), then this function returns a -** string similar to: +** If the ONEPASS optimization is used (if this routine returns true) +** then also write the indices of open cursors used by ONEPASS +** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data +** table and iaCur[1] gets the cursor used by an auxiliary index. +** Either value may be -1, indicating that cursor is not used. +** Any cursors returned will have been opened for writing. ** -** "a=? AND b>?" +** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is +** unable to use the ONEPASS optimization. */ -static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ - Index *pIndex = pLoop->u.btree.pIndex; - u16 nEq = pLoop->u.btree.nEq; - u16 nSkip = pLoop->nSkip; - int i, j; +SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ + memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); + return pWInfo->okOnePass; +} - if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; - sqlite3StrAccumAppend(pStr, " (", 2); - for(i=0; i=nSkip ? "%s=?" : "ANY(%s)", z); - } +/* +** Move the content of pSrc into pDest +*/ +static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ + pDest->n = pSrc->n; + memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); +} - j = i; - if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ - explainAppendTerm(pStr, pIndex, pLoop->u.btree.nBtm, j, i, ">"); - i = 1; +/* +** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. +** +** The new entry might overwrite an existing entry, or it might be +** appended, or it might be discarded. Do whatever is the right thing +** so that pSet keeps the N_OR_COST best entries seen so far. +*/ +static int whereOrInsert( + WhereOrSet *pSet, /* The WhereOrSet to be updated */ + Bitmask prereq, /* Prerequisites of the new entry */ + LogEst rRun, /* Run-cost of the new entry */ + LogEst nOut /* Number of outputs for the new entry */ +){ + u16 i; + WhereOrCost *p; + for(i=pSet->n, p=pSet->a; i>0; i--, p++){ + if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ + goto whereOrInsert_done; + } + if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ + return 0; + } } - if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ - explainAppendTerm(pStr, pIndex, pLoop->u.btree.nTop, j, i, "<"); + if( pSet->na[pSet->n++]; + p->nOut = nOut; + }else{ + p = pSet->a; + for(i=1; in; i++){ + if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; + } + if( p->rRun<=rRun ) return 0; } - sqlite3StrAccumAppend(pStr, ")", 1); +whereOrInsert_done: + p->prereq = prereq; + p->rRun = rRun; + if( p->nOut>nOut ) p->nOut = nOut; + return 1; } /* -** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was -** defined at compile-time. If it is not a no-op, a single OP_Explain opcode -** is added to the output to describe the table scan strategy in pLevel. -** -** If an OP_Explain opcode is added to the VM, its address is returned. -** Otherwise, if no OP_Explain is coded, zero is returned. +** Initialize a preallocated WhereClause structure. */ -SQLITE_PRIVATE int sqlite3WhereExplainOneScan( - Parse *pParse, /* Parse context */ - SrcList *pTabList, /* Table list this loop refers to */ - WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ - int iLevel, /* Value for "level" column of output */ - int iFrom, /* Value for "from" column of output */ - u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +static void whereClauseInit( + WhereClause *pWC, /* The WhereClause to be initialized */ + WhereInfo *pWInfo /* The WHERE processing context */ ){ - int ret = 0; -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) - if( pParse->explain==2 ) -#endif - { - struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; - Vdbe *v = pParse->pVdbe; /* VM being constructed */ - sqlite3 *db = pParse->db; /* Database handle */ - int iId = pParse->iSelectId; /* Select id (left-most output column) */ - int isSearch; /* True for a SEARCH. False for SCAN. */ - WhereLoop *pLoop; /* The controlling WhereLoop object */ - u32 flags; /* Flags that describe this loop */ - char *zMsg; /* Text to add to EQP output */ - StrAccum str; /* EQP output string */ - char zBuf[100]; /* Initial space for EQP output string */ - - pLoop = pLevel->pWLoop; - flags = pLoop->wsFlags; - if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_OR_SUBCLAUSE) ) return 0; + pWC->pWInfo = pWInfo; + pWC->pOuter = 0; + pWC->nTerm = 0; + pWC->nSlot = ArraySize(pWC->aStatic); + pWC->a = pWC->aStatic; +} - isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 - || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) - || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); +/* Forward reference */ +static void whereClauseClear(WhereClause*); - sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); - sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); - if( pItem->pSelect ){ - sqlite3XPrintf(&str, " SUBQUERY %d", pItem->iSelectId); - }else{ - sqlite3XPrintf(&str, " TABLE %s", pItem->zName); - } +/* +** Deallocate all memory associated with a WhereOrInfo object. +*/ +static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ + whereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} - if( pItem->zAlias ){ - sqlite3XPrintf(&str, " AS %s", pItem->zAlias); - } - if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ - const char *zFmt = 0; - Index *pIdx; +/* +** Deallocate all memory associated with a WhereAndInfo object. +*/ +static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ + whereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} - assert( pLoop->u.btree.pIndex!=0 ); - pIdx = pLoop->u.btree.pIndex; - assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); - if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ - if( isSearch ){ - zFmt = "PRIMARY KEY"; - } - }else if( flags & WHERE_PARTIALIDX ){ - zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; - }else if( flags & WHERE_AUTO_INDEX ){ - zFmt = "AUTOMATIC COVERING INDEX"; - }else if( flags & WHERE_IDX_ONLY ){ - zFmt = "COVERING INDEX %s"; - }else{ - zFmt = "INDEX %s"; - } - if( zFmt ){ - sqlite3StrAccumAppend(&str, " USING ", 7); - sqlite3XPrintf(&str, zFmt, pIdx->zName); - explainIndexRange(&str, pLoop); - } - }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ - const char *zRangeOp; - if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ - zRangeOp = "="; - }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ - zRangeOp = ">? AND rowid<"; - }else if( flags&WHERE_BTM_LIMIT ){ - zRangeOp = ">"; - }else{ - assert( flags&WHERE_TOP_LIMIT); - zRangeOp = "<"; - } - sqlite3XPrintf(&str, " USING INTEGER PRIMARY KEY (rowid%s?)",zRangeOp); - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ - sqlite3XPrintf(&str, " VIRTUAL TABLE INDEX %d:%s", - pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); +/* +** Deallocate a WhereClause structure. The WhereClause structure +** itself is not freed. This routine is the inverse of whereClauseInit(). +*/ +static void whereClauseClear(WhereClause *pWC){ + int i; + WhereTerm *a; + sqlite3 *db = pWC->pWInfo->pParse->db; + for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ + if( a->wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, a->pExpr); } -#endif -#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS - if( pLoop->nOut>=10 ){ - sqlite3XPrintf(&str, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); - }else{ - sqlite3StrAccumAppend(&str, " (~1 row)", 9); + if( a->wtFlags & TERM_ORINFO ){ + whereOrInfoDelete(db, a->u.pOrInfo); + }else if( a->wtFlags & TERM_ANDINFO ){ + whereAndInfoDelete(db, a->u.pAndInfo); } -#endif - zMsg = sqlite3StrAccumFinish(&str); - ret = sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg,P4_DYNAMIC); } - return ret; + if( pWC->a!=pWC->aStatic ){ + sqlite3DbFree(db, pWC->a); + } } -#endif /* SQLITE_OMIT_EXPLAIN */ -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* -** Configure the VM passed as the first argument with an -** sqlite3_stmt_scanstatus() entry corresponding to the scan used to -** implement level pLvl. Argument pSrclist is a pointer to the FROM -** clause that the scan reads data from. +** Add a single new WhereTerm entry to the WhereClause object pWC. +** The new WhereTerm object is constructed from Expr p and with wtFlags. +** The index in pWC->a[] of the new WhereTerm is returned on success. +** 0 is returned if the new WhereTerm could not be added due to a memory +** allocation error. The memory allocation failure will be recorded in +** the db->mallocFailed flag so that higher-level functions can detect it. ** -** If argument addrExplain is not 0, it must be the address of an -** OP_Explain instruction that describes the same loop. +** This routine will increase the size of the pWC->a[] array as necessary. +** +** If the wtFlags argument includes TERM_DYNAMIC, then responsibility +** for freeing the expression p is assumed by the WhereClause object pWC. +** This is true even if this routine fails to allocate a new WhereTerm. +** +** WARNING: This routine might reallocate the space used to store +** WhereTerms. All pointers to WhereTerms should be invalidated after +** calling this routine. Such pointers may be reinitialized by referencing +** the pWC->a[] array. */ -SQLITE_PRIVATE void sqlite3WhereAddScanStatus( - Vdbe *v, /* Vdbe to add scanstatus entry to */ - SrcList *pSrclist, /* FROM clause pLvl reads data from */ - WhereLevel *pLvl, /* Level to add scanstatus() entry for */ - int addrExplain /* Address of OP_Explain (or 0) */ -){ - const char *zObj = 0; - WhereLoop *pLoop = pLvl->pWLoop; - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ - zObj = pLoop->u.btree.pIndex->zName; +static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ + WhereTerm *pTerm; + int idx; + testcase( wtFlags & TERM_VIRTUAL ); + if( pWC->nTerm>=pWC->nSlot ){ + WhereTerm *pOld = pWC->a; + sqlite3 *db = pWC->pWInfo->pParse->db; + pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); + if( pWC->a==0 ){ + if( wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, p); + } + pWC->a = pOld; + return 0; + } + memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); + if( pOld!=pWC->aStatic ){ + sqlite3DbFree(db, pOld); + } + pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); + memset(&pWC->a[pWC->nTerm], 0, sizeof(pWC->a[0])*(pWC->nSlot-pWC->nTerm)); + } + pTerm = &pWC->a[idx = pWC->nTerm++]; + if( p && ExprHasProperty(p, EP_Unlikely) ){ + pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; }else{ - zObj = pSrclist->a[pLvl->iFrom].zName; + pTerm->truthProb = 1; } - sqlite3VdbeScanStatus( - v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj - ); + pTerm->pExpr = sqlite3ExprSkipCollate(p); + pTerm->wtFlags = wtFlags; + pTerm->pWC = pWC; + pTerm->iParent = -1; + return idx; } -#endif - /* -** Disable a term in the WHERE clause. Except, do not disable the term -** if it controls a LEFT OUTER JOIN and it did not originate in the ON -** or USING clause of that join. -** -** Consider the term t2.z='ok' in the following queries: -** -** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' -** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' -** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' -** -** The t2.z='ok' is disabled in the in (2) because it originates -** in the ON clause. The term is disabled in (3) because it is not part -** of a LEFT OUTER JOIN. In (1), the term is not disabled. -** -** Disabling a term causes that term to not be tested in the inner loop -** of the join. Disabling is an optimization. When terms are satisfied -** by indices, we disable them to prevent redundant tests in the inner -** loop. We would get the correct results if nothing were ever disabled, -** but joins might run a little slower. The trick is to disable as much -** as we can without disabling too much. If we disabled in (1), we'd get -** the wrong answer. See ticket #813. -** -** If all the children of a term are disabled, then that term is also -** automatically disabled. In this way, terms get disabled if derived -** virtual terms are tested first. For example: +** This routine identifies subexpressions in the WHERE clause where +** each subexpression is separated by the AND operator or some other +** operator specified in the op parameter. The WhereClause structure +** is filled with pointers to subexpressions. For example: ** -** x GLOB 'abc*' AND x>='abc' AND x<'acd' -** \___________/ \______/ \_____/ -** parent child1 child2 +** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) +** \________/ \_______________/ \________________/ +** slot[0] slot[1] slot[2] ** -** Only the parent term was in the original WHERE clause. The child1 -** and child2 terms were added by the LIKE optimization. If both of -** the virtual child terms are valid, then testing of the parent can be -** skipped. +** The original WHERE clause in pExpr is unaltered. All this routine +** does is make slot[] entries point to substructure within pExpr. ** -** Usually the parent term is marked as TERM_CODED. But if the parent -** term was originally TERM_LIKE, then the parent gets TERM_LIKECOND instead. -** The TERM_LIKECOND marking indicates that the term should be coded inside -** a conditional such that is only evaluated on the second pass of a -** LIKE-optimization loop, when scanning BLOBs instead of strings. +** In the previous sentence and in the diagram, "slot[]" refers to +** the WhereClause.a[] array. The slot[] array grows as needed to contain +** all terms of the WHERE clause. */ -static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ - int nLoop = 0; - while( ALWAYS(pTerm!=0) - && (pTerm->wtFlags & TERM_CODED)==0 - && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) - && (pLevel->notReady & pTerm->prereqAll)==0 - ){ - if( nLoop && (pTerm->wtFlags & TERM_LIKE)!=0 ){ - pTerm->wtFlags |= TERM_LIKECOND; - }else{ - pTerm->wtFlags |= TERM_CODED; - } - if( pTerm->iParent<0 ) break; - pTerm = &pTerm->pWC->a[pTerm->iParent]; - pTerm->nChild--; - if( pTerm->nChild!=0 ) break; - nLoop++; +static void whereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ + Expr *pE2 = sqlite3ExprSkipCollate(pExpr); + pWC->op = op; + if( pE2==0 ) return; + if( pE2->op!=op ){ + whereClauseInsert(pWC, pExpr, 0); + }else{ + whereSplit(pWC, pE2->pLeft, op); + whereSplit(pWC, pE2->pRight, op); } } /* -** Code an OP_Affinity opcode to apply the column affinity string zAff -** to the n registers starting at base. -** -** As an optimization, SQLITE_AFF_BLOB entries (which are no-ops) at the -** beginning and end of zAff are ignored. If all entries in zAff are -** SQLITE_AFF_BLOB, then no code gets generated. -** -** This routine makes its own copy of zAff so that the caller is free -** to modify zAff after this routine returns. +** Initialize a WhereMaskSet object */ -static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ - Vdbe *v = pParse->pVdbe; - if( zAff==0 ){ - assert( pParse->db->mallocFailed ); - return; - } - assert( v!=0 ); - - /* Adjust base and n to skip over SQLITE_AFF_BLOB entries at the beginning - ** and end of the affinity string. - */ - while( n>0 && zAff[0]==SQLITE_AFF_BLOB ){ - n--; - base++; - zAff++; - } - while( n>1 && zAff[n-1]==SQLITE_AFF_BLOB ){ - n--; - } - - /* Code the OP_Affinity opcode if there is anything left to do. */ - if( n>0 ){ - sqlite3VdbeAddOp4(v, OP_Affinity, base, n, 0, zAff, n); - sqlite3ExprCacheAffinityChange(pParse, base, n); - } -} +#define initMaskSet(P) (P)->n=0 /* -** Expression pRight, which is the RHS of a comparison operation, is -** either a vector of n elements or, if n==1, a scalar expression. -** Before the comparison operation, affinity zAff is to be applied -** to the pRight values. This function modifies characters within the -** affinity string to SQLITE_AFF_BLOB if either: -** -** * the comparison will be performed with no affinity, or -** * the affinity change in zAff is guaranteed not to change the value. +** Return the bitmask for the given cursor number. Return 0 if +** iCursor is not in the set. */ -static void updateRangeAffinityStr( - Expr *pRight, /* RHS of comparison */ - int n, /* Number of vector elements in comparison */ - char *zAff /* Affinity string to modify */ -){ +static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){ int i; - for(i=0; in<=(int)sizeof(Bitmask)*8 ); + for(i=0; in; i++){ + if( pMaskSet->ix[i]==iCursor ){ + return MASKBIT(i); } } + return 0; } /* -** Generate code for a single equality term of the WHERE clause. An equality -** term can be either X=expr or X IN (...). pTerm is the term to be -** coded. -** -** The current value for the constraint is left in a register, the index -** of which is returned. An attempt is made store the result in iTarget but -** this is only guaranteed for TK_ISNULL and TK_IN constraints. If the -** constraint is a TK_EQ or TK_IS, then the current value might be left in -** some other register and it is the caller's responsibility to compensate. +** Create a new mask for cursor iCursor. ** -** For a constraint of the form X=expr, the expression is evaluated in -** straight-line code. For constraints of the form X IN (...) -** this routine sets up a loop that will iterate over all values of X. +** There is one cursor per table in the FROM clause. The number of +** tables in the FROM clause is limited by a test early in the +** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] +** array will never overflow. */ -static int codeEqualityTerm( - Parse *pParse, /* The parsing context */ - WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ - WhereLevel *pLevel, /* The level of the FROM clause we are working on */ - int iEq, /* Index of the equality term within this level */ - int bRev, /* True for reverse-order IN operations */ - int iTarget /* Attempt to leave results in this register */ -){ - Expr *pX = pTerm->pExpr; - Vdbe *v = pParse->pVdbe; - int iReg; /* Register holding results */ +static void createMask(WhereMaskSet *pMaskSet, int iCursor){ + assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); + pMaskSet->ix[pMaskSet->n++] = iCursor; +} - assert( pLevel->pWLoop->aLTerm[iEq]==pTerm ); - assert( iTarget>0 ); - if( pX->op==TK_EQ || pX->op==TK_IS ){ - iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); - }else if( pX->op==TK_ISNULL ){ - iReg = iTarget; - sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); -#ifndef SQLITE_OMIT_SUBQUERY +/* +** These routines walk (recursively) an expression tree and generate +** a bitmask indicating which tables are used in that expression +** tree. +*/ +static Bitmask exprListTableUsage(WhereMaskSet*, ExprList*); +static Bitmask exprSelectTableUsage(WhereMaskSet*, Select*); +static Bitmask exprTableUsage(WhereMaskSet *pMaskSet, Expr *p){ + Bitmask mask = 0; + if( p==0 ) return 0; + if( p->op==TK_COLUMN ){ + mask = getMask(pMaskSet, p->iTable); + return mask; + } + mask = exprTableUsage(pMaskSet, p->pRight); + mask |= exprTableUsage(pMaskSet, p->pLeft); + if( ExprHasProperty(p, EP_xIsSelect) ){ + mask |= exprSelectTableUsage(pMaskSet, p->x.pSelect); }else{ - int eType = IN_INDEX_NOOP; - int iTab; - struct InLoop *pIn; - WhereLoop *pLoop = pLevel->pWLoop; - int i; - int nEq = 0; - int *aiMap = 0; - - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 - && pLoop->u.btree.pIndex!=0 - && pLoop->u.btree.pIndex->aSortOrder[iEq] - ){ - testcase( iEq==0 ); - testcase( bRev ); - bRev = !bRev; + mask |= exprListTableUsage(pMaskSet, p->x.pList); + } + return mask; +} +static Bitmask exprListTableUsage(WhereMaskSet *pMaskSet, ExprList *pList){ + int i; + Bitmask mask = 0; + if( pList ){ + for(i=0; inExpr; i++){ + mask |= exprTableUsage(pMaskSet, pList->a[i].pExpr); } - assert( pX->op==TK_IN ); - iReg = iTarget; - - for(i=0; iaLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){ - disableTerm(pLevel, pTerm); - return iTarget; + } + return mask; +} +static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){ + Bitmask mask = 0; + while( pS ){ + SrcList *pSrc = pS->pSrc; + mask |= exprListTableUsage(pMaskSet, pS->pEList); + mask |= exprListTableUsage(pMaskSet, pS->pGroupBy); + mask |= exprListTableUsage(pMaskSet, pS->pOrderBy); + mask |= exprTableUsage(pMaskSet, pS->pWhere); + mask |= exprTableUsage(pMaskSet, pS->pHaving); + if( ALWAYS(pSrc!=0) ){ + int i; + for(i=0; inSrc; i++){ + mask |= exprSelectTableUsage(pMaskSet, pSrc->a[i].pSelect); + mask |= exprTableUsage(pMaskSet, pSrc->a[i].pOn); } } - for(i=iEq;inLTerm; i++){ - if( ALWAYS(pLoop->aLTerm[i]) && pLoop->aLTerm[i]->pExpr==pX ) nEq++; - } - - if( (pX->flags & EP_xIsSelect)==0 || pX->x.pSelect->pEList->nExpr==1 ){ - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0); - }else{ - Select *pSelect = pX->x.pSelect; - sqlite3 *db = pParse->db; - u16 savedDbOptFlags = db->dbOptFlags; - ExprList *pOrigRhs = pSelect->pEList; - ExprList *pOrigLhs = pX->pLeft->x.pList; - ExprList *pRhs = 0; /* New Select.pEList for RHS */ - ExprList *pLhs = 0; /* New pX->pLeft vector */ - - for(i=iEq;inLTerm; i++){ - if( pLoop->aLTerm[i]->pExpr==pX ){ - int iField = pLoop->aLTerm[i]->iField - 1; - Expr *pNewRhs = sqlite3ExprDup(db, pOrigRhs->a[iField].pExpr, 0); - Expr *pNewLhs = sqlite3ExprDup(db, pOrigLhs->a[iField].pExpr, 0); + pS = pS->pPrior; + } + return mask; +} - pRhs = sqlite3ExprListAppend(pParse, pRhs, pNewRhs); - pLhs = sqlite3ExprListAppend(pParse, pLhs, pNewLhs); - } - } - if( !db->mallocFailed ){ - Expr *pLeft = pX->pLeft; - - if( pSelect->pOrderBy ){ - /* If the SELECT statement has an ORDER BY clause, zero the - ** iOrderByCol variables. These are set to non-zero when an - ** ORDER BY term exactly matches one of the terms of the - ** result-set. Since the result-set of the SELECT statement may - ** have been modified or reordered, these variables are no longer - ** set correctly. Since setting them is just an optimization, - ** it's easiest just to zero them here. */ - ExprList *pOrderBy = pSelect->pOrderBy; - for(i=0; inExpr; i++){ - pOrderBy->a[i].u.x.iOrderByCol = 0; - } - } +/* +** Return TRUE if the given operator is one of the operators that is +** allowed for an indexable WHERE clause term. The allowed operators are +** "=", "<", ">", "<=", ">=", "IN", and "IS NULL" +*/ +static int allowedOp(int op){ + assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL; +} - /* Take care here not to generate a TK_VECTOR containing only a - ** single value. Since the parser never creates such a vector, some - ** of the subroutines do not handle this case. */ - if( pLhs->nExpr==1 ){ - pX->pLeft = pLhs->a[0].pExpr; - }else{ - pLeft->x.pList = pLhs; - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int) * nEq); - testcase( aiMap==0 ); - } - pSelect->pEList = pRhs; - db->dbOptFlags |= SQLITE_QueryFlattener; - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap); - db->dbOptFlags = savedDbOptFlags; - testcase( aiMap!=0 && aiMap[0]!=0 ); - pSelect->pEList = pOrigRhs; - pLeft->x.pList = pOrigLhs; - pX->pLeft = pLeft; - } - sqlite3ExprListDelete(pParse->db, pLhs); - sqlite3ExprListDelete(pParse->db, pRhs); +/* +** Commute a comparison operator. Expressions of the form "X op Y" +** are converted into "Y op X". +** +** If left/right precedence rules come into play when determining the +** collating sequence, then COLLATE operators are adjusted to ensure +** that the collating sequence does not change. For example: +** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on +** the left hand side of a comparison overrides any collation sequence +** attached to the right. For the same reason the EP_Collate flag +** is not commuted. +*/ +static void exprCommute(Parse *pParse, Expr *pExpr){ + u16 expRight = (pExpr->pRight->flags & EP_Collate); + u16 expLeft = (pExpr->pLeft->flags & EP_Collate); + assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); + if( expRight==expLeft ){ + /* Either X and Y both have COLLATE operator or neither do */ + if( expRight ){ + /* Both X and Y have COLLATE operators. Make sure X is always + ** used by clearing the EP_Collate flag from Y. */ + pExpr->pRight->flags &= ~EP_Collate; + }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ + /* Neither X nor Y have COLLATE operators, but X has a non-default + ** collating sequence. So add the EP_Collate marker on X to cause + ** it to be searched first. */ + pExpr->pLeft->flags |= EP_Collate; } + } + SWAP(Expr*,pExpr->pRight,pExpr->pLeft); + if( pExpr->op>=TK_GT ){ + assert( TK_LT==TK_GT+2 ); + assert( TK_GE==TK_LE+2 ); + assert( TK_GT>TK_EQ ); + assert( TK_GTop>=TK_GT && pExpr->op<=TK_GE ); + pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; + } +} - if( eType==IN_INDEX_INDEX_DESC ){ - testcase( bRev ); - bRev = !bRev; - } - iTab = pX->iTable; - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); - VdbeCoverageIf(v, bRev); - VdbeCoverageIf(v, !bRev); - assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); +/* +** Translate from TK_xx operator to WO_xx bitmask. +*/ +static u16 operatorMask(int op){ + u16 c; + assert( allowedOp(op) ); + if( op==TK_IN ){ + c = WO_IN; + }else if( op==TK_ISNULL ){ + c = WO_ISNULL; + }else{ + assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); + c = (u16)(WO_EQ<<(op-TK_EQ)); + } + assert( op!=TK_ISNULL || c==WO_ISNULL ); + assert( op!=TK_IN || c==WO_IN ); + assert( op!=TK_EQ || c==WO_EQ ); + assert( op!=TK_LT || c==WO_LT ); + assert( op!=TK_LE || c==WO_LE ); + assert( op!=TK_GT || c==WO_GT ); + assert( op!=TK_GE || c==WO_GE ); + return c; +} - pLoop->wsFlags |= WHERE_IN_ABLE; - if( pLevel->u.in.nIn==0 ){ - pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - } +/* +** Advance to the next WhereTerm that matches according to the criteria +** established when the pScan object was initialized by whereScanInit(). +** Return NULL if there are no more matching WhereTerms. +*/ +static WhereTerm *whereScanNext(WhereScan *pScan){ + int iCur; /* The cursor on the LHS of the term */ + int iColumn; /* The column on the LHS of the term. -1 for IPK */ + Expr *pX; /* An expression being tested */ + WhereClause *pWC; /* Shorthand for pScan->pWC */ + WhereTerm *pTerm; /* The term being tested */ + int k = pScan->k; /* Where to start scanning */ - i = pLevel->u.in.nIn; - pLevel->u.in.nIn += nEq; - pLevel->u.in.aInLoop = - sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, - sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); - pIn = pLevel->u.in.aInLoop; - if( pIn ){ - int iMap = 0; /* Index in aiMap[] */ - pIn += i; - for(i=iEq;inLTerm; i++){ - if( pLoop->aLTerm[i]->pExpr==pX ){ - int iOut = iReg + i - iEq; - if( eType==IN_INDEX_ROWID ){ - testcase( nEq>1 ); /* Happens with a UNIQUE index on ROWID */ - pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut); - }else{ - int iCol = aiMap ? aiMap[iMap++] : 0; - pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut); + while( pScan->iEquiv<=pScan->nEquiv ){ + iCur = pScan->aEquiv[pScan->iEquiv-2]; + iColumn = pScan->aEquiv[pScan->iEquiv-1]; + while( (pWC = pScan->pWC)!=0 ){ + for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ + if( pTerm->leftCursor==iCur + && pTerm->u.leftColumn==iColumn + && (pScan->iEquiv<=2 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + ){ + if( (pTerm->eOperator & WO_EQUIV)!=0 + && pScan->nEquivaEquiv) + ){ + int j; + pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); + assert( pX->op==TK_COLUMN ); + for(j=0; jnEquiv; j+=2){ + if( pScan->aEquiv[j]==pX->iTable + && pScan->aEquiv[j+1]==pX->iColumn ){ + break; + } + } + if( j==pScan->nEquiv ){ + pScan->aEquiv[j] = pX->iTable; + pScan->aEquiv[j+1] = pX->iColumn; + pScan->nEquiv += 2; + } } - sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v); - if( i==iEq ){ - pIn->iCur = iTab; - pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; - }else{ - pIn->eEndLoopOp = OP_Noop; + if( (pTerm->eOperator & pScan->opMask)!=0 ){ + /* Verify the affinity and collating sequence match */ + if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + Parse *pParse = pWC->pWInfo->pParse; + pX = pTerm->pExpr; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse, + pX->pLeft, pX->pRight); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + continue; + } + } + if( (pTerm->eOperator & WO_EQ)!=0 + && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN + && pX->iTable==pScan->aEquiv[0] + && pX->iColumn==pScan->aEquiv[1] + ){ + continue; + } + pScan->k = k+1; + return pTerm; } - pIn++; } } - }else{ - pLevel->u.in.nIn = 0; + pScan->pWC = pScan->pWC->pOuter; + k = 0; } - sqlite3DbFree(pParse->db, aiMap); -#endif + pScan->pWC = pScan->pOrigWC; + k = 0; + pScan->iEquiv += 2; } - disableTerm(pLevel, pTerm); - return iReg; + return 0; } /* -** Generate code that will evaluate all == and IN constraints for an -** index scan. -** -** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). -** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 -** The index has as many as three equality constraints, but in this -** example, the third "c" value is an inequality. So only two -** constraints are coded. This routine will generate code to evaluate -** a==5 and b IN (1,2,3). The current values for a and b will be stored -** in consecutive registers and the index of the first register is returned. -** -** In the example above nEq==2. But this subroutine works for any value -** of nEq including 0. If nEq==0, this routine is nearly a no-op. -** The only thing it does is allocate the pLevel->iMem memory cell and -** compute the affinity string. -** -** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints -** are == or IN and are covered by the nEq. nExtraReg is 1 if there is -** an inequality constraint (such as the "c>=5 AND c<10" in the example) that -** occurs after the nEq quality constraints. -** -** This routine allocates a range of nEq+nExtraReg memory cells and returns -** the index of the first memory cell in that range. The code that -** calls this routine will use that memory range to store keys for -** start and termination conditions of the loop. -** key value of the loop. If one or more IN operators appear, then -** this routine allocates an additional nEq memory cells for internal -** use. +** Initialize a WHERE clause scanner object. Return a pointer to the +** first match. Return NULL if there are no matches. ** -** Before returning, *pzAff is set to point to a buffer containing a -** copy of the column affinity string of the index allocated using -** sqlite3DbMalloc(). Except, entries in the copy of the string associated -** with equality constraints that use BLOB or NONE affinity are set to -** SQLITE_AFF_BLOB. This is to deal with SQL such as the following: +** The scanner will be searching the WHERE clause pWC. It will look +** for terms of the form "X " where X is column iColumn of table +** iCur. The must be one of the operators described by opMask. ** -** CREATE TABLE t1(a TEXT PRIMARY KEY, b); -** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; +** If the search is for X and the WHERE clause contains terms of the +** form X=Y then this routine might also return terms of the form +** "Y ". The number of levels of transitivity is limited, +** but is enough to handle most commonly occurring SQL statements. ** -** In the example above, the index on t1(a) has TEXT affinity. But since -** the right hand side of the equality constraint (t2.b) has BLOB/NONE affinity, -** no conversion should be attempted before using a t2.b value as part of -** a key to search the index. Hence the first byte in the returned affinity -** string in this example would be set to SQLITE_AFF_BLOB. +** If X is not the INTEGER PRIMARY KEY then X must be compatible with +** index pIdx. */ -static int codeAllEqualityTerms( - Parse *pParse, /* Parsing context */ - WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ - int bRev, /* Reverse the order of IN operators */ - int nExtraReg, /* Number of extra registers to allocate */ - char **pzAff /* OUT: Set to point to affinity string */ +static WhereTerm *whereScanInit( + WhereScan *pScan, /* The WhereScan object being initialized */ + WhereClause *pWC, /* The WHERE clause to be scanned */ + int iCur, /* Cursor to scan for */ + int iColumn, /* Column to scan for */ + u32 opMask, /* Operator(s) to scan for */ + Index *pIdx /* Must be compatible with this index */ ){ - u16 nEq; /* The number of == or IN constraints to code */ - u16 nSkip; /* Number of left-most columns to skip */ - Vdbe *v = pParse->pVdbe; /* The vm under construction */ - Index *pIdx; /* The index being used for this loop */ - WhereTerm *pTerm; /* A single constraint term */ - WhereLoop *pLoop; /* The WhereLoop object */ - int j; /* Loop counter */ - int regBase; /* Base register */ - int nReg; /* Number of registers to allocate */ - char *zAff; /* Affinity string to return */ - - /* This module is only called on query plans that use an index. */ - pLoop = pLevel->pWLoop; - assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); - nEq = pLoop->u.btree.nEq; - nSkip = pLoop->nSkip; - pIdx = pLoop->u.btree.pIndex; - assert( pIdx!=0 ); - - /* Figure out how many memory cells we will need then allocate them. - */ - regBase = pParse->nMem + 1; - nReg = pLoop->u.btree.nEq + nExtraReg; - pParse->nMem += nReg; - - zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx)); - assert( zAff!=0 || pParse->db->mallocFailed ); - - if( nSkip ){ - int iIdxCur = pLevel->iIdxCur; - sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); - j = sqlite3VdbeAddOp0(v, OP_Goto); - pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), - iIdxCur, 0, regBase, nSkip); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - sqlite3VdbeJumpHere(v, j); - for(j=0; jaiColumn[j]==XN_EXPR ); - VdbeComment((v, "%s", explainIndexColumnName(pIdx, j))); - } - } + int j; - /* Evaluate the equality constraints - */ - assert( zAff==0 || (int)strlen(zAff)>=nEq ); - for(j=nSkip; jaLTerm[j]; - assert( pTerm!=0 ); - /* The following testcase is true for indices with redundant columns. - ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ - testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); - if( r1!=regBase+j ){ - if( nReg==1 ){ - sqlite3ReleaseTempReg(pParse, regBase); - regBase = r1; - }else{ - sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); - } - } - if( pTerm->eOperator & WO_IN ){ - if( pTerm->pExpr->flags & EP_xIsSelect ){ - /* No affinity ever needs to be (or should be) applied to a value - ** from the RHS of an "? IN (SELECT ...)" expression. The - ** sqlite3FindInIndex() routine has already ensured that the - ** affinity of the comparison has been applied to the value. */ - if( zAff ) zAff[j] = SQLITE_AFF_BLOB; - } - }else if( (pTerm->eOperator & WO_ISNULL)==0 ){ - Expr *pRight = pTerm->pExpr->pRight; - if( (pTerm->wtFlags & TERM_IS)==0 && sqlite3ExprCanBeNull(pRight) ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); - VdbeCoverage(v); - } - if( zAff ){ - if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_BLOB ){ - zAff[j] = SQLITE_AFF_BLOB; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ - zAff[j] = SQLITE_AFF_BLOB; - } - } + /* memset(pScan, 0, sizeof(*pScan)); */ + pScan->pOrigWC = pWC; + pScan->pWC = pWC; + if( pIdx && iColumn>=0 ){ + pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; + for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ + if( NEVER(j>pIdx->nColumn) ) return 0; } + pScan->zCollName = pIdx->azColl[j]; + }else{ + pScan->idxaff = 0; + pScan->zCollName = 0; } - *pzAff = zAff; - return regBase; + pScan->opMask = opMask; + pScan->k = 0; + pScan->aEquiv[0] = iCur; + pScan->aEquiv[1] = iColumn; + pScan->nEquiv = 2; + pScan->iEquiv = 2; + return whereScanNext(pScan); } -#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS /* -** If the most recently coded instruction is a constant range constraint -** (a string literal) that originated from the LIKE optimization, then -** set P3 and P5 on the OP_String opcode so that the string will be cast -** to a BLOB at appropriate times. +** Search for a term in the WHERE clause that is of the form "X " +** where X is a reference to the iColumn of table iCur and is one of +** the WO_xx operator codes specified by the op parameter. +** Return a pointer to the term. Return 0 if not found. ** -** The LIKE optimization trys to evaluate "x LIKE 'abc%'" as a range -** expression: "x>='ABC' AND x<'abd'". But this requires that the range -** scan loop run twice, once for strings and a second time for BLOBs. -** The OP_String opcodes on the second pass convert the upper and lower -** bound string constants to blobs. This routine makes the necessary changes -** to the OP_String opcodes for that to happen. +** The term returned might by Y= if there is another constraint in +** the WHERE clause that specifies that X=Y. Any such constraints will be +** identified by the WO_EQUIV bit in the pTerm->eOperator field. The +** aEquiv[] array holds X and all its equivalents, with each SQL variable +** taking up two slots in aEquiv[]. The first slot is for the cursor number +** and the second is for the column number. There are 22 slots in aEquiv[] +** so that means we can look for X plus up to 10 other equivalent values. +** Hence a search for X will return if X=A1 and A1=A2 and A2=A3 +** and ... and A9=A10 and A10=. ** -** Except, of course, if SQLITE_LIKE_DOESNT_MATCH_BLOBS is defined, then -** only the one pass through the string space is required, so this routine -** becomes a no-op. +** If there are multiple terms in the WHERE clause of the form "X " +** then try for the one with no dependencies on - in other words where +** is a constant expression of some kind. Only return entries of +** the form "X Y" where Y is a column in another table if no terms of +** the form "X " exist. If no terms with a constant RHS +** exist, try to return a term that does not use WO_EQUIV. */ -static void whereLikeOptimizationStringFixup( - Vdbe *v, /* prepared statement under construction */ - WhereLevel *pLevel, /* The loop that contains the LIKE operator */ - WhereTerm *pTerm /* The upper or lower bound just coded */ +static WhereTerm *findTerm( + WhereClause *pWC, /* The WHERE clause to be searched */ + int iCur, /* Cursor number of LHS */ + int iColumn, /* Column number of LHS */ + Bitmask notReady, /* RHS must not overlap with this mask */ + u32 op, /* Mask of WO_xx values describing operator */ + Index *pIdx /* Must be compatible with this index, if not NULL */ ){ - if( pTerm->wtFlags & TERM_LIKEOPT ){ - VdbeOp *pOp; - assert( pLevel->iLikeRepCntr>0 ); - pOp = sqlite3VdbeGetOp(v, -1); - assert( pOp!=0 ); - assert( pOp->opcode==OP_String8 - || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); - pOp->p3 = (int)(pLevel->iLikeRepCntr>>1); /* Register holding counter */ - pOp->p5 = (u8)(pLevel->iLikeRepCntr&1); /* ASC or DESC */ + WhereTerm *pResult = 0; + WhereTerm *p; + WhereScan scan; + + p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); + while( p ){ + if( (p->prereqRight & notReady)==0 ){ + if( p->prereqRight==0 && (p->eOperator&WO_EQ)!=0 ){ + return p; + } + if( pResult==0 ) pResult = p; + } + p = whereScanNext(&scan); } + return pResult; } -#else -# define whereLikeOptimizationStringFixup(A,B,C) -#endif -#ifdef SQLITE_ENABLE_CURSOR_HINTS -/* -** Information is passed from codeCursorHint() down to individual nodes of -** the expression tree (by sqlite3WalkExpr()) using an instance of this -** structure. -*/ -struct CCurHint { - int iTabCur; /* Cursor for the main table */ - int iIdxCur; /* Cursor for the index, if pIdx!=0. Unused otherwise */ - Index *pIdx; /* The index used to access the table */ -}; +/* Forward reference */ +static void exprAnalyze(SrcList*, WhereClause*, int); /* -** This function is called for every node of an expression that is a candidate -** for a cursor hint on an index cursor. For TK_COLUMN nodes that reference -** the table CCurHint.iTabCur, verify that the same column can be -** accessed through the index. If it cannot, then set pWalker->eCode to 1. +** Call exprAnalyze on all terms in a WHERE clause. */ -static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){ - struct CCurHint *pHint = pWalker->u.pCCurHint; - assert( pHint->pIdx!=0 ); - if( pExpr->op==TK_COLUMN - && pExpr->iTable==pHint->iTabCur - && sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn)<0 - ){ - pWalker->eCode = 1; +static void exprAnalyzeAll( + SrcList *pTabList, /* the FROM clause */ + WhereClause *pWC /* the WHERE clause to be analyzed */ +){ + int i; + for(i=pWC->nTerm-1; i>=0; i--){ + exprAnalyze(pTabList, pWC, i); } - return WRC_Continue; } +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION /* -** Test whether or not expression pExpr, which was part of a WHERE clause, -** should be included in the cursor-hint for a table that is on the rhs -** of a LEFT JOIN. Set Walker.eCode to non-zero before returning if the -** expression is not suitable. -** -** An expression is unsuitable if it might evaluate to non NULL even if -** a TK_COLUMN node that does affect the value of the expression is set -** to NULL. For example: +** Check to see if the given expression is a LIKE or GLOB operator that +** can be optimized using inequality constraints. Return TRUE if it is +** so and false if not. ** -** col IS NULL -** col IS NOT NULL -** coalesce(col, 1) -** CASE WHEN col THEN 0 ELSE 1 END +** In order for the operator to be optimizible, the RHS must be a string +** literal that does not begin with a wildcard. The LHS must be a column +** that may only be NULL, a string, or a BLOB, never a number. (This means +** that virtual tables cannot participate in the LIKE optimization.) If the +** collating sequence for the column on the LHS must be appropriate for +** the operator. */ -static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_IS - || pExpr->op==TK_ISNULL || pExpr->op==TK_ISNOT - || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE +static int isLikeOrGlob( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* Test this expression */ + Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ + int *pisComplete, /* True if the only wildcard is % in the last character */ + int *pnoCase /* True if uppercase is equivalent to lowercase */ +){ + const char *z = 0; /* String on RHS of LIKE operator */ + Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ + ExprList *pList; /* List of operands to the LIKE operator */ + int c; /* One character in z[] */ + int cnt; /* Number of non-wildcard prefix characters */ + char wc[3]; /* Wildcard characters */ + sqlite3 *db = pParse->db; /* Database connection */ + sqlite3_value *pVal = 0; + int op; /* Opcode of pRight */ + + if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ + return 0; + } +#ifdef SQLITE_EBCDIC + if( *pnoCase ) return 0; +#endif + pList = pExpr->x.pList; + pLeft = pList->a[1].pExpr; + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) /* Value might be numeric */ ){ - pWalker->eCode = 1; - }else if( pExpr->op==TK_FUNCTION ){ - int d1; - char d2[3]; - if( 0==sqlite3IsLikeFunction(pWalker->pParse->db, pExpr, &d1, d2) ){ - pWalker->eCode = 1; + /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must + ** be the name of an indexed column with TEXT affinity. */ + return 0; + } + assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ + + pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); + op = pRight->op; + if( op==TK_VARIABLE ){ + Vdbe *pReprepare = pParse->pReprepare; + int iCol = pRight->iColumn; + pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_NONE); + if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ + z = (char *)sqlite3_value_text(pVal); + } + sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); + assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); + }else if( op==TK_STRING ){ + z = pRight->u.zToken; + } + if( z ){ + cnt = 0; + while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ + cnt++; + } + if( cnt!=0 && 255!=(u8)z[cnt-1] ){ + Expr *pPrefix; + *pisComplete = c==wc[0] && z[cnt+1]==0; + pPrefix = sqlite3Expr(db, TK_STRING, z); + if( pPrefix ) pPrefix->u.zToken[cnt] = 0; + *ppPrefix = pPrefix; + if( op==TK_VARIABLE ){ + Vdbe *v = pParse->pVdbe; + sqlite3VdbeSetVarmask(v, pRight->iColumn); + if( *pisComplete && pRight->u.zToken[1] ){ + /* If the rhs of the LIKE expression is a variable, and the current + ** value of the variable means there is no need to invoke the LIKE + ** function, then no OP_Variable will be added to the program. + ** This causes problems for the sqlite3_bind_parameter_name() + ** API. To work around them, add a dummy OP_Variable here. + */ + int r1 = sqlite3GetTempReg(pParse); + sqlite3ExprCodeTarget(pParse, pRight, r1); + sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0); + sqlite3ReleaseTempReg(pParse, r1); + } + } + }else{ + z = 0; } } - return WRC_Continue; + sqlite3ValueFree(pVal); + return (z!=0); } +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ +#ifndef SQLITE_OMIT_VIRTUALTABLE /* -** This function is called on every node of an expression tree used as an -** argument to the OP_CursorHint instruction. If the node is a TK_COLUMN -** that accesses any table other than the one identified by -** CCurHint.iTabCur, then do the following: -** -** 1) allocate a register and code an OP_Column instruction to read -** the specified column into the new register, and +** Check to see if the given expression is of the form ** -** 2) transform the expression node to a TK_REGISTER node that reads -** from the newly populated register. +** column MATCH expr ** -** Also, if the node is a TK_COLUMN that does access the table idenified -** by pCCurHint.iTabCur, and an index is being used (which we will -** know because CCurHint.pIdx!=0) then transform the TK_COLUMN into -** an access of the index rather than the original table. +** If it is then return TRUE. If not, return FALSE. */ -static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ - int rc = WRC_Continue; - struct CCurHint *pHint = pWalker->u.pCCurHint; - if( pExpr->op==TK_COLUMN ){ - if( pExpr->iTable!=pHint->iTabCur ){ - Vdbe *v = pWalker->pParse->pVdbe; - int reg = ++pWalker->pParse->nMem; /* Register for column value */ - sqlite3ExprCodeGetColumnOfTable( - v, pExpr->pTab, pExpr->iTable, pExpr->iColumn, reg - ); - pExpr->op = TK_REGISTER; - pExpr->iTable = reg; - }else if( pHint->pIdx!=0 ){ - pExpr->iTable = pHint->iIdxCur; - pExpr->iColumn = sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn); - assert( pExpr->iColumn>=0 ); - } - }else if( pExpr->op==TK_AGG_FUNCTION ){ - /* An aggregate function in the WHERE clause of a query means this must - ** be a correlated sub-query, and expression pExpr is an aggregate from - ** the parent context. Do not walk the function arguments in this case. - ** - ** todo: It should be possible to replace this node with a TK_REGISTER - ** expression, as the result of the expression must be stored in a - ** register at this point. The same holds for TK_AGG_COLUMN nodes. */ - rc = WRC_Prune; +static int isMatchOfColumn( + Expr *pExpr /* Test this expression */ +){ + ExprList *pList; + + if( pExpr->op!=TK_FUNCTION ){ + return 0; } - return rc; + if( sqlite3StrICmp(pExpr->u.zToken,"match")!=0 ){ + return 0; + } + pList = pExpr->x.pList; + if( pList->nExpr!=2 ){ + return 0; + } + if( pList->a[1].pExpr->op != TK_COLUMN ){ + return 0; + } + return 1; } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ /* -** Insert an OP_CursorHint instruction if it is appropriate to do so. +** If the pBase expression originated in the ON or USING clause of +** a join, then transfer the appropriate markings over to derived. */ -static void codeCursorHint( - struct SrcList_item *pTabItem, /* FROM clause item */ - WhereInfo *pWInfo, /* The where clause */ - WhereLevel *pLevel, /* Which loop to provide hints for */ - WhereTerm *pEndRange /* Hint this end-of-scan boundary term if not NULL */ -){ - Parse *pParse = pWInfo->pParse; - sqlite3 *db = pParse->db; - Vdbe *v = pParse->pVdbe; - Expr *pExpr = 0; - WhereLoop *pLoop = pLevel->pWLoop; - int iCur; - WhereClause *pWC; - WhereTerm *pTerm; - int i, j; - struct CCurHint sHint; - Walker sWalker; - - if( OptimizationDisabled(db, SQLITE_CursorHints) ) return; - iCur = pLevel->iTabCur; - assert( iCur==pWInfo->pTabList->a[pLevel->iFrom].iCursor ); - sHint.iTabCur = iCur; - sHint.iIdxCur = pLevel->iIdxCur; - sHint.pIdx = pLoop->u.btree.pIndex; - memset(&sWalker, 0, sizeof(sWalker)); - sWalker.pParse = pParse; - sWalker.u.pCCurHint = &sHint; - pWC = &pWInfo->sWC; - for(i=0; inTerm; i++){ - pTerm = &pWC->a[i]; - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( pTerm->prereqAll & pLevel->notReady ) continue; - - /* Any terms specified as part of the ON(...) clause for any LEFT - ** JOIN for which the current table is not the rhs are omitted - ** from the cursor-hint. - ** - ** If this table is the rhs of a LEFT JOIN, "IS" or "IS NULL" terms - ** that were specified as part of the WHERE clause must be excluded. - ** This is to address the following: - ** - ** SELECT ... t1 LEFT JOIN t2 ON (t1.a=t2.b) WHERE t2.c IS NULL; - ** - ** Say there is a single row in t2 that matches (t1.a=t2.b), but its - ** t2.c values is not NULL. If the (t2.c IS NULL) constraint is - ** pushed down to the cursor, this row is filtered out, causing - ** SQLite to synthesize a row of NULL values. Which does match the - ** WHERE clause, and so the query returns a row. Which is incorrect. - ** - ** For the same reason, WHERE terms such as: - ** - ** WHERE 1 = (t2.c IS NULL) - ** - ** are also excluded. See codeCursorHintIsOrFunction() for details. - */ - if( pTabItem->fg.jointype & JT_LEFT ){ - Expr *pExpr = pTerm->pExpr; - if( !ExprHasProperty(pExpr, EP_FromJoin) - || pExpr->iRightJoinTable!=pTabItem->iCursor - ){ - sWalker.eCode = 0; - sWalker.xExprCallback = codeCursorHintIsOrFunction; - sqlite3WalkExpr(&sWalker, pTerm->pExpr); - if( sWalker.eCode ) continue; - } - }else{ - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue; - } - - /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize - ** the cursor. These terms are not needed as hints for a pure range - ** scan (that has no == terms) so omit them. */ - if( pLoop->u.btree.nEq==0 && pTerm!=pEndRange ){ - for(j=0; jnLTerm && pLoop->aLTerm[j]!=pTerm; j++){} - if( jnLTerm ) continue; - } - - /* No subqueries or non-deterministic functions allowed */ - if( sqlite3ExprContainsSubquery(pTerm->pExpr) ) continue; - - /* For an index scan, make sure referenced columns are actually in - ** the index. */ - if( sHint.pIdx!=0 ){ - sWalker.eCode = 0; - sWalker.xExprCallback = codeCursorHintCheckExpr; - sqlite3WalkExpr(&sWalker, pTerm->pExpr); - if( sWalker.eCode ) continue; - } - - /* If we survive all prior tests, that means this term is worth hinting */ - pExpr = sqlite3ExprAnd(db, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0)); - } - if( pExpr!=0 ){ - sWalker.xExprCallback = codeCursorHintFixExpr; - sqlite3WalkExpr(&sWalker, pExpr); - sqlite3VdbeAddOp4(v, OP_CursorHint, - (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0, - (const char*)pExpr, P4_EXPR); +static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ + if( pDerived ){ + pDerived->flags |= pBase->flags & EP_FromJoin; + pDerived->iRightJoinTable = pBase->iRightJoinTable; } } -#else -# define codeCursorHint(A,B,C,D) /* No-op */ -#endif /* SQLITE_ENABLE_CURSOR_HINTS */ /* -** Cursor iCur is open on an intkey b-tree (a table). Register iRowid contains -** a rowid value just read from cursor iIdxCur, open on index pIdx. This -** function generates code to do a deferred seek of cursor iCur to the -** rowid stored in register iRowid. -** -** Normally, this is just: -** -** OP_Seek $iCur $iRowid -** -** However, if the scan currently being coded is a branch of an OR-loop and -** the statement currently being coded is a SELECT, then P3 of the OP_Seek -** is set to iIdxCur and P4 is set to point to an array of integers -** containing one entry for each column of the table cursor iCur is open -** on. For each table column, if the column is the i'th column of the -** index, then the corresponding array entry is set to (i+1). If the column -** does not appear in the index at all, the array entry is set to 0. +** Mark term iChild as being a child of term iParent */ -static void codeDeferredSeek( - WhereInfo *pWInfo, /* Where clause context */ - Index *pIdx, /* Index scan is using */ - int iCur, /* Cursor for IPK b-tree */ - int iIdxCur /* Index cursor */ -){ - Parse *pParse = pWInfo->pParse; /* Parse context */ - Vdbe *v = pParse->pVdbe; /* Vdbe to generate code within */ - - assert( iIdxCur>0 ); - assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 ); - - sqlite3VdbeAddOp3(v, OP_Seek, iIdxCur, 0, iCur); - if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE) - && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask) - ){ - int i; - Table *pTab = pIdx->pTable; - int *ai = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*(pTab->nCol+1)); - if( ai ){ - ai[0] = pTab->nCol; - for(i=0; inColumn-1; i++){ - assert( pIdx->aiColumn[i]nCol ); - if( pIdx->aiColumn[i]>=0 ) ai[pIdx->aiColumn[i]+1] = i+1; - } - sqlite3VdbeChangeP4(v, -1, (char*)ai, P4_INTARRAY); - } +static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ + pWC->a[iChild].iParent = iParent; + pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; + pWC->a[iParent].nChild++; +} + +/* +** Return the N-th AND-connected subterm of pTerm. Or if pTerm is not +** a conjunction, then return just pTerm when N==0. If N is exceeds +** the number of available subterms, return NULL. +*/ +static WhereTerm *whereNthSubterm(WhereTerm *pTerm, int N){ + if( pTerm->eOperator!=WO_AND ){ + return N==0 ? pTerm : 0; + } + if( Nu.pAndInfo->wc.nTerm ){ + return &pTerm->u.pAndInfo->wc.a[N]; } + return 0; } /* -** If the expression passed as the second argument is a vector, generate -** code to write the first nReg elements of the vector into an array -** of registers starting with iReg. +** Subterms pOne and pTwo are contained within WHERE clause pWC. The +** two subterms are in disjunction - they are OR-ed together. +** +** If these two terms are both of the form: "A op B" with the same +** A and B values but different operators and if the operators are +** compatible (if one is = and the other is <, for example) then +** add a new virtual AND term to pWC that is the combination of the +** two. +** +** Some examples: +** +** x x<=y +** x=y OR x=y --> x=y +** x<=y OR x x<=y +** +** The following is NOT generated: ** -** If the expression is not a vector, then nReg must be passed 1. In -** this case, generate code to evaluate the expression and leave the -** result in register iReg. +** xy --> x!=y */ -static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){ - assert( nReg>0 ); - if( sqlite3ExprIsVector(p) ){ -#ifndef SQLITE_OMIT_SUBQUERY - if( (p->flags & EP_xIsSelect) ){ - Vdbe *v = pParse->pVdbe; - int iSelect = sqlite3CodeSubselect(pParse, p, 0, 0); - sqlite3VdbeAddOp3(v, OP_Copy, iSelect, iReg, nReg-1); - }else -#endif - { - int i; - ExprList *pList = p->x.pList; - assert( nReg<=pList->nExpr ); - for(i=0; ia[i].pExpr, iReg+i); - } +static void whereCombineDisjuncts( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* The complete WHERE clause */ + WhereTerm *pOne, /* First disjunct */ + WhereTerm *pTwo /* Second disjunct */ +){ + u16 eOp = pOne->eOperator | pTwo->eOperator; + sqlite3 *db; /* Database connection (for malloc) */ + Expr *pNew; /* New virtual expression */ + int op; /* Operator for the combined expression */ + int idxNew; /* Index in pWC of the next virtual term */ + + if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; + if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; + if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp + && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; + assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); + assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); + if( sqlite3ExprCompare(pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; + if( sqlite3ExprCompare(pOne->pExpr->pRight, pTwo->pExpr->pRight, -1) )return; + /* If we reach this point, it means the two subterms can be combined */ + if( (eOp & (eOp-1))!=0 ){ + if( eOp & (WO_LT|WO_LE) ){ + eOp = WO_LE; + }else{ + assert( eOp & (WO_GT|WO_GE) ); + eOp = WO_GE; } - }else{ - assert( nReg==1 ); - sqlite3ExprCode(pParse, p, iReg); } + db = pWC->pWInfo->pParse->db; + pNew = sqlite3ExprDup(db, pOne->pExpr, 0); + if( pNew==0 ) return; + for(op=TK_EQ; eOp!=(WO_EQ<<(op-TK_EQ)); op++){ assert( opop = op; + idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); + exprAnalyze(pSrc, pWC, idxNew); } +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) /* -** Generate code for the start of the iLevel-th loop in the WHERE clause -** implementation described by pWInfo. +** Analyze a term that consists of two or more OR-connected +** subterms. So in: +** +** ... WHERE (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13) +** ^^^^^^^^^^^^^^^^^^^^ +** +** This routine analyzes terms such as the middle term in the above example. +** A WhereOrTerm object is computed and attached to the term under +** analysis, regardless of the outcome of the analysis. Hence: +** +** WhereTerm.wtFlags |= TERM_ORINFO +** WhereTerm.u.pOrInfo = a dynamically allocated WhereOrTerm object +** +** The term being analyzed must have two or more of OR-connected subterms. +** A single subterm might be a set of AND-connected sub-subterms. +** Examples of terms under analysis: +** +** (A) t1.x=t2.y OR t1.x=t2.z OR t1.y=15 OR t1.z=t3.a+5 +** (B) x=expr1 OR expr2=x OR x=expr3 +** (C) t1.x=t2.y OR (t1.x=t2.z AND t1.y=15) +** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') +** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) +** (F) x>A OR (x=A AND y>=B) +** +** CASE 1: +** +** If all subterms are of the form T.C=expr for some single column of C and +** a single table T (as shown in example B above) then create a new virtual +** term that is an equivalent IN expression. In other words, if the term +** being analyzed is: +** +** x = expr1 OR expr2 = x OR x = expr3 +** +** then create a new virtual term like this: +** +** x IN (expr1,expr2,expr3) +** +** CASE 2: +** +** If there are exactly two disjuncts one side has x>A and the other side +** has x=A (for the same x and A) then add a new virtual conjunct term to the +** WHERE clause of the form "x>=A". Example: +** +** x>A OR (x=A AND y>B) adds: x>=A +** +** The added conjunct can sometimes be helpful in query planning. +** +** CASE 3: +** +** If all subterms are indexable by a single table T, then set +** +** WhereTerm.eOperator = WO_OR +** WhereTerm.u.pOrInfo->indexable |= the cursor number for table T +** +** A subterm is "indexable" if it is of the form +** "T.C " where C is any column of table T and +** is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN". +** A subterm is also indexable if it is an AND of two or more +** subsubterms at least one of which is indexable. Indexable AND +** subterms have their eOperator set to WO_AND and they have +** u.pAndInfo set to a dynamically allocated WhereAndTerm object. +** +** From another point of view, "indexable" means that the subterm could +** potentially be used with an index if an appropriate index exists. +** This analysis does not consider whether or not the index exists; that +** is decided elsewhere. This analysis only looks at whether subterms +** appropriate for indexing exist. +** +** All examples A through E above satisfy case 2. But if a term +** also satisfies case 1 (such as B) we know that the optimizer will +** always prefer case 1, so in that case we pretend that case 2 is not +** satisfied. +** +** It might be the case that multiple tables are indexable. For example, +** (E) above is indexable on tables P, Q, and R. +** +** Terms that satisfy case 2 are candidates for lookup by using +** separate indices to find rowids for each subterm and composing +** the union of all rowids using a RowSet object. This is similar +** to "bitmap indices" in other database engines. +** +** OTHERWISE: +** +** If neither case 1 nor case 2 apply, then leave the eOperator set to +** zero. This term is not useful for search. */ -SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( - WhereInfo *pWInfo, /* Complete information about the WHERE clause */ - int iLevel, /* Which level of pWInfo->a[] should be coded */ - Bitmask notReady /* Which tables are currently available */ +static void exprAnalyzeOrTerm( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the complete WHERE clause */ + int idxTerm /* Index of the OR-term to be analyzed */ ){ - int j, k; /* Loop counters */ - int iCur; /* The VDBE cursor for the table */ - int addrNxt; /* Where to jump to continue with the next IN case */ - int omitTable; /* True if we use the index only */ - int bRev; /* True if we need to scan in reverse order */ - WhereLevel *pLevel; /* The where level to be coded */ - WhereLoop *pLoop; /* The WhereLoop object being coded */ - WhereClause *pWC; /* Decomposition of the entire WHERE clause */ - WhereTerm *pTerm; /* A WHERE clause term */ - Parse *pParse; /* Parsing context */ - sqlite3 *db; /* Database connection */ - Vdbe *v; /* The prepared stmt under constructions */ - struct SrcList_item *pTabItem; /* FROM clause term being coded */ - int addrBrk; /* Jump here to break out of the loop */ - int addrCont; /* Jump here to continue with next cycle */ - int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ - int iReleaseReg = 0; /* Temp register to free before returning */ - - pParse = pWInfo->pParse; - v = pParse->pVdbe; - pWC = &pWInfo->sWC; - db = pParse->db; - pLevel = &pWInfo->a[iLevel]; - pLoop = pLevel->pWLoop; - pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; - iCur = pTabItem->iCursor; - pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); - bRev = (pWInfo->revMask>>iLevel)&1; - omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 - && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0; - VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + Parse *pParse = pWInfo->pParse; /* Parser context */ + sqlite3 *db = pParse->db; /* Database connection */ + WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ + Expr *pExpr = pTerm->pExpr; /* The expression of the term */ + int i; /* Loop counters */ + WhereClause *pOrWc; /* Breakup of pTerm into subterms */ + WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ + WhereOrInfo *pOrInfo; /* Additional information associated with pTerm */ + Bitmask chngToIN; /* Tables that might satisfy case 1 */ + Bitmask indexable; /* Tables that are indexable, satisfying case 2 */ - /* Create labels for the "break" and "continue" instructions - ** for the current loop. Jump to addrBrk to break out of a loop. - ** Jump to cont to go immediately to the next iteration of the - ** loop. - ** - ** When there is an IN operator, we also have a "addrNxt" label that - ** means to continue with the next IN value combination. When - ** there are no IN operators in the constraints, the "addrNxt" label - ** is the same as "addrBrk". + /* + ** Break the OR clause into its separate subterms. The subterms are + ** stored in a WhereClause structure containing within the WhereOrInfo + ** object that is attached to the original OR clause term. */ - addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); + assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 ); + assert( pExpr->op==TK_OR ); + pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo)); + if( pOrInfo==0 ) return; + pTerm->wtFlags |= TERM_ORINFO; + pOrWc = &pOrInfo->wc; + whereClauseInit(pOrWc, pWInfo); + whereSplit(pOrWc, pExpr, TK_OR); + exprAnalyzeAll(pSrc, pOrWc); + if( db->mallocFailed ) return; + assert( pOrWc->nTerm>=2 ); - /* If this is the right table of a LEFT OUTER JOIN, allocate and - ** initialize a memory cell that records if this table matches any - ** row of the left table of the join. + /* + ** Compute the set of tables that might satisfy cases 1 or 2. */ - if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ - pLevel->iLeftJoin = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); - VdbeComment((v, "init LEFT JOIN no-match flag")); - } - - /* Special case of a FROM clause subquery implemented as a co-routine */ - if( pTabItem->fg.viaCoroutine ){ - int regYield = pTabItem->regReturn; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); - pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); - VdbeCoverage(v); - VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); - pLevel->op = OP_Goto; - }else - -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - /* Case 1: The table is a virtual-table. Use the VFilter and VNext - ** to access the data. - */ - int iReg; /* P3 Value for OP_VFilter */ - int addrNotFound; - int nConstraint = pLoop->nLTerm; - int iIn; /* Counter for IN constraints */ - - sqlite3ExprCachePush(pParse); - iReg = sqlite3GetTempRange(pParse, nConstraint+2); - addrNotFound = pLevel->addrBrk; - for(j=0; jaLTerm[j]; - if( NEVER(pTerm==0) ) continue; - if( pTerm->eOperator & WO_IN ){ - codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); - addrNotFound = pLevel->addrNxt; - }else{ - Expr *pRight = pTerm->pExpr->pRight; - codeExprOrVector(pParse, pRight, iTarget, 1); - } - } - sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); - sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); - sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, - pLoop->u.vtab.idxStr, - pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); - VdbeCoverage(v); - pLoop->u.vtab.needFree = 0; - pLevel->p1 = iCur; - pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext; - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - iIn = pLevel->u.in.nIn; - for(j=nConstraint-1; j>=0; j--){ - pTerm = pLoop->aLTerm[j]; - if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){ - disableTerm(pLevel, pTerm); - }else if( (pTerm->eOperator & WO_IN)!=0 ){ - Expr *pCompare; /* The comparison operator */ - Expr *pRight; /* RHS of the comparison */ - VdbeOp *pOp; /* Opcode to access the value of the IN constraint */ - - /* Reload the constraint value into reg[iReg+j+2]. The same value - ** was loaded into the same register prior to the OP_VFilter, but - ** the xFilter implementation might have changed the datatype or - ** encoding of the value in the register, so it *must* be reloaded. */ - assert( pLevel->u.in.aInLoop!=0 || db->mallocFailed ); + indexable = ~(Bitmask)0; + chngToIN = ~(Bitmask)0; + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ + if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ + WhereAndInfo *pAndInfo; + assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); + chngToIN = 0; + pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); + if( pAndInfo ){ + WhereClause *pAndWC; + WhereTerm *pAndTerm; + int j; + Bitmask b = 0; + pOrTerm->u.pAndInfo = pAndInfo; + pOrTerm->wtFlags |= TERM_ANDINFO; + pOrTerm->eOperator = WO_AND; + pAndWC = &pAndInfo->wc; + whereClauseInit(pAndWC, pWC->pWInfo); + whereSplit(pAndWC, pOrTerm->pExpr, TK_AND); + exprAnalyzeAll(pSrc, pAndWC); + pAndWC->pOuter = pWC; + testcase( db->mallocFailed ); if( !db->mallocFailed ){ - assert( iIn>0 ); - pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[--iIn].addrInTop); - assert( pOp->opcode==OP_Column || pOp->opcode==OP_Rowid ); - assert( pOp->opcode!=OP_Column || pOp->p3==iReg+j+2 ); - assert( pOp->opcode!=OP_Rowid || pOp->p2==iReg+j+2 ); - testcase( pOp->opcode==OP_Rowid ); - sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3); - } - - /* Generate code that will continue to the next row if - ** the IN constraint is not satisfied */ - pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0, 0); - assert( pCompare!=0 || db->mallocFailed ); - if( pCompare ){ - pCompare->pLeft = pTerm->pExpr->pLeft; - pCompare->pRight = pRight = sqlite3Expr(db, TK_REGISTER, 0); - if( pRight ){ - pRight->iTable = iReg+j+2; - sqlite3ExprIfFalse(pParse, pCompare, pLevel->addrCont, 0); + for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ + assert( pAndTerm->pExpr ); + if( allowedOp(pAndTerm->pExpr->op) ){ + b |= getMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); + } } - pCompare->pLeft = 0; - sqlite3ExprDelete(db, pCompare); } + indexable &= b; } - } - /* These registers need to be preserved in case there is an IN operator - ** loop. So we could deallocate the registers here (and potentially - ** reuse them later) if (pLoop->wsFlags & WHERE_IN_ABLE)==0. But it seems - ** simpler and safer to simply not reuse the registers. - ** - ** sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); - */ - sqlite3ExprCachePop(pParse); - }else -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - - if( (pLoop->wsFlags & WHERE_IPK)!=0 - && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 - ){ - /* Case 2: We can directly reference a single row using an - ** equality comparison against the ROWID field. Or - ** we reference multiple rows using a "rowid IN (...)" - ** construct. - */ - assert( pLoop->u.btree.nEq==1 ); - pTerm = pLoop->aLTerm[0]; - assert( pTerm!=0 ); - assert( pTerm->pExpr!=0 ); - assert( omitTable==0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - iReleaseReg = ++pParse->nMem; - iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); - if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); - addrNxt = pLevel->addrNxt; - sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg); - VdbeCoverage(v); - sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - VdbeComment((v, "pk")); - pLevel->op = OP_Noop; - }else if( (pLoop->wsFlags & WHERE_IPK)!=0 - && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 - ){ - /* Case 3: We have an inequality comparison against the ROWID field. - */ - int testOp = OP_Noop; - int start; - int memEndValue = 0; - WhereTerm *pStart, *pEnd; - - assert( omitTable==0 ); - j = 0; - pStart = pEnd = 0; - if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; - if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; - assert( pStart!=0 || pEnd!=0 ); - if( bRev ){ - pTerm = pStart; - pStart = pEnd; - pEnd = pTerm; - } - codeCursorHint(pTabItem, pWInfo, pLevel, pEnd); - if( pStart ){ - Expr *pX; /* The expression that defines the start bound */ - int r1, rTemp; /* Registers for holding the start boundary */ - int op; /* Cursor seek operation */ - - /* The following constant maps TK_xx codes into corresponding - ** seek opcodes. It depends on a particular ordering of TK_xx - */ - const u8 aMoveOp[] = { - /* TK_GT */ OP_SeekGT, - /* TK_LE */ OP_SeekLE, - /* TK_LT */ OP_SeekLT, - /* TK_GE */ OP_SeekGE - }; - assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ - assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ - assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ - - assert( (pStart->wtFlags & TERM_VNULL)==0 ); - testcase( pStart->wtFlags & TERM_VIRTUAL ); - pX = pStart->pExpr; - assert( pX!=0 ); - testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ - if( sqlite3ExprIsVector(pX->pRight) ){ - r1 = rTemp = sqlite3GetTempReg(pParse); - codeExprOrVector(pParse, pX->pRight, r1, 1); - op = aMoveOp[(pX->op - TK_GT) | 0x0001]; - }else{ - r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); - disableTerm(pLevel, pStart); - op = aMoveOp[(pX->op - TK_GT)]; - } - sqlite3VdbeAddOp3(v, op, iCur, addrBrk, r1); - VdbeComment((v, "pk")); - VdbeCoverageIf(v, pX->op==TK_GT); - VdbeCoverageIf(v, pX->op==TK_LE); - VdbeCoverageIf(v, pX->op==TK_LT); - VdbeCoverageIf(v, pX->op==TK_GE); - sqlite3ExprCacheAffinityChange(pParse, r1, 1); - sqlite3ReleaseTempReg(pParse, rTemp); + }else if( pOrTerm->wtFlags & TERM_COPIED ){ + /* Skip this term for now. We revisit it when we process the + ** corresponding TERM_VIRTUAL term */ }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - } - if( pEnd ){ - Expr *pX; - pX = pEnd->pExpr; - assert( pX!=0 ); - assert( (pEnd->wtFlags & TERM_VNULL)==0 ); - testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ - testcase( pEnd->wtFlags & TERM_VIRTUAL ); - memEndValue = ++pParse->nMem; - codeExprOrVector(pParse, pX->pRight, memEndValue, 1); - if( 0==sqlite3ExprIsVector(pX->pRight) - && (pX->op==TK_LT || pX->op==TK_GT) - ){ - testOp = bRev ? OP_Le : OP_Ge; - }else{ - testOp = bRev ? OP_Lt : OP_Gt; + Bitmask b; + b = getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); + if( pOrTerm->wtFlags & TERM_VIRTUAL ){ + WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; + b |= getMask(&pWInfo->sMaskSet, pOther->leftCursor); } - if( 0==sqlite3ExprIsVector(pX->pRight) ){ - disableTerm(pLevel, pEnd); + indexable &= b; + if( (pOrTerm->eOperator & WO_EQ)==0 ){ + chngToIN = 0; + }else{ + chngToIN &= b; } } - start = sqlite3VdbeCurrentAddr(v); - pLevel->op = bRev ? OP_Prev : OP_Next; - pLevel->p1 = iCur; - pLevel->p2 = start; - assert( pLevel->p5==0 ); - if( testOp!=OP_Noop ){ - iRowidReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); - VdbeCoverageIf(v, testOp==OP_Le); - VdbeCoverageIf(v, testOp==OP_Lt); - VdbeCoverageIf(v, testOp==OP_Ge); - VdbeCoverageIf(v, testOp==OP_Gt); - sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); - } - }else if( pLoop->wsFlags & WHERE_INDEXED ){ - /* Case 4: A scan using an index. - ** - ** The WHERE clause may contain zero or more equality - ** terms ("==" or "IN" operators) that refer to the N - ** left-most columns of the index. It may also contain - ** inequality constraints (>, <, >= or <=) on the indexed - ** column that immediately follows the N equalities. Only - ** the right-most column can be an inequality - the rest must - ** use the "==" and "IN" operators. For example, if the - ** index is on (x,y,z), then the following clauses are all - ** optimized: - ** - ** x=5 - ** x=5 AND y=10 - ** x=5 AND y<10 - ** x=5 AND y>5 AND y<10 - ** x=5 AND y=5 AND z<=10 - ** - ** The z<10 term of the following cannot be used, only - ** the x=5 term: - ** - ** x=5 AND z<10 - ** - ** N may be zero if there are inequality constraints. - ** If there are no inequality constraints, then N is at - ** least one. - ** - ** This case is also used when there are no WHERE clause - ** constraints but an index is selected anyway, in order - ** to force the output order to conform to an ORDER BY. - */ - static const u8 aStartOp[] = { - 0, - 0, - OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ - OP_Last, /* 3: (!start_constraints && startEq && bRev) */ - OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ - OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ - OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ - OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ - }; - static const u8 aEndOp[] = { - OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ - OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ - OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ - OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ - }; - u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ - u16 nBtm = pLoop->u.btree.nBtm; /* Length of BTM vector */ - u16 nTop = pLoop->u.btree.nTop; /* Length of TOP vector */ - int regBase; /* Base register holding constraint values */ - WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ - WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ - int startEq; /* True if range start uses ==, >= or <= */ - int endEq; /* True if range end uses ==, >= or <= */ - int start_constraints; /* Start of range is constrained */ - int nConstraint; /* Number of constraint terms */ - Index *pIdx; /* The index we will be using */ - int iIdxCur; /* The VDBE cursor for the index */ - int nExtraReg = 0; /* Number of extra registers needed */ - int op; /* Instruction opcode */ - char *zStartAff; /* Affinity for start of range constraint */ - char *zEndAff = 0; /* Affinity for end of range constraint */ - u8 bSeekPastNull = 0; /* True to seek past initial nulls */ - u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ + } - pIdx = pLoop->u.btree.pIndex; - iIdxCur = pLevel->iIdxCur; - assert( nEq>=pLoop->nSkip ); + /* + ** Record the set of tables that satisfy case 3. The set might be + ** empty. + */ + pOrInfo->indexable = indexable; + pTerm->eOperator = indexable==0 ? 0 : WO_OR; - /* If this loop satisfies a sort order (pOrderBy) request that - ** was passed to this function to implement a "SELECT min(x) ..." - ** query, then the caller will only allow the loop to run for - ** a single iteration. This means that the first row returned - ** should not have a NULL value stored in 'x'. If column 'x' is - ** the first one after the nEq equality constraints in the index, - ** this requires some special handling. - */ - assert( pWInfo->pOrderBy==0 - || pWInfo->pOrderBy->nExpr==1 - || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); - if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && pWInfo->nOBSat>0 - && (pIdx->nKeyCol>nEq) - ){ - assert( pLoop->nSkip==0 ); - bSeekPastNull = 1; - nExtraReg = 1; + /* For a two-way OR, attempt to implementation case 2. + */ + if( indexable && pOrWc->nTerm==2 ){ + int iOne = 0; + WhereTerm *pOne; + while( (pOne = whereNthSubterm(&pOrWc->a[0],iOne++))!=0 ){ + int iTwo = 0; + WhereTerm *pTwo; + while( (pTwo = whereNthSubterm(&pOrWc->a[1],iTwo++))!=0 ){ + whereCombineDisjuncts(pSrc, pWC, pOne, pTwo); + } } + } - /* Find any inequality constraint terms for the start and end - ** of the range. + /* + ** chngToIN holds a set of tables that *might* satisfy case 1. But + ** we have to do some additional checking to see if case 1 really + ** is satisfied. + ** + ** chngToIN will hold either 0, 1, or 2 bits. The 0-bit case means + ** that there is no possibility of transforming the OR clause into an + ** IN operator because one or more terms in the OR clause contain + ** something other than == on a column in the single table. The 1-bit + ** case means that every term of the OR clause is of the form + ** "table.column=expr" for some single table. The one bit that is set + ** will correspond to the common table. We still need to check to make + ** sure the same column is used on all terms. The 2-bit case is when + ** the all terms are of the form "table1.column=table2.column". It + ** might be possible to form an IN operator with either table1.column + ** or table2.column as the LHS if either is common to every term of + ** the OR clause. + ** + ** Note that terms of the form "table.column1=table.column2" (the + ** same table on both sizes of the ==) cannot be optimized. + */ + if( chngToIN ){ + int okToChngToIN = 0; /* True if the conversion to IN is valid */ + int iColumn = -1; /* Column index on lhs of IN operator */ + int iCursor = -1; /* Table cursor common to all terms */ + int j = 0; /* Loop counter */ + + /* Search for a table and column that appears on one side or the + ** other of the == operator in every subterm. That table and column + ** will be recorded in iCursor and iColumn. There might not be any + ** such table and column. Set okToChngToIN if an appropriate table + ** and column is found but leave okToChngToIN false if not found. */ - j = nEq; - if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ - pRangeStart = pLoop->aLTerm[j++]; - nExtraReg = MAX(nExtraReg, pLoop->u.btree.nBtm); - /* Like optimization range constraints always occur in pairs */ - assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || - (pLoop->wsFlags & WHERE_TOP_LIMIT)!=0 ); - } - if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ - pRangeEnd = pLoop->aLTerm[j++]; - nExtraReg = MAX(nExtraReg, pLoop->u.btree.nTop); -#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS - if( (pRangeEnd->wtFlags & TERM_LIKEOPT)!=0 ){ - assert( pRangeStart!=0 ); /* LIKE opt constraints */ - assert( pRangeStart->wtFlags & TERM_LIKEOPT ); /* occur in pairs */ - pLevel->iLikeRepCntr = (u32)++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 1, (int)pLevel->iLikeRepCntr); - VdbeComment((v, "LIKE loop counter")); - pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v); - /* iLikeRepCntr actually stores 2x the counter register number. The - ** bottom bit indicates whether the search order is ASC or DESC. */ - testcase( bRev ); - testcase( pIdx->aSortOrder[nEq]==SQLITE_SO_DESC ); - assert( (bRev & ~1)==0 ); - pLevel->iLikeRepCntr <<=1; - pLevel->iLikeRepCntr |= bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC); - } -#endif - if( pRangeStart==0 ){ - j = pIdx->aiColumn[nEq]; - if( (j>=0 && pIdx->pTable->aCol[j].notNull==0) || j==XN_EXPR ){ - bSeekPastNull = 1; + for(j=0; j<2 && !okToChngToIN; j++){ + pOrTerm = pOrWc->a; + for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ + assert( pOrTerm->eOperator & WO_EQ ); + pOrTerm->wtFlags &= ~TERM_OR_OK; + if( pOrTerm->leftCursor==iCursor ){ + /* This is the 2-bit case and we are on the second iteration and + ** current term is from the first iteration. So skip this term. */ + assert( j==1 ); + continue; + } + if( (chngToIN & getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor))==0 ){ + /* This term must be of the form t1.a==t2.b where t2 is in the + ** chngToIN set but t1 is not. This term will be either preceded + ** or follwed by an inverted copy (t2.b==t1.a). Skip this term + ** and use its inversion. */ + testcase( pOrTerm->wtFlags & TERM_COPIED ); + testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); + assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) ); + continue; } + iColumn = pOrTerm->u.leftColumn; + iCursor = pOrTerm->leftCursor; + break; } - } - assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); + if( i<0 ){ + /* No candidate table+column was found. This can only occur + ** on the second iteration */ + assert( j==1 ); + assert( IsPowerOfTwo(chngToIN) ); + assert( chngToIN==getMask(&pWInfo->sMaskSet, iCursor) ); + break; + } + testcase( j==1 ); - /* If we are doing a reverse order scan on an ascending index, or - ** a forward order scan on a descending index, interchange the - ** start and end terms (pRangeStart and pRangeEnd). - */ - if( (nEqnKeyCol && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) - || (bRev && pIdx->nKeyCol==nEq) - ){ - SWAP(WhereTerm *, pRangeEnd, pRangeStart); - SWAP(u8, bSeekPastNull, bStopAtNull); - SWAP(u8, nBtm, nTop); + /* We have found a candidate table and column. Check to see if that + ** table and column is common to every term in the OR clause */ + okToChngToIN = 1; + for(; i>=0 && okToChngToIN; i--, pOrTerm++){ + assert( pOrTerm->eOperator & WO_EQ ); + if( pOrTerm->leftCursor!=iCursor ){ + pOrTerm->wtFlags &= ~TERM_OR_OK; + }else if( pOrTerm->u.leftColumn!=iColumn ){ + okToChngToIN = 0; + }else{ + int affLeft, affRight; + /* If the right-hand side is also a column, then the affinities + ** of both right and left sides must be such that no type + ** conversions are required on the right. (Ticket #2249) + */ + affRight = sqlite3ExprAffinity(pOrTerm->pExpr->pRight); + affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft); + if( affRight!=0 && affRight!=affLeft ){ + okToChngToIN = 0; + }else{ + pOrTerm->wtFlags |= TERM_OR_OK; + } + } + } } - /* Generate code to evaluate all constraint terms using == or IN - ** and store the values of those terms in an array of registers - ** starting at regBase. + /* At this point, okToChngToIN is true if original pTerm satisfies + ** case 1. In that case, construct a new virtual term that is + ** pTerm converted into an IN operator. */ - codeCursorHint(pTabItem, pWInfo, pLevel, pRangeEnd); - regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); - assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); - if( zStartAff && nTop ){ - zEndAff = sqlite3DbStrDup(db, &zStartAff[nEq]); - } - addrNxt = pLevel->addrNxt; - - testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); - testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); - testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); - testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); - startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); - endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); - start_constraints = pRangeStart || nEq>0; + if( okToChngToIN ){ + Expr *pDup; /* A transient duplicate expression */ + ExprList *pList = 0; /* The RHS of the IN operator */ + Expr *pLeft = 0; /* The LHS of the IN operator */ + Expr *pNew; /* The complete IN operator */ - /* Seek the index cursor to the start of the range. */ - nConstraint = nEq; - if( pRangeStart ){ - Expr *pRight = pRangeStart->pExpr->pRight; - codeExprOrVector(pParse, pRight, regBase+nEq, nBtm); - whereLikeOptimizationStringFixup(v, pLevel, pRangeStart); - if( (pRangeStart->wtFlags & TERM_VNULL)==0 - && sqlite3ExprCanBeNull(pRight) - ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); - VdbeCoverage(v); + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ + if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; + assert( pOrTerm->eOperator & WO_EQ ); + assert( pOrTerm->leftCursor==iCursor ); + assert( pOrTerm->u.leftColumn==iColumn ); + pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); + pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup); + pLeft = pOrTerm->pExpr->pLeft; } - if( zStartAff ){ - updateRangeAffinityStr(pRight, nBtm, &zStartAff[nEq]); - } - nConstraint += nBtm; - testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); - if( sqlite3ExprIsVector(pRight)==0 ){ - disableTerm(pLevel, pRangeStart); + assert( pLeft!=0 ); + pDup = sqlite3ExprDup(db, pLeft, 0); + pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0); + if( pNew ){ + int idxNew; + transferJoinMarkings(pNew, pExpr); + assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + pNew->x.pList = pList; + idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + pTerm = &pWC->a[idxTerm]; + markTermAsChild(pWC, idxNew, idxTerm); }else{ - startEq = 1; + sqlite3ExprListDelete(db, pList); } - bSeekPastNull = 0; - }else if( bSeekPastNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - nConstraint++; - startEq = 0; - start_constraints = 1; + pTerm->eOperator = WO_NOOP; /* case 1 trumps case 3 */ } - codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); - if( pLoop->nSkip>0 && nConstraint==pLoop->nSkip ){ - /* The skip-scan logic inside the call to codeAllEqualityConstraints() - ** above has already left the cursor sitting on the correct row, - ** so no further seeking is needed */ + } +} +#endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ + +/* +** The input to this routine is an WhereTerm structure with only the +** "pExpr" field filled in. The job of this routine is to analyze the +** subexpression and populate all the other fields of the WhereTerm +** structure. +** +** If the expression is of the form " X" it gets commuted +** to the standard form of "X ". +** +** If the expression is of the form "X Y" where both X and Y are +** columns, then the original expression is unchanged and a new virtual +** term of the form "Y X" is added to the WHERE clause and +** analyzed separately. The original term is marked with TERM_COPIED +** and the new term is marked with TERM_DYNAMIC (because it's pExpr +** needs to be freed with the WhereClause) and TERM_VIRTUAL (because it +** is a commuted copy of a prior term.) The original term has nChild=1 +** and the copy has idxParent set to the index of the original term. +*/ +static void exprAnalyze( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the WHERE clause */ + int idxTerm /* Index of the term to be analyzed */ +){ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + WhereTerm *pTerm; /* The term to be analyzed */ + WhereMaskSet *pMaskSet; /* Set of table index masks */ + Expr *pExpr; /* The expression to be analyzed */ + Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ + Bitmask prereqAll; /* Prerequesites of pExpr */ + Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ + Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ + int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ + int noCase = 0; /* uppercase equivalent to lowercase */ + int op; /* Top-level operator. pExpr->op */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + sqlite3 *db = pParse->db; /* Database connection */ + + if( db->mallocFailed ){ + return; + } + pTerm = &pWC->a[idxTerm]; + pMaskSet = &pWInfo->sMaskSet; + pExpr = pTerm->pExpr; + assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); + prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); + op = pExpr->op; + if( op==TK_IN ){ + assert( pExpr->pRight==0 ); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + pTerm->prereqRight = exprSelectTableUsage(pMaskSet, pExpr->x.pSelect); }else{ - op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; - assert( op!=0 ); - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - VdbeCoverage(v); - VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); - VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); - VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); - VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); - VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); - VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); + pTerm->prereqRight = exprListTableUsage(pMaskSet, pExpr->x.pList); } - - /* Load the value for the inequality constraint at the end of the - ** range (if any). - */ - nConstraint = nEq; - if( pRangeEnd ){ - Expr *pRight = pRangeEnd->pExpr->pRight; - sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); - codeExprOrVector(pParse, pRight, regBase+nEq, nTop); - whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); - if( (pRangeEnd->wtFlags & TERM_VNULL)==0 - && sqlite3ExprCanBeNull(pRight) - ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); - VdbeCoverage(v); - } - if( zEndAff ){ - updateRangeAffinityStr(pRight, nTop, zEndAff); - codeApplyAffinity(pParse, regBase+nEq, nTop, zEndAff); + }else if( op==TK_ISNULL ){ + pTerm->prereqRight = 0; + }else{ + pTerm->prereqRight = exprTableUsage(pMaskSet, pExpr->pRight); + } + prereqAll = exprTableUsage(pMaskSet, pExpr); + if( ExprHasProperty(pExpr, EP_FromJoin) ){ + Bitmask x = getMask(pMaskSet, pExpr->iRightJoinTable); + prereqAll |= x; + extraRight = x-1; /* ON clause terms may not be used with an index + ** on left table of a LEFT JOIN. Ticket #3015 */ + } + pTerm->prereqAll = prereqAll; + pTerm->leftCursor = -1; + pTerm->iParent = -1; + pTerm->eOperator = 0; + if( allowedOp(op) ){ + Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); + Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); + u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; + if( pLeft->op==TK_COLUMN ){ + pTerm->leftCursor = pLeft->iTable; + pTerm->u.leftColumn = pLeft->iColumn; + pTerm->eOperator = operatorMask(op) & opMask; + } + if( pRight && pRight->op==TK_COLUMN ){ + WhereTerm *pNew; + Expr *pDup; + u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ + if( pTerm->leftCursor>=0 ){ + int idxNew; + pDup = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDup); + return; + } + idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); + if( idxNew==0 ) return; + pNew = &pWC->a[idxNew]; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + if( pExpr->op==TK_EQ + && !ExprHasProperty(pExpr, EP_FromJoin) + && OptimizationEnabled(db, SQLITE_Transitive) + ){ + pTerm->eOperator |= WO_EQUIV; + eExtraOp = WO_EQUIV; + } }else{ - assert( pParse->db->mallocFailed ); + pDup = pExpr; + pNew = pTerm; } - nConstraint += nTop; - testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); + exprCommute(pParse, pDup); + pLeft = sqlite3ExprSkipCollate(pDup->pLeft); + pNew->leftCursor = pLeft->iTable; + pNew->u.leftColumn = pLeft->iColumn; + testcase( (prereqLeft | extraRight) != prereqLeft ); + pNew->prereqRight = prereqLeft | extraRight; + pNew->prereqAll = prereqAll; + pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; + } + } - if( sqlite3ExprIsVector(pRight)==0 ){ - disableTerm(pLevel, pRangeEnd); - }else{ - endEq = 1; - } - }else if( bStopAtNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - endEq = 0; - nConstraint++; +#ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION + /* If a term is the BETWEEN operator, create two new virtual terms + ** that define the range that the BETWEEN implements. For example: + ** + ** a BETWEEN b AND c + ** + ** is converted into: + ** + ** (a BETWEEN b AND c) AND (a>=b) AND (a<=c) + ** + ** The two new terms are added onto the end of the WhereClause object. + ** The new terms are "dynamic" and are children of the original BETWEEN + ** term. That means that if the BETWEEN term is coded, the children are + ** skipped. Or, if the children are satisfied by an index, the original + ** BETWEEN term is skipped. + */ + else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){ + ExprList *pList = pExpr->x.pList; + int i; + static const u8 ops[] = {TK_GE, TK_LE}; + assert( pList!=0 ); + assert( pList->nExpr==2 ); + for(i=0; i<2; i++){ + Expr *pNewExpr; + int idxNew; + pNewExpr = sqlite3PExpr(pParse, ops[i], + sqlite3ExprDup(db, pExpr->pLeft, 0), + sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); + transferJoinMarkings(pNewExpr, pExpr); + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + pTerm = &pWC->a[idxTerm]; + markTermAsChild(pWC, idxNew, idxTerm); } - sqlite3DbFree(db, zStartAff); - sqlite3DbFree(db, zEndAff); + } +#endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ - /* Top of the loop body */ - pLevel->p2 = sqlite3VdbeCurrentAddr(v); +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) + /* Analyze a term that is composed of two or more subterms connected by + ** an OR operator. + */ + else if( pExpr->op==TK_OR ){ + assert( pWC->op==TK_AND ); + exprAnalyzeOrTerm(pSrc, pWC, idxTerm); + pTerm = &pWC->a[idxTerm]; + } +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ - /* Check if the index cursor is past the end of the range. */ - if( nConstraint ){ - op = aEndOp[bRev*2 + endEq]; - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); - testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); - testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); - testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); - } +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION + /* Add constraints to reduce the search space on a LIKE or GLOB + ** operator. + ** + ** A like pattern of the form "x LIKE 'aBc%'" is changed into constraints + ** + ** x>='ABC' AND x<'abd' AND x LIKE 'aBc%' + ** + ** The last character of the prefix "abc" is incremented to form the + ** termination condition "abd". If case is not significant (the default + ** for LIKE) then the lower-bound is made all uppercase and the upper- + ** bound is made all lowercase so that the bounds also work when comparing + ** BLOBs. + */ + if( pWC->op==TK_AND + && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) + ){ + Expr *pLeft; /* LHS of LIKE/GLOB operator */ + Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ + Expr *pNewExpr1; + Expr *pNewExpr2; + int idxNew1; + int idxNew2; + const char *zCollSeqName; /* Name of collating sequence */ + const u16 wtFlags = TERM_LIKEOPT | TERM_VIRTUAL | TERM_DYNAMIC; - /* Seek the table cursor, if required */ - if( omitTable ){ - /* pIdx is a covering index. No need to access the main table. */ - }else if( HasRowid(pIdx->pTable) ){ - if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE)!=0 ){ - iRowidReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg); - VdbeCoverage(v); - }else{ - codeDeferredSeek(pWInfo, pIdx, iCur, iIdxCur); - } - }else if( iCur!=iIdxCur ){ - Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); - iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); - for(j=0; jnKeyCol; j++){ - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); + pLeft = pExpr->x.pList->a[1].pExpr; + pStr2 = sqlite3ExprDup(db, pStr1, 0); + + /* Convert the lower bound to upper-case and the upper bound to + ** lower-case (upper-case is less than lower-case in ASCII) so that + ** the range constraints also work for BLOBs + */ + if( noCase && !pParse->db->mallocFailed ){ + int i; + char c; + pTerm->wtFlags |= TERM_LIKE; + for(i=0; (c = pStr1->u.zToken[i])!=0; i++){ + pStr1->u.zToken[i] = sqlite3Toupper(c); + pStr2->u.zToken[i] = sqlite3Tolower(c); } - sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, - iRowidReg, pPk->nKeyCol); VdbeCoverage(v); } - /* Record the instruction used to terminate the loop. */ - if( pLoop->wsFlags & WHERE_ONEROW ){ - pLevel->op = OP_Noop; - }else if( bRev ){ - pLevel->op = OP_Prev; - }else{ - pLevel->op = OP_Next; + if( !db->mallocFailed ){ + u8 c, *pC; /* Last character before the first wildcard */ + pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; + c = *pC; + if( noCase ){ + /* The point is to increment the last character before the first + ** wildcard. But if we increment '@', that will push it into the + ** alphabetic range where case conversions will mess up the + ** inequality. To avoid this, make sure to also run the full + ** LIKE on all candidate expressions by clearing the isComplete flag + */ + if( c=='A'-1 ) isComplete = 0; + c = sqlite3UpperToLower[c]; + } + *pC = c + 1; } - pLevel->p1 = iIdxCur; - pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; - if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; - }else{ - assert( pLevel->p5==0 ); + zCollSeqName = noCase ? "NOCASE" : "BINARY"; + pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); + pNewExpr1 = sqlite3PExpr(pParse, TK_GE, + sqlite3ExprAddCollateString(pParse,pNewExpr1,zCollSeqName), + pStr1, 0); + transferJoinMarkings(pNewExpr1, pExpr); + idxNew1 = whereClauseInsert(pWC, pNewExpr1, wtFlags); + testcase( idxNew1==0 ); + exprAnalyze(pSrc, pWC, idxNew1); + pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); + pNewExpr2 = sqlite3PExpr(pParse, TK_LT, + sqlite3ExprAddCollateString(pParse,pNewExpr2,zCollSeqName), + pStr2, 0); + transferJoinMarkings(pNewExpr2, pExpr); + idxNew2 = whereClauseInsert(pWC, pNewExpr2, wtFlags); + testcase( idxNew2==0 ); + exprAnalyze(pSrc, pWC, idxNew2); + pTerm = &pWC->a[idxTerm]; + if( isComplete ){ + markTermAsChild(pWC, idxNew1, idxTerm); + markTermAsChild(pWC, idxNew2, idxTerm); } - }else + } +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ -#ifndef SQLITE_OMIT_OR_OPTIMIZATION - if( pLoop->wsFlags & WHERE_MULTI_OR ){ - /* Case 5: Two or more separately indexed terms connected by OR - ** - ** Example: - ** - ** CREATE TABLE t1(a,b,c,d); - ** CREATE INDEX i1 ON t1(a); - ** CREATE INDEX i2 ON t1(b); - ** CREATE INDEX i3 ON t1(c); - ** - ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) - ** - ** In the example, there are three indexed terms connected by OR. - ** The top of the loop looks like this: - ** - ** Null 1 # Zero the rowset in reg 1 - ** - ** Then, for each indexed term, the following. The arguments to - ** RowSetTest are such that the rowid of the current row is inserted - ** into the RowSet. If it is already present, control skips the - ** Gosub opcode and jumps straight to the code generated by WhereEnd(). - ** - ** sqlite3WhereBegin() - ** RowSetTest # Insert rowid into rowset - ** Gosub 2 A - ** sqlite3WhereEnd() - ** - ** Following the above, code to terminate the loop. Label A, the target - ** of the Gosub above, jumps to the instruction right after the Goto. - ** - ** Null 1 # Zero the rowset in reg 1 - ** Goto B # The loop is finished. - ** - ** A: # Return data, whatever. - ** - ** Return 2 # Jump back to the Gosub - ** - ** B: - ** - ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then - ** use an ephemeral index instead of a RowSet to record the primary - ** keys of the rows we have already seen. - ** - */ - WhereClause *pOrWc; /* The OR-clause broken out into subterms */ - SrcList *pOrTab; /* Shortened table list or OR-clause generation */ - Index *pCov = 0; /* Potential covering index (or NULL) */ - int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ - - int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ - int regRowset = 0; /* Register for RowSet object */ - int regRowid = 0; /* Register holding rowid */ - int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ - int iRetInit; /* Address of regReturn init */ - int untestedTerms = 0; /* Some terms not completely tested */ - int ii; /* Loop counter */ - u16 wctrlFlags; /* Flags for sub-WHERE clause */ - Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ - Table *pTab = pTabItem->pTab; - - pTerm = pLoop->aLTerm[0]; - assert( pTerm!=0 ); - assert( pTerm->eOperator & WO_OR ); - assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); - pOrWc = &pTerm->u.pOrInfo->wc; - pLevel->op = OP_Return; - pLevel->p1 = regReturn; - - /* Set up a new SrcList in pOrTab containing the table being scanned - ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. - ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). - */ - if( pWInfo->nLevel>1 ){ - int nNotReady; /* The number of notReady tables */ - struct SrcList_item *origSrc; /* Original list of tables */ - nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3StackAllocRaw(db, - sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); - if( pOrTab==0 ) return notReady; - pOrTab->nAlloc = (u8)(nNotReady + 1); - pOrTab->nSrc = pOrTab->nAlloc; - memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); - origSrc = pWInfo->pTabList->a; - for(k=1; k<=nNotReady; k++){ - memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); - } - }else{ - pOrTab = pWInfo->pTabList; - } - - /* Initialize the rowset register to contain NULL. An SQL NULL is - ** equivalent to an empty rowset. Or, create an ephemeral index - ** capable of holding primary keys in the case of a WITHOUT ROWID. - ** - ** Also initialize regReturn to contain the address of the instruction - ** immediately following the OP_Return at the bottom of the loop. This - ** is required in a few obscure LEFT JOIN cases where control jumps - ** over the top of the loop into the body of it. In this case the - ** correct response for the end-of-loop code (the OP_Return) is to - ** fall through to the next instruction, just as an OP_Next does if - ** called on an uninitialized cursor. - */ - if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - if( HasRowid(pTab) ){ - regRowset = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); - }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pTab); - regRowset = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); - sqlite3VdbeSetP4KeyInfo(pParse, pPk); - } - regRowid = ++pParse->nMem; - } - iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Add a WO_MATCH auxiliary term to the constraint set if the + ** current expression is of the form: column MATCH expr. + ** This information is used by the xBestIndex methods of + ** virtual tables. The native query optimizer does not attempt + ** to do anything with MATCH functions. + */ + if( isMatchOfColumn(pExpr) ){ + int idxNew; + Expr *pRight, *pLeft; + WhereTerm *pNewTerm; + Bitmask prereqColumn, prereqExpr; - /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y - ** Then for every term xN, evaluate as the subexpression: xN AND z - ** That way, terms in y that are factored into the disjunction will - ** be picked up by the recursive calls to sqlite3WhereBegin() below. - ** - ** Actually, each subexpression is converted to "xN AND w" where w is - ** the "interesting" terms of z - terms that did not originate in the - ** ON or USING clause of a LEFT JOIN, and terms that are usable as - ** indices. - ** - ** This optimization also only applies if the (x1 OR x2 OR ...) term - ** is not contained in the ON clause of a LEFT JOIN. - ** See ticket http://www.sqlite.org/src/info/f2369304e4 - */ - if( pWC->nTerm>1 ){ - int iTerm; - for(iTerm=0; iTermnTerm; iTerm++){ - Expr *pExpr = pWC->a[iTerm].pExpr; - if( &pWC->a[iTerm] == pTerm ) continue; - if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); - testcase( pWC->a[iTerm].wtFlags & TERM_CODED ); - if( (pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_CODED))!=0 ) continue; - if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); - pExpr = sqlite3ExprDup(db, pExpr, 0); - pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); - } - if( pAndExpr ){ - pAndExpr = sqlite3PExpr(pParse, TK_AND|TKFLG_DONTFOLD, 0, pAndExpr, 0); - } + pRight = pExpr->x.pList->a[0].pExpr; + pLeft = pExpr->x.pList->a[1].pExpr; + prereqExpr = exprTableUsage(pMaskSet, pRight); + prereqColumn = exprTableUsage(pMaskSet, pLeft); + if( (prereqExpr & prereqColumn)==0 ){ + Expr *pNewExpr; + pNewExpr = sqlite3PExpr(pParse, TK_MATCH, + 0, sqlite3ExprDup(db, pRight, 0), 0); + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = prereqExpr; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_MATCH; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; } + } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ - /* Run a separate WHERE clause for each term of the OR clause. After - ** eliminating duplicates from other WHERE clauses, the action for each - ** sub-WHERE clause is to to invoke the main loop body as a subroutine. - */ - wctrlFlags = WHERE_OR_SUBCLAUSE | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE); - for(ii=0; iinTerm; ii++){ - WhereTerm *pOrTerm = &pOrWc->a[ii]; - if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ - WhereInfo *pSubWInfo; /* Info for single OR-term scan */ - Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ - int jmp1 = 0; /* Address of jump operation */ - if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ - pAndExpr->pLeft = pOrExpr; - pOrExpr = pAndExpr; - } - /* Loop through table entries that match term pOrTerm. */ - WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, - wctrlFlags, iCovCur); - assert( pSubWInfo || pParse->nErr || db->mallocFailed ); - if( pSubWInfo ){ - WhereLoop *pSubLoop; - int addrExplain = sqlite3WhereExplainOneScan( - pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 - ); - sqlite3WhereAddScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain); - - /* This is the sub-WHERE clause body. First skip over - ** duplicate rows from prior sub-WHERE clauses, and record the - ** rowid (or PRIMARY KEY) for the current row so that the same - ** row will be skipped in subsequent sub-WHERE clauses. - */ - if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - int r; - int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); - if( HasRowid(pTab) ){ - r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); - jmp1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, - r,iSet); - VdbeCoverage(v); - }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pTab); - int nPk = pPk->nKeyCol; - int iPk; - - /* Read the PK into an array of temp registers. */ - r = sqlite3GetTempRange(pParse, nPk); - for(iPk=0; iPkaiColumn[iPk]; - sqlite3ExprCodeGetColumnToReg(pParse, pTab, iCol, iCur, r+iPk); - } - - /* Check if the temp table already contains this key. If so, - ** the row has already been included in the result set and - ** can be ignored (by jumping past the Gosub below). Otherwise, - ** insert the key into the temp table and proceed with processing - ** the row. - ** - ** Use some of the same optimizations as OP_RowSetTest: If iSet - ** is zero, assume that the key cannot already be present in - ** the temp table. And if iSet is -1, assume that there is no - ** need to insert the key into the temp table, as it will never - ** be tested for. */ - if( iSet ){ - jmp1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); - VdbeCoverage(v); - } - if( iSet>=0 ){ - sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); - sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); - if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - } - - /* Release the array of temp registers */ - sqlite3ReleaseTempRange(pParse, r, nPk); - } - } - - /* Invoke the main loop body as a subroutine */ - sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); - - /* Jump here (skipping the main loop body subroutine) if the - ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ - if( jmp1 ) sqlite3VdbeJumpHere(v, jmp1); - - /* The pSubWInfo->untestedTerms flag means that this OR term - ** contained one or more AND term from a notReady table. The - ** terms from the notReady table could not be tested and will - ** need to be tested later. - */ - if( pSubWInfo->untestedTerms ) untestedTerms = 1; - - /* If all of the OR-connected terms are optimized using the same - ** index, and the index is opened using the same cursor number - ** by each call to sqlite3WhereBegin() made by this loop, it may - ** be possible to use that index as a covering index. - ** - ** If the call to sqlite3WhereBegin() above resulted in a scan that - ** uses an index, and this is either the first OR-connected term - ** processed or the index is the same as that used by all previous - ** terms, set pCov to the candidate covering index. Otherwise, set - ** pCov to NULL to indicate that no candidate covering index will - ** be available. - */ - pSubLoop = pSubWInfo->a[0].pWLoop; - assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); - if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 - && (ii==0 || pSubLoop->u.btree.pIndex==pCov) - && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) - ){ - assert( pSubWInfo->a[0].iIdxCur==iCovCur ); - pCov = pSubLoop->u.btree.pIndex; - }else{ - pCov = 0; - } - - /* Finish the loop through table entries that match term pOrTerm. */ - sqlite3WhereEnd(pSubWInfo); - } - } - } - pLevel->u.pCovidx = pCov; - if( pCov ) pLevel->iIdxCur = iCovCur; - if( pAndExpr ){ - pAndExpr->pLeft = 0; - sqlite3ExprDelete(db, pAndExpr); - } - sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); - sqlite3VdbeGoto(v, pLevel->addrBrk); - sqlite3VdbeResolveLabel(v, iLoopBody); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + /* When sqlite_stat3 histogram data is available an operator of the + ** form "x IS NOT NULL" can sometimes be evaluated more efficiently + ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a + ** virtual term of that form. + ** + ** Note that the virtual term must be tagged with TERM_VNULL. This + ** TERM_VNULL tag will suppress the not-null check at the beginning + ** of the loop. Without the TERM_VNULL flag, the not-null check at + ** the start of the loop will prevent any results from being returned. + */ + if( pExpr->op==TK_NOTNULL + && pExpr->pLeft->op==TK_COLUMN + && pExpr->pLeft->iColumn>=0 + && OptimizationEnabled(db, SQLITE_Stat34) + ){ + Expr *pNewExpr; + Expr *pLeft = pExpr->pLeft; + int idxNew; + WhereTerm *pNewTerm; - if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); - if( !untestedTerms ) disableTerm(pLevel, pTerm); - }else -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + pNewExpr = sqlite3PExpr(pParse, TK_GT, + sqlite3ExprDup(db, pLeft, 0), + sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0); - { - /* Case 6: There is no usable index. We must do a complete - ** scan of the entire table. - */ - static const u8 aStep[] = { OP_Next, OP_Prev }; - static const u8 aStart[] = { OP_Rewind, OP_Last }; - assert( bRev==0 || bRev==1 ); - if( pTabItem->fg.isRecursive ){ - /* Tables marked isRecursive have only a single row that is stored in - ** a pseudo-cursor. No need to Rewind or Next such cursors. */ - pLevel->op = OP_Noop; - }else{ - codeCursorHint(pTabItem, pWInfo, pLevel, 0); - pLevel->op = aStep[bRev]; - pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + idxNew = whereClauseInsert(pWC, pNewExpr, + TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); + if( idxNew ){ + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = 0; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_GT; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; } } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - pLevel->addrVisit = sqlite3VdbeCurrentAddr(v); -#endif - - /* Insert code to test every subexpression that can be completely - ** computed using the current set of tables. + /* Prevent ON clause terms of a LEFT JOIN from being used to drive + ** an index for tables to the left of the join. */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE; - int skipLikeAddr = 0; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ - testcase( pWInfo->untestedTerms==0 - && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ); - pWInfo->untestedTerms = 1; - continue; - } - pE = pTerm->pExpr; - assert( pE!=0 ); - if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ - continue; - } - if( pTerm->wtFlags & TERM_LIKECOND ){ - /* If the TERM_LIKECOND flag is set, that means that the range search - ** is sufficient to guarantee that the LIKE operator is true, so we - ** can skip the call to the like(A,B) function. But this only works - ** for strings. So do not skip the call to the function on the pass - ** that compares BLOBs. */ -#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS - continue; -#else - u32 x = pLevel->iLikeRepCntr; - assert( x>0 ); - skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)? OP_IfNot : OP_If, (int)(x>>1)); - VdbeCoverage(v); -#endif - } - sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); - if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); - pTerm->wtFlags |= TERM_CODED; - } + pTerm->prereqRight |= extraRight; +} - /* Insert code to test for implied constraints based on transitivity - ** of the "==" operator. - ** - ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" - ** and we are coding the t1 loop and the t2 loop has not yet coded, - ** then we cannot use the "t1.a=t2.b" constraint, but we can code - ** the implied "t1.a=123" constraint. - */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE, sEAlt; - WhereTerm *pAlt; - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) continue; - if( (pTerm->eOperator & WO_EQUIV)==0 ) continue; - if( pTerm->leftCursor!=iCur ) continue; - if( pLevel->iLeftJoin ) continue; - pE = pTerm->pExpr; - assert( !ExprHasProperty(pE, EP_FromJoin) ); - assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); - pAlt = sqlite3WhereFindTerm(pWC, iCur, pTerm->u.leftColumn, notReady, - WO_EQ|WO_IN|WO_IS, 0); - if( pAlt==0 ) continue; - if( pAlt->wtFlags & (TERM_CODED) ) continue; - testcase( pAlt->eOperator & WO_EQ ); - testcase( pAlt->eOperator & WO_IS ); - testcase( pAlt->eOperator & WO_IN ); - VdbeModuleComment((v, "begin transitive constraint")); - sEAlt = *pAlt->pExpr; - sEAlt.pLeft = pE->pLeft; - sqlite3ExprIfFalse(pParse, &sEAlt, addrCont, SQLITE_JUMPIFNULL); - } +/* +** This function searches pList for an entry that matches the iCol-th column +** of index pIdx. +** +** If such an expression is found, its index in pList->a[] is returned. If +** no expression is found, -1 is returned. +*/ +static int findIndexCol( + Parse *pParse, /* Parse context */ + ExprList *pList, /* Expression list to search */ + int iBase, /* Cursor for table associated with pIdx */ + Index *pIdx, /* Index to match column of */ + int iCol /* Column of index to match */ +){ + int i; + const char *zColl = pIdx->azColl[iCol]; - /* For a LEFT OUTER JOIN, generate code that will record the fact that - ** at least one row of the right table has matched the left table. - */ - if( pLevel->iLeftJoin ){ - pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); - VdbeComment((v, "record LEFT JOIN hit")); - sqlite3ExprCacheClear(pParse); - for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ - assert( pWInfo->untestedTerms ); - continue; + for(i=0; inExpr; i++){ + Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); + if( p->op==TK_COLUMN + && p->iColumn==pIdx->aiColumn[iCol] + && p->iTable==iBase + ){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); + if( pColl && 0==sqlite3StrICmp(pColl->zName, zColl) ){ + return i; } - assert( pTerm->pExpr ); - sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); - pTerm->wtFlags |= TERM_CODED; } } - return pLevel->notReady; + return -1; } -/************** End of wherecode.c *******************************************/ -/************** Begin file whereexpr.c ***************************************/ /* -** 2015-06-08 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This module contains C code that generates VDBE code used to process -** the WHERE clause of SQL statements. +** Return true if the DISTINCT expression-list passed as the third argument +** is redundant. ** -** This file was originally part of where.c but was split out to improve -** readability and editabiliity. This file contains utility routines for -** analyzing Expr objects in the WHERE clause. +** A DISTINCT list is redundant if the database contains some subset of +** columns that are unique and non-null. */ -/* #include "sqliteInt.h" */ -/* #include "whereInt.h" */ - -/* Forward declarations */ -static void exprAnalyze(SrcList*, WhereClause*, int); +static int isDistinctRedundant( + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* The FROM clause */ + WhereClause *pWC, /* The WHERE clause */ + ExprList *pDistinct /* The result set that needs to be DISTINCT */ +){ + Table *pTab; + Index *pIdx; + int i; + int iBase; -/* -** Deallocate all memory associated with a WhereOrInfo object. -*/ -static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ - sqlite3WhereClauseClear(&p->wc); - sqlite3DbFree(db, p); -} + /* If there is more than one table or sub-select in the FROM clause of + ** this query, then it will not be possible to show that the DISTINCT + ** clause is redundant. */ + if( pTabList->nSrc!=1 ) return 0; + iBase = pTabList->a[0].iCursor; + pTab = pTabList->a[0].pTab; -/* -** Deallocate all memory associated with a WhereAndInfo object. -*/ -static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ - sqlite3WhereClauseClear(&p->wc); - sqlite3DbFree(db, p); -} + /* If any of the expressions is an IPK column on table iBase, then return + ** true. Note: The (p->iTable==iBase) part of this test may be false if the + ** current SELECT is a correlated sub-query. + */ + for(i=0; inExpr; i++){ + Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); + if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; + } -/* -** Add a single new WhereTerm entry to the WhereClause object pWC. -** The new WhereTerm object is constructed from Expr p and with wtFlags. -** The index in pWC->a[] of the new WhereTerm is returned on success. -** 0 is returned if the new WhereTerm could not be added due to a memory -** allocation error. The memory allocation failure will be recorded in -** the db->mallocFailed flag so that higher-level functions can detect it. -** -** This routine will increase the size of the pWC->a[] array as necessary. -** -** If the wtFlags argument includes TERM_DYNAMIC, then responsibility -** for freeing the expression p is assumed by the WhereClause object pWC. -** This is true even if this routine fails to allocate a new WhereTerm. -** -** WARNING: This routine might reallocate the space used to store -** WhereTerms. All pointers to WhereTerms should be invalidated after -** calling this routine. Such pointers may be reinitialized by referencing -** the pWC->a[] array. -*/ -static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ - WhereTerm *pTerm; - int idx; - testcase( wtFlags & TERM_VIRTUAL ); - if( pWC->nTerm>=pWC->nSlot ){ - WhereTerm *pOld = pWC->a; - sqlite3 *db = pWC->pWInfo->pParse->db; - pWC->a = sqlite3DbMallocRawNN(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); - if( pWC->a==0 ){ - if( wtFlags & TERM_DYNAMIC ){ - sqlite3ExprDelete(db, p); + /* Loop through all indices on the table, checking each to see if it makes + ** the DISTINCT qualifier redundant. It does so if: + ** + ** 1. The index is itself UNIQUE, and + ** + ** 2. All of the columns in the index are either part of the pDistinct + ** list, or else the WHERE clause contains a term of the form "col=X", + ** where X is a constant value. The collation sequences of the + ** comparison and select-list expressions must match those of the index. + ** + ** 3. All of those index columns for which the WHERE clause does not + ** contain a "col=X" term are subject to a NOT NULL constraint. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !IsUniqueIndex(pIdx) ) continue; + for(i=0; inKeyCol; i++){ + i16 iCol = pIdx->aiColumn[i]; + if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ + int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i); + if( iIdxCol<0 || pTab->aCol[iCol].notNull==0 ){ + break; + } } - pWC->a = pOld; - return 0; } - memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); - if( pOld!=pWC->aStatic ){ - sqlite3DbFree(db, pOld); + if( i==pIdx->nKeyCol ){ + /* This index implies that the DISTINCT qualifier is redundant. */ + return 1; } - pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); - } - pTerm = &pWC->a[idx = pWC->nTerm++]; - if( p && ExprHasProperty(p, EP_Unlikely) ){ - pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; - }else{ - pTerm->truthProb = 1; } - pTerm->pExpr = sqlite3ExprSkipCollate(p); - pTerm->wtFlags = wtFlags; - pTerm->pWC = pWC; - pTerm->iParent = -1; - memset(&pTerm->eOperator, 0, - sizeof(WhereTerm) - offsetof(WhereTerm,eOperator)); - return idx; + + return 0; } + /* -** Return TRUE if the given operator is one of the operators that is -** allowed for an indexable WHERE clause term. The allowed operators are -** "=", "<", ">", "<=", ">=", "IN", "IS", and "IS NULL" +** Estimate the logarithm of the input value to base 2. */ -static int allowedOp(int op){ - assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL || op==TK_IS; +static LogEst estLog(LogEst N){ + return N<=10 ? 0 : sqlite3LogEst(N) - 33; } /* -** Commute a comparison operator. Expressions of the form "X op Y" -** are converted into "Y op X". -** -** If left/right precedence rules come into play when determining the -** collating sequence, then COLLATE operators are adjusted to ensure -** that the collating sequence does not change. For example: -** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on -** the left hand side of a comparison overrides any collation sequence -** attached to the right. For the same reason the EP_Collate flag -** is not commuted. +** Two routines for printing the content of an sqlite3_index_info +** structure. Used for testing and debugging only. If neither +** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines +** are no-ops. */ -static void exprCommute(Parse *pParse, Expr *pExpr){ - u16 expRight = (pExpr->pRight->flags & EP_Collate); - u16 expLeft = (pExpr->pLeft->flags & EP_Collate); - assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); - if( expRight==expLeft ){ - /* Either X and Y both have COLLATE operator or neither do */ - if( expRight ){ - /* Both X and Y have COLLATE operators. Make sure X is always - ** used by clearing the EP_Collate flag from Y. */ - pExpr->pRight->flags &= ~EP_Collate; - }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ - /* Neither X nor Y have COLLATE operators, but X has a non-default - ** collating sequence. So add the EP_Collate marker on X to cause - ** it to be searched first. */ - pExpr->pLeft->flags |= EP_Collate; - } +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) +static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", + i, + p->aConstraint[i].iColumn, + p->aConstraint[i].iTermOffset, + p->aConstraint[i].op, + p->aConstraint[i].usable); } - SWAP(Expr*,pExpr->pRight,pExpr->pLeft); - if( pExpr->op>=TK_GT ){ - assert( TK_LT==TK_GT+2 ); - assert( TK_GE==TK_LE+2 ); - assert( TK_GT>TK_EQ ); - assert( TK_GTop>=TK_GT && pExpr->op<=TK_GE ); - pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; + for(i=0; inOrderBy; i++){ + sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", + i, + p->aOrderBy[i].iColumn, + p->aOrderBy[i].desc); + } +} +static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", + i, + p->aConstraintUsage[i].argvIndex, + p->aConstraintUsage[i].omit); } + sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum); + sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); + sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); + sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); + sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } +#else +#define TRACE_IDX_INPUTS(A) +#define TRACE_IDX_OUTPUTS(A) +#endif +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* -** Translate from TK_xx operator to WO_xx bitmask. +** Return TRUE if the WHERE clause term pTerm is of a form where it +** could be used with an index to access pSrc, assuming an appropriate +** index existed. */ -static u16 operatorMask(int op){ - u16 c; - assert( allowedOp(op) ); - if( op==TK_IN ){ - c = WO_IN; - }else if( op==TK_ISNULL ){ - c = WO_ISNULL; - }else if( op==TK_IS ){ - c = WO_IS; - }else{ - assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); - c = (u16)(WO_EQ<<(op-TK_EQ)); - } - assert( op!=TK_ISNULL || c==WO_ISNULL ); - assert( op!=TK_IN || c==WO_IN ); - assert( op!=TK_EQ || c==WO_EQ ); - assert( op!=TK_LT || c==WO_LT ); - assert( op!=TK_LE || c==WO_LE ); - assert( op!=TK_GT || c==WO_GT ); - assert( op!=TK_GE || c==WO_GE ); - assert( op!=TK_IS || c==WO_IS ); - return c; +static int termCanDriveIndex( + WhereTerm *pTerm, /* WHERE clause term to check */ + struct SrcList_item *pSrc, /* Table we are trying to access */ + Bitmask notReady /* Tables in outer loops of the join */ +){ + char aff; + if( pTerm->leftCursor!=pSrc->iCursor ) return 0; + if( (pTerm->eOperator & WO_EQ)==0 ) return 0; + if( (pTerm->prereqRight & notReady)!=0 ) return 0; + if( pTerm->u.leftColumn<0 ) return 0; + aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; + if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; + return 1; } +#endif -#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* -** Check to see if the given expression is a LIKE or GLOB operator that -** can be optimized using inequality constraints. Return TRUE if it is -** so and false if not. -** -** In order for the operator to be optimizible, the RHS must be a string -** literal that does not begin with a wildcard. The LHS must be a column -** that may only be NULL, a string, or a BLOB, never a number. (This means -** that virtual tables cannot participate in the LIKE optimization.) The -** collating sequence for the column on the LHS must be appropriate for -** the operator. +** Generate code to construct the Index object for an automatic index +** and to set up the WhereLevel object pLevel so that the code generator +** makes use of the automatic index. */ -static int isLikeOrGlob( - Parse *pParse, /* Parsing and code generating context */ - Expr *pExpr, /* Test this expression */ - Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ - int *pisComplete, /* True if the only wildcard is % in the last character */ - int *pnoCase /* True if uppercase is equivalent to lowercase */ +static void constructAutomaticIndex( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause */ + struct SrcList_item *pSrc, /* The FROM clause term to get the next index */ + Bitmask notReady, /* Mask of cursors that are not available */ + WhereLevel *pLevel /* Write new index here */ ){ - const char *z = 0; /* String on RHS of LIKE operator */ - Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ - ExprList *pList; /* List of operands to the LIKE operator */ - int c; /* One character in z[] */ - int cnt; /* Number of non-wildcard prefix characters */ - char wc[3]; /* Wildcard characters */ - sqlite3 *db = pParse->db; /* Database connection */ - sqlite3_value *pVal = 0; - int op; /* Opcode of pRight */ - int rc; /* Result code to return */ + int nKeyCol; /* Number of columns in the constructed index */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + WhereTerm *pWCEnd; /* End of pWC->a[] */ + Index *pIdx; /* Object describing the transient index */ + Vdbe *v; /* Prepared statement under construction */ + int addrInit; /* Address of the initialization bypass jump */ + Table *pTable; /* The table being indexed */ + int addrTop; /* Top of the index fill loop */ + int regRecord; /* Register holding an index record */ + int n; /* Column counter */ + int i; /* Loop counter */ + int mxBitCol; /* Maximum column in pSrc->colUsed */ + CollSeq *pColl; /* Collating sequence to on a column */ + WhereLoop *pLoop; /* The Loop object */ + char *zNotUsed; /* Extra space on the end of pIdx */ + Bitmask idxCols; /* Bitmap of columns used for indexing */ + Bitmask extraCols; /* Bitmap of additional columns */ + u8 sentWarning = 0; /* True if a warnning has been issued */ + Expr *pPartial = 0; /* Partial Index Expression */ + int iContinue = 0; /* Jump here to skip excluded rows */ - if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ - return 0; + /* Generate code to skip over the creation and initialization of the + ** transient index on 2nd and subsequent iterations of the loop. */ + v = pParse->pVdbe; + assert( v!=0 ); + addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v); + + /* Count the number of columns that will be added to the index + ** and used to match WHERE clause constraints */ + nKeyCol = 0; + pTable = pSrc->pTab; + pWCEnd = &pWC->a[pWC->nTerm]; + pLoop = pLevel->pWLoop; + idxCols = 0; + for(pTerm=pWC->a; pTermpExpr; + assert( !ExprHasProperty(pExpr, EP_FromJoin) /* prereq always non-zero */ + || pExpr->iRightJoinTable!=pSrc->iCursor /* for the right-hand */ + || pLoop->prereq!=0 ); /* table of a LEFT JOIN */ + if( pLoop->prereq==0 + && (pTerm->wtFlags & TERM_VIRTUAL)==0 + && !ExprHasProperty(pExpr, EP_FromJoin) + && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){ + pPartial = sqlite3ExprAnd(pParse->db, pPartial, + sqlite3ExprDup(pParse->db, pExpr, 0)); + } + if( termCanDriveIndex(pTerm, pSrc, notReady) ){ + int iCol = pTerm->u.leftColumn; + Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS ); + testcase( iCol==BMS-1 ); + if( !sentWarning ){ + sqlite3_log(SQLITE_WARNING_AUTOINDEX, + "automatic index on %s(%s)", pTable->zName, + pTable->aCol[iCol].zName); + sentWarning = 1; + } + if( (idxCols & cMask)==0 ){ + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ + goto end_auto_index_create; + } + pLoop->aLTerm[nKeyCol++] = pTerm; + idxCols |= cMask; + } + } } -#ifdef SQLITE_EBCDIC - if( *pnoCase ) return 0; -#endif - pList = pExpr->x.pList; - pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN - || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT - || IsVirtual(pLeft->pTab) /* Value might be numeric */ - ){ - /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must - ** be the name of an indexed column with TEXT affinity. */ - return 0; + assert( nKeyCol>0 ); + pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; + pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED + | WHERE_AUTO_INDEX; + + /* Count the number of additional columns needed to create a + ** covering index. A "covering index" is an index that contains all + ** columns that are needed by the query. With a covering index, the + ** original table never needs to be accessed. Automatic indices must + ** be a covering index because the index will not be updated if the + ** original table changes and the index and table cannot both be used + ** if they go out of sync. + */ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + mxBitCol = MIN(BMS-1,pTable->nCol); + testcase( pTable->nCol==BMS-1 ); + testcase( pTable->nCol==BMS-2 ); + for(i=0; icolUsed & MASKBIT(BMS-1) ){ + nKeyCol += pTable->nCol - BMS + 1; } - assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ - pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); - op = pRight->op; - if( op==TK_VARIABLE ){ - Vdbe *pReprepare = pParse->pReprepare; - int iCol = pRight->iColumn; - pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_BLOB); - if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ - z = (char *)sqlite3_value_text(pVal); + /* Construct the Index object to describe this index */ + pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); + if( pIdx==0 ) goto end_auto_index_create; + pLoop->u.btree.pIndex = pIdx; + pIdx->zName = "auto-index"; + pIdx->pTable = pTable; + n = 0; + idxCols = 0; + for(pTerm=pWC->a; pTermu.leftColumn; + Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS-1 ); + testcase( iCol==BMS ); + if( (idxCols & cMask)==0 ){ + Expr *pX = pTerm->pExpr; + idxCols |= cMask; + pIdx->aiColumn[n] = pTerm->u.leftColumn; + pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + pIdx->azColl[n] = pColl ? pColl->zName : "BINARY"; + n++; + } } - sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); - assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); - }else if( op==TK_STRING ){ - z = pRight->u.zToken; } - if( z ){ - cnt = 0; - while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ - cnt++; + assert( (u32)n==pLoop->u.btree.nEq ); + + /* Add additional columns needed to make the automatic index into + ** a covering index */ + for(i=0; iaiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; } - if( cnt!=0 && 255!=(u8)z[cnt-1] ){ - Expr *pPrefix; - *pisComplete = c==wc[0] && z[cnt+1]==0; - pPrefix = sqlite3Expr(db, TK_STRING, z); - if( pPrefix ) pPrefix->u.zToken[cnt] = 0; - *ppPrefix = pPrefix; - if( op==TK_VARIABLE ){ - Vdbe *v = pParse->pVdbe; - sqlite3VdbeSetVarmask(v, pRight->iColumn); - if( *pisComplete && pRight->u.zToken[1] ){ - /* If the rhs of the LIKE expression is a variable, and the current - ** value of the variable means there is no need to invoke the LIKE - ** function, then no OP_Variable will be added to the program. - ** This causes problems for the sqlite3_bind_parameter_name() - ** API. To work around them, add a dummy OP_Variable here. - */ - int r1 = sqlite3GetTempReg(pParse); - sqlite3ExprCodeTarget(pParse, pRight, r1); - sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0); - sqlite3ReleaseTempReg(pParse, r1); - } - } - }else{ - z = 0; + } + if( pSrc->colUsed & MASKBIT(BMS-1) ){ + for(i=BMS-1; inCol; i++){ + pIdx->aiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; } } + assert( n==nKeyCol ); + pIdx->aiColumn[n] = -1; + pIdx->azColl[n] = "BINARY"; - rc = (z!=0); - sqlite3ValueFree(pVal); - return rc; -} -#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + /* Create the automatic index */ + assert( pLevel->iIdxCur>=0 ); + pLevel->iIdxCur = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "for %s", pTable->zName)); + + /* Fill the automatic index with content */ + sqlite3ExprCachePush(pParse); + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + if( pPartial ){ + iContinue = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + pLoop->wsFlags |= WHERE_PARTIALIDX; + } + regRecord = sqlite3GetTempReg(pParse); + sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); + sqlite3VdbeJumpHere(v, addrTop); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ExprCachePop(pParse); + + /* Jump here when skipping the initialization */ + sqlite3VdbeJumpHere(v, addrInit); +end_auto_index_create: + sqlite3ExprDelete(pParse->db, pPartial); +} +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ #ifndef SQLITE_OMIT_VIRTUALTABLE /* -** Check to see if the given expression is of the form -** -** column OP expr -** -** where OP is one of MATCH, GLOB, LIKE or REGEXP and "column" is a -** column of a virtual table. -** -** If it is then return TRUE. If not, return FALSE. +** Allocate and populate an sqlite3_index_info structure. It is the +** responsibility of the caller to eventually release the structure +** by passing the pointer returned by this function to sqlite3_free(). */ -static int isMatchOfColumn( - Expr *pExpr, /* Test this expression */ - unsigned char *peOp2 /* OUT: 0 for MATCH, or else an op2 value */ -){ - static const struct Op2 { - const char *zOp; - unsigned char eOp2; - } aOp[] = { - { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, - { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, - { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, - { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } - }; - ExprList *pList; - Expr *pCol; /* Column reference */ - int i; +static sqlite3_index_info *allocateIndexInfo( + Parse *pParse, + WhereClause *pWC, + struct SrcList_item *pSrc, + ExprList *pOrderBy +){ + int i, j; + int nTerm; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_orderby *pIdxOrderBy; + struct sqlite3_index_constraint_usage *pUsage; + WhereTerm *pTerm; + int nOrderBy; + sqlite3_index_info *pIdxInfo; - if( pExpr->op!=TK_FUNCTION ){ - return 0; + /* Count the number of possible WHERE clause constraints referring + ** to this virtual table */ + for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + if( pTerm->leftCursor != pSrc->iCursor ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; + nTerm++; } - pList = pExpr->x.pList; - if( pList==0 || pList->nExpr!=2 ){ - return 0; + + /* If the ORDER BY clause contains only columns in the current + ** virtual table then allocate space for the aOrderBy part of + ** the sqlite3_index_info structure. + */ + nOrderBy = 0; + if( pOrderBy ){ + int n = pOrderBy->nExpr; + for(i=0; ia[i].pExpr; + if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; + } + if( i==n){ + nOrderBy = n; + } } - pCol = pList->a[1].pExpr; - if( pCol->op!=TK_COLUMN || !IsVirtual(pCol->pTab) ){ + + /* Allocate the sqlite3_index_info structure + */ + pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) + + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm + + sizeof(*pIdxOrderBy)*nOrderBy ); + if( pIdxInfo==0 ){ + sqlite3ErrorMsg(pParse, "out of memory"); return 0; } - for(i=0; iu.zToken, aOp[i].zOp)==0 ){ - *peOp2 = aOp[i].eOp2; - return 1; - } + + /* Initialize the structure. The sqlite3_index_info structure contains + ** many fields that are declared "const" to prevent xBestIndex from + ** changing them. We have to do some funky casting in order to + ** initialize those fields. + */ + pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1]; + pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; + pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; + *(int*)&pIdxInfo->nConstraint = nTerm; + *(int*)&pIdxInfo->nOrderBy = nOrderBy; + *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons; + *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; + *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage = + pUsage; + + for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + u8 op; + if( pTerm->leftCursor != pSrc->iCursor ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; + pIdxCons[j].iColumn = pTerm->u.leftColumn; + pIdxCons[j].iTermOffset = i; + op = (u8)pTerm->eOperator & WO_ALL; + if( op==WO_IN ) op = WO_EQ; + pIdxCons[j].op = op; + /* The direct assignment in the previous line is possible only because + ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The + ** following asserts verify this fact. */ + assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); + assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); + assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); + assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); + assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); + assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); + assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + j++; } - return 0; + for(i=0; ia[i].pExpr; + pIdxOrderBy[i].iColumn = pExpr->iColumn; + pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; + } + + return pIdxInfo; } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ /* -** If the pBase expression originated in the ON or USING clause of -** a join, then transfer the appropriate markings over to derived. -*/ -static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ - if( pDerived ){ - pDerived->flags |= pBase->flags & EP_FromJoin; - pDerived->iRightJoinTable = pBase->iRightJoinTable; - } -} - -/* -** Mark term iChild as being a child of term iParent +** The table object reference passed as the second argument to this function +** must represent a virtual table. This function invokes the xBestIndex() +** method of the virtual table with the sqlite3_index_info object that +** comes in as the 3rd argument to this function. +** +** If an error occurs, pParse is populated with an error message and a +** non-zero value is returned. Otherwise, 0 is returned and the output +** part of the sqlite3_index_info structure is left populated. +** +** Whether or not an error is returned, it is the responsibility of the +** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates +** that this is required. */ -static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ - pWC->a[iChild].iParent = iParent; - pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; - pWC->a[iParent].nChild++; -} +static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ + sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; + int i; + int rc; -/* -** Return the N-th AND-connected subterm of pTerm. Or if pTerm is not -** a conjunction, then return just pTerm when N==0. If N is exceeds -** the number of available subterms, return NULL. -*/ -static WhereTerm *whereNthSubterm(WhereTerm *pTerm, int N){ - if( pTerm->eOperator!=WO_AND ){ - return N==0 ? pTerm : 0; + TRACE_IDX_INPUTS(p); + rc = pVtab->pModule->xBestIndex(pVtab, p); + TRACE_IDX_OUTPUTS(p); + + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ){ + pParse->db->mallocFailed = 1; + }else if( !pVtab->zErrMsg ){ + sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); + }else{ + sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); + } } - if( Nu.pAndInfo->wc.nTerm ){ - return &pTerm->u.pAndInfo->wc.a[N]; + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; + + for(i=0; inConstraint; i++){ + if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ + sqlite3ErrorMsg(pParse, + "table %s: xBestIndex returned an invalid plan", pTab->zName); + } } - return 0; + + return pParse->nErr; } +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* -** Subterms pOne and pTwo are contained within WHERE clause pWC. The -** two subterms are in disjunction - they are OR-ed together. -** -** If these two terms are both of the form: "A op B" with the same -** A and B values but different operators and if the operators are -** compatible (if one is = and the other is <, for example) then -** add a new virtual AND term to pWC that is the combination of the -** two. -** -** Some examples: -** -** x x<=y -** x=y OR x=y --> x=y -** x<=y OR x x<=y +** Estimate the location of a particular key among all keys in an +** index. Store the results in aStat as follows: ** -** The following is NOT generated: +** aStat[0] Est. number of rows less than pRec +** aStat[1] Est. number of rows equal to pRec ** -** xy --> x!=y +** Return the index of the sample that is the smallest sample that +** is greater than or equal to pRec. Note that this index is not an index +** into the aSample[] array - it is an index into a virtual set of samples +** based on the contents of aSample[] and the number of fields in record +** pRec. */ -static void whereCombineDisjuncts( - SrcList *pSrc, /* the FROM clause */ - WhereClause *pWC, /* The complete WHERE clause */ - WhereTerm *pOne, /* First disjunct */ - WhereTerm *pTwo /* Second disjunct */ +static int whereKeyStats( + Parse *pParse, /* Database connection */ + Index *pIdx, /* Index to consider domain of */ + UnpackedRecord *pRec, /* Vector of values to consider */ + int roundUp, /* Round up if true. Round down if false */ + tRowcnt *aStat /* OUT: stats written here */ ){ - u16 eOp = pOne->eOperator | pTwo->eOperator; - sqlite3 *db; /* Database connection (for malloc) */ - Expr *pNew; /* New virtual expression */ - int op; /* Operator for the combined expression */ - int idxNew; /* Index in pWC of the next virtual term */ + IndexSample *aSample = pIdx->aSample; + int iCol; /* Index of required stats in anEq[] etc. */ + int i; /* Index of first sample >= pRec */ + int iSample; /* Smallest sample larger than or equal to pRec */ + int iMin = 0; /* Smallest sample not yet tested */ + int iTest; /* Next sample to test */ + int res; /* Result of comparison operation */ + int nField; /* Number of fields in pRec */ + tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */ - if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; - if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; - if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp - && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; - assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); - assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); - if( sqlite3ExprCompare(pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; - if( sqlite3ExprCompare(pOne->pExpr->pRight, pTwo->pExpr->pRight, -1) )return; - /* If we reach this point, it means the two subterms can be combined */ - if( (eOp & (eOp-1))!=0 ){ - if( eOp & (WO_LT|WO_LE) ){ - eOp = WO_LE; +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER( pParse ); +#endif + assert( pRec!=0 ); + assert( pIdx->nSample>0 ); + assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol ); + + /* Do a binary search to find the first sample greater than or equal + ** to pRec. If pRec contains a single field, the set of samples to search + ** is simply the aSample[] array. If the samples in aSample[] contain more + ** than one fields, all fields following the first are ignored. + ** + ** If pRec contains N fields, where N is more than one, then as well as the + ** samples in aSample[] (truncated to N fields), the search also has to + ** consider prefixes of those samples. For example, if the set of samples + ** in aSample is: + ** + ** aSample[0] = (a, 5) + ** aSample[1] = (a, 10) + ** aSample[2] = (b, 5) + ** aSample[3] = (c, 100) + ** aSample[4] = (c, 105) + ** + ** Then the search space should ideally be the samples above and the + ** unique prefixes [a], [b] and [c]. But since that is hard to organize, + ** the code actually searches this set: + ** + ** 0: (a) + ** 1: (a, 5) + ** 2: (a, 10) + ** 3: (a, 10) + ** 4: (b) + ** 5: (b, 5) + ** 6: (c) + ** 7: (c, 100) + ** 8: (c, 105) + ** 9: (c, 105) + ** + ** For each sample in the aSample[] array, N samples are present in the + ** effective sample array. In the above, samples 0 and 1 are based on + ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc. + ** + ** Often, sample i of each block of N effective samples has (i+1) fields. + ** Except, each sample may be extended to ensure that it is greater than or + ** equal to the previous sample in the array. For example, in the above, + ** sample 2 is the first sample of a block of N samples, so at first it + ** appears that it should be 1 field in size. However, that would make it + ** smaller than sample 1, so the binary search would not work. As a result, + ** it is extended to two fields. The duplicates that this creates do not + ** cause any problems. + */ + nField = pRec->nField; + iCol = 0; + iSample = pIdx->nSample * nField; + do{ + int iSamp; /* Index in aSample[] of test sample */ + int n; /* Number of fields in test sample */ + + iTest = (iMin+iSample)/2; + iSamp = iTest / nField; + if( iSamp>0 ){ + /* The proposed effective sample is a prefix of sample aSample[iSamp]. + ** Specifically, the shortest prefix of at least (1 + iTest%nField) + ** fields that is greater than the previous effective sample. */ + for(n=(iTest % nField) + 1; nnField = n; + res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec); + if( res<0 ){ + iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1]; + iMin = iTest+1; + }else if( res==0 && ndb->mallocFailed==0 ){ + if( res==0 ){ + /* If (res==0) is true, then pRec must be equal to sample i. */ + assert( inSample ); + assert( iCol==nField-1 ); + pRec->nField = nField; + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) + || pParse->db->mallocFailed + ); + }else{ + /* Unless i==pIdx->nSample, indicating that pRec is larger than + ** all samples in the aSample[] array, pRec must be smaller than the + ** (iCol+1) field prefix of sample i. */ + assert( i<=pIdx->nSample && i>=0 ); + pRec->nField = iCol+1; + assert( i==pIdx->nSample + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 + || pParse->db->mallocFailed ); + + /* if i==0 and iCol==0, then record pRec is smaller than all samples + ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must + ** be greater than or equal to the (iCol) field prefix of sample i. + ** If (i>0), then pRec must also be greater than sample (i-1). */ + if( iCol>0 ){ + pRec->nField = iCol; + assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 + || pParse->db->mallocFailed ); + } + if( i>0 ){ + pRec->nField = nField; + assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 + || pParse->db->mallocFailed ); + } } } - db = pWC->pWInfo->pParse->db; - pNew = sqlite3ExprDup(db, pOne->pExpr, 0); - if( pNew==0 ) return; - for(op=TK_EQ; eOp!=(WO_EQ<<(op-TK_EQ)); op++){ assert( opop = op; - idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); - exprAnalyze(pSrc, pWC, idxNew); +#endif /* ifdef SQLITE_DEBUG */ + + if( res==0 ){ + /* Record pRec is equal to sample i */ + assert( iCol==nField-1 ); + aStat[0] = aSample[i].anLt[iCol]; + aStat[1] = aSample[i].anEq[iCol]; + }else{ + /* At this point, the (iCol+1) field prefix of aSample[i] is the first + ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec + ** is larger than all samples in the array. */ + tRowcnt iUpper, iGap; + if( i>=pIdx->nSample ){ + iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + }else{ + iUpper = aSample[i].anLt[iCol]; + } + + if( iLower>=iUpper ){ + iGap = 0; + }else{ + iGap = iUpper - iLower; + } + if( roundUp ){ + iGap = (iGap*2)/3; + }else{ + iGap = iGap/3; + } + aStat[0] = iLower + iGap; + aStat[1] = pIdx->aAvgEq[iCol]; + } + + /* Restore the pRec->nField value before returning. */ + pRec->nField = nField; + return i; } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ -#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) /* -** Analyze a term that consists of two or more OR-connected -** subterms. So in: -** -** ... WHERE (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13) -** ^^^^^^^^^^^^^^^^^^^^ -** -** This routine analyzes terms such as the middle term in the above example. -** A WhereOrTerm object is computed and attached to the term under -** analysis, regardless of the outcome of the analysis. Hence: -** -** WhereTerm.wtFlags |= TERM_ORINFO -** WhereTerm.u.pOrInfo = a dynamically allocated WhereOrTerm object -** -** The term being analyzed must have two or more of OR-connected subterms. -** A single subterm might be a set of AND-connected sub-subterms. -** Examples of terms under analysis: -** -** (A) t1.x=t2.y OR t1.x=t2.z OR t1.y=15 OR t1.z=t3.a+5 -** (B) x=expr1 OR expr2=x OR x=expr3 -** (C) t1.x=t2.y OR (t1.x=t2.z AND t1.y=15) -** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') -** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) -** (F) x>A OR (x=A AND y>=B) -** -** CASE 1: -** -** If all subterms are of the form T.C=expr for some single column of C and -** a single table T (as shown in example B above) then create a new virtual -** term that is an equivalent IN expression. In other words, if the term -** being analyzed is: -** -** x = expr1 OR expr2 = x OR x = expr3 +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. ** -** then create a new virtual term like this: +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. +*/ +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); + } + } + return nRet; +} + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** This function is called to estimate the number of rows visited by a +** range-scan on a skip-scan index. For example: ** -** x IN (expr1,expr2,expr3) +** CREATE INDEX i1 ON t1(a, b, c); +** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; ** -** CASE 2: +** Value pLoop->nOut is currently set to the estimated number of rows +** visited for scanning (a=? AND b=?). This function reduces that estimate +** by some factor to account for the (c BETWEEN ? AND ?) expression based +** on the stat4 data for the index. this scan will be peformed multiple +** times (once for each (a,b) combination that matches a=?) is dealt with +** by the caller. ** -** If there are exactly two disjuncts and one side has x>A and the other side -** has x=A (for the same x and A) then add a new virtual conjunct term to the -** WHERE clause of the form "x>=A". Example: +** It does this by scanning through all stat4 samples, comparing values +** extracted from pLower and pUpper with the corresponding column in each +** sample. If L and U are the number of samples found to be less than or +** equal to the values extracted from pLower and pUpper respectively, and +** N is the total number of samples, the pLoop->nOut value is adjusted +** as follows: ** -** x>A OR (x=A AND y>B) adds: x>=A +** nOut = nOut * ( min(U - L, 1) / N ) ** -** The added conjunct can sometimes be helpful in query planning. +** If pLower is NULL, or a value cannot be extracted from the term, L is +** set to zero. If pUpper is NULL, or a value cannot be extracted from it, +** U is set to N. ** -** CASE 3: +** Normally, this function sets *pbDone to 1 before returning. However, +** if no value can be extracted from either pLower or pUpper (and so the +** estimate of the number of rows delivered remains unchanged), *pbDone +** is left as is. ** -** If all subterms are indexable by a single table T, then set +** If an error occurs, an SQLite error code is returned. Otherwise, +** SQLITE_OK. +*/ +static int whereRangeSkipScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop, /* Update the .nOut value of this loop */ + int *pbDone /* Set to true if at least one expr. value extracted */ +){ + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + sqlite3 *db = pParse->db; + int nLower = -1; + int nUpper = p->nSample+1; + int rc = SQLITE_OK; + int iCol = p->aiColumn[nEq]; + u8 aff = iCol>=0 ? p->pTable->aCol[iCol].affinity : SQLITE_AFF_INTEGER; + CollSeq *pColl; + + sqlite3_value *p1 = 0; /* Value extracted from pLower */ + sqlite3_value *p2 = 0; /* Value extracted from pUpper */ + sqlite3_value *pVal = 0; /* Value extracted from record */ + + pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); + if( pLower ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); + nLower = 0; + } + if( pUpper && rc==SQLITE_OK ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); + nUpper = p2 ? 0 : p->nSample; + } + + if( p1 || p2 ){ + int i; + int nDiff; + for(i=0; rc==SQLITE_OK && inSample; i++){ + rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); + if( rc==SQLITE_OK && p1 ){ + int res = sqlite3MemCompare(p1, pVal, pColl); + if( res>=0 ) nLower++; + } + if( rc==SQLITE_OK && p2 ){ + int res = sqlite3MemCompare(p2, pVal, pColl); + if( res>=0 ) nUpper++; + } + } + nDiff = (nUpper - nLower); + if( nDiff<=0 ) nDiff = 1; + + /* If there is both an upper and lower bound specified, and the + ** comparisons indicate that they are close together, use the fallback + ** method (assume that the scan visits 1/64 of the rows) for estimating + ** the number of rows visited. Otherwise, estimate the number of rows + ** using the method described in the header comment for this function. */ + if( nDiff!=1 || pUpper==0 || pLower==0 ){ + int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); + pLoop->nOut -= nAdjust; + *pbDone = 1; + WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + nLower, nUpper, nAdjust*-1, pLoop->nOut)); + } + + }else{ + assert( *pbDone==0 ); + } + + sqlite3ValueFree(p1); + sqlite3ValueFree(p2); + sqlite3ValueFree(pVal); + + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +/* +** This function is used to estimate the number of rows that will be visited +** by scanning an index for a range of values. The range may have an upper +** bound, a lower bound, or both. The WHERE clause terms that set the upper +** and lower bounds are represented by pLower and pUpper respectively. For +** example, assuming that index p is on t1(a): ** -** WhereTerm.eOperator = WO_OR -** WhereTerm.u.pOrInfo->indexable |= the cursor number for table T +** ... FROM t1 WHERE a > ? AND a < ? ... +** |_____| |_____| +** | | +** pLower pUpper ** -** A subterm is "indexable" if it is of the form -** "T.C " where C is any column of table T and -** is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN". -** A subterm is also indexable if it is an AND of two or more -** subsubterms at least one of which is indexable. Indexable AND -** subterms have their eOperator set to WO_AND and they have -** u.pAndInfo set to a dynamically allocated WhereAndTerm object. +** If either of the upper or lower bound is not present, then NULL is passed in +** place of the corresponding WhereTerm. ** -** From another point of view, "indexable" means that the subterm could -** potentially be used with an index if an appropriate index exists. -** This analysis does not consider whether or not the index exists; that -** is decided elsewhere. This analysis only looks at whether subterms -** appropriate for indexing exist. +** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index +** column subject to the range constraint. Or, equivalently, the number of +** equality constraints optimized by the proposed index scan. For example, +** assuming index p is on t1(a, b), and the SQL query is: ** -** All examples A through E above satisfy case 3. But if a term -** also satisfies case 1 (such as B) we know that the optimizer will -** always prefer case 1, so in that case we pretend that case 3 is not -** satisfied. +** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... ** -** It might be the case that multiple tables are indexable. For example, -** (E) above is indexable on tables P, Q, and R. +** then nEq is set to 1 (as the range restricted column, b, is the second +** left-most column of the index). Or, if the query is: ** -** Terms that satisfy case 3 are candidates for lookup by using -** separate indices to find rowids for each subterm and composing -** the union of all rowids using a RowSet object. This is similar -** to "bitmap indices" in other database engines. +** ... FROM t1 WHERE a > ? AND a < ? ... ** -** OTHERWISE: +** then nEq is set to 0. ** -** If none of cases 1, 2, or 3 apply, then leave the eOperator set to -** zero. This term is not useful for search. +** When this function is called, *pnOut is set to the sqlite3LogEst() of the +** number of rows that the index scan is expected to visit without +** considering the range constraints. If nEq is 0, then *pnOut is the number of +** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) +** to account for the range constraints pLower and pUpper. +** +** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be +** used, a single range inequality reduces the search space by a factor of 4. +** and a pair of constraints (x>? AND x123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ ){ - WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ - Parse *pParse = pWInfo->pParse; /* Parser context */ - sqlite3 *db = pParse->db; /* Database connection */ - WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ - Expr *pExpr = pTerm->pExpr; /* The expression of the term */ - int i; /* Loop counters */ - WhereClause *pOrWc; /* Breakup of pTerm into subterms */ - WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ - WhereOrInfo *pOrInfo; /* Additional information associated with pTerm */ - Bitmask chngToIN; /* Tables that might satisfy case 1 */ - Bitmask indexable; /* Tables that are indexable, satisfying case 2 */ + int rc = SQLITE_OK; + int nOut = pLoop->nOut; + LogEst nNew; - /* - ** Break the OR clause into its separate subterms. The subterms are - ** stored in a WhereClause structure containing within the WhereOrInfo - ** object that is attached to the original OR clause term. - */ - assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 ); - assert( pExpr->op==TK_OR ); - pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo)); - if( pOrInfo==0 ) return; - pTerm->wtFlags |= TERM_ORINFO; - pOrWc = &pOrInfo->wc; - memset(pOrWc->aStatic, 0, sizeof(pOrWc->aStatic)); - sqlite3WhereClauseInit(pOrWc, pWInfo); - sqlite3WhereSplit(pOrWc, pExpr, TK_OR); - sqlite3WhereExprAnalyze(pSrc, pOrWc); - if( db->mallocFailed ) return; - assert( pOrWc->nTerm>=2 ); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; - /* - ** Compute the set of tables that might satisfy cases 1 or 3. - */ - indexable = ~(Bitmask)0; - chngToIN = ~(Bitmask)0; - for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ - if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ - WhereAndInfo *pAndInfo; - assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); - chngToIN = 0; - pAndInfo = sqlite3DbMallocRawNN(db, sizeof(*pAndInfo)); - if( pAndInfo ){ - WhereClause *pAndWC; - WhereTerm *pAndTerm; - int j; - Bitmask b = 0; - pOrTerm->u.pAndInfo = pAndInfo; - pOrTerm->wtFlags |= TERM_ANDINFO; - pOrTerm->eOperator = WO_AND; - pAndWC = &pAndInfo->wc; - memset(pAndWC->aStatic, 0, sizeof(pAndWC->aStatic)); - sqlite3WhereClauseInit(pAndWC, pWC->pWInfo); - sqlite3WhereSplit(pAndWC, pOrTerm->pExpr, TK_AND); - sqlite3WhereExprAnalyze(pSrc, pAndWC); - pAndWC->pOuter = pWC; - if( !db->mallocFailed ){ - for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ - assert( pAndTerm->pExpr ); - if( allowedOp(pAndTerm->pExpr->op) - || pAndTerm->eOperator==WO_MATCH - ){ - b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); - } - } - } - indexable &= b; + if( p->nSample>0 && nEqnSampleCol ){ + if( nEq==pBuilder->nRecValid ){ + UnpackedRecord *pRec = pBuilder->pRec; + tRowcnt a[2]; + u8 aff; + + /* Variable iLower will be set to the estimate of the number of rows in + ** the index that are less than the lower bound of the range query. The + ** lower bound being the concatenation of $P and $L, where $P is the + ** key-prefix formed by the nEq values matched against the nEq left-most + ** columns of the index, and $L is the value in pLower. + ** + ** Or, if pLower is NULL or $L cannot be extracted from it (because it + ** is not a simple variable or literal value), the lower bound of the + ** range is $P. Due to a quirk in the way whereKeyStats() works, even + ** if $L is available, whereKeyStats() is called for both ($P) and + ** ($P:$L) and the larger of the two returned values is used. + ** + ** Similarly, iUpper is to be set to the estimate of the number of rows + ** less than the upper bound of the range query. Where the upper bound + ** is either ($P) or ($P:$U). Again, even if $U is available, both values + ** of iUpper are requested of whereKeyStats() and the smaller used. + ** + ** The number of rows between the two bounds is then just iUpper-iLower. + */ + tRowcnt iLower; /* Rows less than the lower bound */ + tRowcnt iUpper; /* Rows less than the upper bound */ + int iLwrIdx = -2; /* aSample[] for the lower bound */ + int iUprIdx = -1; /* aSample[] for the upper bound */ + + if( pRec ){ + testcase( pRec->nField!=pBuilder->nRecValid ); + pRec->nField = pBuilder->nRecValid; } - }else if( pOrTerm->wtFlags & TERM_COPIED ){ - /* Skip this term for now. We revisit it when we process the - ** corresponding TERM_VIRTUAL term */ - }else{ - Bitmask b; - b = sqlite3WhereGetMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); - if( pOrTerm->wtFlags & TERM_VIRTUAL ){ - WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; - b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pOther->leftCursor); + if( nEq==p->nKeyCol ){ + aff = SQLITE_AFF_INTEGER; + }else{ + aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; } - indexable &= b; - if( (pOrTerm->eOperator & WO_EQ)==0 ){ - chngToIN = 0; + /* Determine iLower and iUpper using ($P) only. */ + if( nEq==0 ){ + iLower = 0; + iUpper = p->nRowEst0; }else{ - chngToIN &= b; + /* Note: this call could be optimized away - since the same values must + ** have been requested when testing key $P in whereEqualScanEst(). */ + whereKeyStats(pParse, p, pRec, 0, a); + iLower = a[0]; + iUpper = a[0] + a[1]; } - } - } - - /* - ** Record the set of tables that satisfy case 3. The set might be - ** empty. - */ - pOrInfo->indexable = indexable; - pTerm->eOperator = indexable==0 ? 0 : WO_OR; - /* For a two-way OR, attempt to implementation case 2. - */ - if( indexable && pOrWc->nTerm==2 ){ - int iOne = 0; - WhereTerm *pOne; - while( (pOne = whereNthSubterm(&pOrWc->a[0],iOne++))!=0 ){ - int iTwo = 0; - WhereTerm *pTwo; - while( (pTwo = whereNthSubterm(&pOrWc->a[1],iTwo++))!=0 ){ - whereCombineDisjuncts(pSrc, pWC, pOne, pTwo); + assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + assert( p->aSortOrder!=0 ); + if( p->aSortOrder[nEq] ){ + /* The roles of pLower and pUpper are swapped for a DESC index */ + SWAP(WhereTerm*, pLower, pUpper); } - } - } - - /* - ** chngToIN holds a set of tables that *might* satisfy case 1. But - ** we have to do some additional checking to see if case 1 really - ** is satisfied. - ** - ** chngToIN will hold either 0, 1, or 2 bits. The 0-bit case means - ** that there is no possibility of transforming the OR clause into an - ** IN operator because one or more terms in the OR clause contain - ** something other than == on a column in the single table. The 1-bit - ** case means that every term of the OR clause is of the form - ** "table.column=expr" for some single table. The one bit that is set - ** will correspond to the common table. We still need to check to make - ** sure the same column is used on all terms. The 2-bit case is when - ** the all terms are of the form "table1.column=table2.column". It - ** might be possible to form an IN operator with either table1.column - ** or table2.column as the LHS if either is common to every term of - ** the OR clause. - ** - ** Note that terms of the form "table.column1=table.column2" (the - ** same table on both sizes of the ==) cannot be optimized. - */ - if( chngToIN ){ - int okToChngToIN = 0; /* True if the conversion to IN is valid */ - int iColumn = -1; /* Column index on lhs of IN operator */ - int iCursor = -1; /* Table cursor common to all terms */ - int j = 0; /* Loop counter */ - /* Search for a table and column that appears on one side or the - ** other of the == operator in every subterm. That table and column - ** will be recorded in iCursor and iColumn. There might not be any - ** such table and column. Set okToChngToIN if an appropriate table - ** and column is found but leave okToChngToIN false if not found. - */ - for(j=0; j<2 && !okToChngToIN; j++){ - pOrTerm = pOrWc->a; - for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ - assert( pOrTerm->eOperator & WO_EQ ); - pOrTerm->wtFlags &= ~TERM_OR_OK; - if( pOrTerm->leftCursor==iCursor ){ - /* This is the 2-bit case and we are on the second iteration and - ** current term is from the first iteration. So skip this term. */ - assert( j==1 ); - continue; - } - if( (chngToIN & sqlite3WhereGetMask(&pWInfo->sMaskSet, - pOrTerm->leftCursor))==0 ){ - /* This term must be of the form t1.a==t2.b where t2 is in the - ** chngToIN set but t1 is not. This term will be either preceded - ** or follwed by an inverted copy (t2.b==t1.a). Skip this term - ** and use its inversion. */ - testcase( pOrTerm->wtFlags & TERM_COPIED ); - testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); - assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) ); - continue; + /* If possible, improve on the iLower estimate using ($P:$L). */ + if( pLower ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pLower->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); + iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + if( iNew>iLower ) iLower = iNew; + nOut--; + pLower = 0; } - iColumn = pOrTerm->u.leftColumn; - iCursor = pOrTerm->leftCursor; - break; - } - if( i<0 ){ - /* No candidate table+column was found. This can only occur - ** on the second iteration */ - assert( j==1 ); - assert( IsPowerOfTwo(chngToIN) ); - assert( chngToIN==sqlite3WhereGetMask(&pWInfo->sMaskSet, iCursor) ); - break; } - testcase( j==1 ); - /* We have found a candidate table and column. Check to see if that - ** table and column is common to every term in the OR clause */ - okToChngToIN = 1; - for(; i>=0 && okToChngToIN; i--, pOrTerm++){ - assert( pOrTerm->eOperator & WO_EQ ); - if( pOrTerm->leftCursor!=iCursor ){ - pOrTerm->wtFlags &= ~TERM_OR_OK; - }else if( pOrTerm->u.leftColumn!=iColumn ){ - okToChngToIN = 0; - }else{ - int affLeft, affRight; - /* If the right-hand side is also a column, then the affinities - ** of both right and left sides must be such that no type - ** conversions are required on the right. (Ticket #2249) - */ - affRight = sqlite3ExprAffinity(pOrTerm->pExpr->pRight); - affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft); - if( affRight!=0 && affRight!=affLeft ){ - okToChngToIN = 0; - }else{ - pOrTerm->wtFlags |= TERM_OR_OK; - } + /* If possible, improve on the iUpper estimate using ($P:$U). */ + if( pUpper ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pUpper->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); + iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + if( iNewnTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ - if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; - assert( pOrTerm->eOperator & WO_EQ ); - assert( pOrTerm->leftCursor==iCursor ); - assert( pOrTerm->u.leftColumn==iColumn ); - pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); - pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup); - pLeft = pOrTerm->pExpr->pLeft; - } - assert( pLeft!=0 ); - pDup = sqlite3ExprDup(db, pLeft, 0); - pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0); - if( pNew ){ - int idxNew; - transferJoinMarkings(pNew, pExpr); - assert( !ExprHasProperty(pNew, EP_xIsSelect) ); - pNew->x.pList = pList; - idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - exprAnalyze(pSrc, pWC, idxNew); - pTerm = &pWC->a[idxTerm]; - markTermAsChild(pWC, idxNew, idxTerm); - }else{ - sqlite3ExprListDelete(db, pList); + pBuilder->pRec = pRec; + if( rc==SQLITE_OK ){ + if( iUpper>iLower ){ + nNew = sqlite3LogEst(iUpper - iLower); + /* TUNING: If both iUpper and iLower are derived from the same + ** sample, then assume they are 4x more selective. This brings + ** the estimated selectivity more in line with what it would be + ** if estimated without the use of STAT3/4 tables. */ + if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); + }else{ + nNew = 10; assert( 10==sqlite3LogEst(2) ); + } + if( nNeweOperator = WO_NOOP; /* case 1 trumps case 3 */ + }else{ + int bDone = 0; + rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone); + if( bDone ) return rc; } } -} -#endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ +#else + UNUSED_PARAMETER(pParse); + UNUSED_PARAMETER(pBuilder); + assert( pLower || pUpper ); +#endif + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); -/* -** We already know that pExpr is a binary operator where both operands are -** column references. This routine checks to see if pExpr is an equivalence -** relation: -** 1. The SQLITE_Transitive optimization must be enabled -** 2. Must be either an == or an IS operator -** 3. Not originating in the ON clause of an OUTER JOIN -** 4. The affinities of A and B must be compatible -** 5a. Both operands use the same collating sequence OR -** 5b. The overall collating sequence is BINARY -** If this routine returns TRUE, that means that the RHS can be substituted -** for the LHS anyplace else in the WHERE clause where the LHS column occurs. -** This is an optimization. No harm comes from returning 0. But if 1 is -** returned when it should not be, then incorrect answers might result. -*/ -static int termIsEquivalence(Parse *pParse, Expr *pExpr){ - char aff1, aff2; - CollSeq *pColl; - const char *zColl1, *zColl2; - if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0; - if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0; - if( ExprHasProperty(pExpr, EP_FromJoin) ) return 0; - aff1 = sqlite3ExprAffinity(pExpr->pLeft); - aff2 = sqlite3ExprAffinity(pExpr->pRight); - if( aff1!=aff2 - && (!sqlite3IsNumericAffinity(aff1) || !sqlite3IsNumericAffinity(aff2)) - ){ - return 0; + /* TUNING: If there is both an upper and lower limit and neither limit + ** has an application-defined likelihood(), assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ + nNew -= 20; } - pColl = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight); - if( pColl==0 || sqlite3StrICmp(pColl->zName, "BINARY")==0 ) return 1; - pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); - zColl1 = pColl ? pColl->zName : 0; - pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight); - zColl2 = pColl ? pColl->zName : 0; - return sqlite3_stricmp(zColl1, zColl2)==0; -} -/* -** Recursively walk the expressions of a SELECT statement and generate -** a bitmask indicating which tables are used in that expression -** tree. -*/ -static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ - Bitmask mask = 0; - while( pS ){ - SrcList *pSrc = pS->pSrc; - mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pEList); - mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pGroupBy); - mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pOrderBy); - mask |= sqlite3WhereExprUsage(pMaskSet, pS->pWhere); - mask |= sqlite3WhereExprUsage(pMaskSet, pS->pHaving); - if( ALWAYS(pSrc!=0) ){ - int i; - for(i=0; inSrc; i++){ - mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); - mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].pOn); - } - } - pS = pS->pPrior; + nOut -= (pLower!=0) + (pUpper!=0); + if( nNew<10 ) nNew = 10; + if( nNewnOut>nOut ){ + WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + pLoop->nOut, nOut)); } - return mask; +#endif + pLoop->nOut = (LogEst)nOut; + return rc; } +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* -** Expression pExpr is one operand of a comparison operator that might -** be useful for indexing. This routine checks to see if pExpr appears -** in any index. Return TRUE (1) if pExpr is an indexed term and return -** FALSE (0) if not. If TRUE is returned, also set *piCur to the cursor -** number of the table that is indexed and *piColumn to the column number -** of the column that is indexed, or XN_EXPR (-2) if an expression is being -** indexed. +** Estimate the number of rows that will be returned based on +** an equality constraint x=VALUE and where that VALUE occurs in +** the histogram data. This only works when x is the left-most +** column of an index and sqlite_stat3 histogram data is available +** for that index. When pExpr==NULL that means the constraint is +** "x IS NULL" instead of "x=VALUE". +** +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. ** -** If pExpr is a TK_COLUMN column reference, then this routine always returns -** true even if that particular column is not indexed, because the column -** might be added to an automatic index later. +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. */ -static int exprMightBeIndexed( - SrcList *pFrom, /* The FROM clause */ - int op, /* The specific comparison operator */ - Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ - Expr *pExpr, /* An operand of a comparison operator */ - int *piCur, /* Write the referenced table cursor number here */ - int *piColumn /* Write the referenced table column number here */ +static int whereEqualScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereLoopBuilder *pBuilder, + Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ + tRowcnt *pnRow /* Write the revised row estimate here */ ){ - Index *pIdx; - int i; - int iCur; + Index *p = pBuilder->pNew->u.btree.pIndex; + int nEq = pBuilder->pNew->u.btree.nEq; + UnpackedRecord *pRec = pBuilder->pRec; + u8 aff; /* Column affinity */ + int rc; /* Subfunction return code */ + tRowcnt a[2]; /* Statistics */ + int bOk; - /* If this expression is a vector to the left or right of a - ** inequality constraint (>, <, >= or <=), perform the processing - ** on the first element of the vector. */ - assert( TK_GT+1==TK_LE && TK_GT+2==TK_LT && TK_GT+3==TK_GE ); - assert( TK_ISop==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){ - pExpr = pExpr->x.pList->a[0].pExpr; - } + assert( nEq>=1 ); + assert( nEq<=p->nColumn ); + assert( p->aSample!=0 ); + assert( p->nSample>0 ); + assert( pBuilder->nRecValidop==TK_COLUMN ){ - *piCur = pExpr->iTable; - *piColumn = pExpr->iColumn; - return 1; + /* If values are not available for all fields of the index to the left + ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */ + if( pBuilder->nRecValid<(nEq-1) ){ + return SQLITE_NOTFOUND; } - if( mPrereq==0 ) return 0; /* No table references */ - if( (mPrereq&(mPrereq-1))!=0 ) return 0; /* Refs more than one table */ - for(i=0; mPrereq>1; i++, mPrereq>>=1){} - iCur = pFrom->a[i].iCursor; - for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->aColExpr==0 ) continue; - for(i=0; inKeyCol; i++){ - if( pIdx->aiColumn[i]!=XN_EXPR ) continue; - if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ - *piCur = iCur; - *piColumn = XN_EXPR; - return 1; - } - } + + /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() + ** below would return the same value. */ + if( nEq>=p->nColumn ){ + *pnRow = 1; + return SQLITE_OK; } - return 0; + + aff = p->pTable->aCol[p->aiColumn[nEq-1]].affinity; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); + pBuilder->pRec = pRec; + if( rc!=SQLITE_OK ) return rc; + if( bOk==0 ) return SQLITE_NOTFOUND; + pBuilder->nRecValid = nEq; + + whereKeyStats(pParse, p, pRec, 0, a); + WHERETRACE(0x10,("equality scan regions: %d\n", (int)a[1])); + *pnRow = a[1]; + + return rc; } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* -** The input to this routine is an WhereTerm structure with only the -** "pExpr" field filled in. The job of this routine is to analyze the -** subexpression and populate all the other fields of the WhereTerm -** structure. +** Estimate the number of rows that will be returned based on +** an IN constraint where the right-hand side of the IN operator +** is a list of values. Example: ** -** If the expression is of the form " X" it gets commuted -** to the standard form of "X ". +** WHERE x IN (1,2,3,4) ** -** If the expression is of the form "X Y" where both X and Y are -** columns, then the original expression is unchanged and a new virtual -** term of the form "Y X" is added to the WHERE clause and -** analyzed separately. The original term is marked with TERM_COPIED -** and the new term is marked with TERM_DYNAMIC (because it's pExpr -** needs to be freed with the WhereClause) and TERM_VIRTUAL (because it -** is a commuted copy of a prior term.) The original term has nChild=1 -** and the copy has idxParent set to the index of the original term. +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. +** +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. */ -static void exprAnalyze( - SrcList *pSrc, /* the FROM clause */ - WhereClause *pWC, /* the WHERE clause */ - int idxTerm /* Index of the term to be analyzed */ +static int whereInScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereLoopBuilder *pBuilder, + ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ + tRowcnt *pnRow /* Write the revised row estimate here */ ){ - WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ - WhereTerm *pTerm; /* The term to be analyzed */ - WhereMaskSet *pMaskSet; /* Set of table index masks */ - Expr *pExpr; /* The expression to be analyzed */ - Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ - Bitmask prereqAll; /* Prerequesites of pExpr */ - Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ - Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ - int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ - int noCase = 0; /* uppercase equivalent to lowercase */ - int op; /* Top-level operator. pExpr->op */ - Parse *pParse = pWInfo->pParse; /* Parsing context */ - sqlite3 *db = pParse->db; /* Database connection */ - unsigned char eOp2; /* op2 value for LIKE/REGEXP/GLOB */ + Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); + int nRecValid = pBuilder->nRecValid; + int rc = SQLITE_OK; /* Subfunction return code */ + tRowcnt nEst; /* Number of rows for a single term */ + tRowcnt nRowEst = 0; /* New estimate of the number of rows */ + int i; /* Loop counter */ - if( db->mallocFailed ){ - return; - } - pTerm = &pWC->a[idxTerm]; - pMaskSet = &pWInfo->sMaskSet; - pExpr = pTerm->pExpr; - assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); - prereqLeft = sqlite3WhereExprUsage(pMaskSet, pExpr->pLeft); - op = pExpr->op; - if( op==TK_IN ){ - assert( pExpr->pRight==0 ); - if( sqlite3ExprCheckIN(pParse, pExpr) ) return; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - pTerm->prereqRight = exprSelectUsage(pMaskSet, pExpr->x.pSelect); - }else{ - pTerm->prereqRight = sqlite3WhereExprListUsage(pMaskSet, pExpr->x.pList); - } - }else if( op==TK_ISNULL ){ - pTerm->prereqRight = 0; - }else{ - pTerm->prereqRight = sqlite3WhereExprUsage(pMaskSet, pExpr->pRight); + assert( p->aSample!=0 ); + for(i=0; rc==SQLITE_OK && inExpr; i++){ + nEst = nRow0; + rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); + nRowEst += nEst; + pBuilder->nRecValid = nRecValid; } - prereqAll = sqlite3WhereExprUsage(pMaskSet, pExpr); - if( ExprHasProperty(pExpr, EP_FromJoin) ){ - Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->iRightJoinTable); - prereqAll |= x; - extraRight = x-1; /* ON clause terms may not be used with an index - ** on left table of a LEFT JOIN. Ticket #3015 */ + + if( rc==SQLITE_OK ){ + if( nRowEst > nRow0 ) nRowEst = nRow0; + *pnRow = nRowEst; + WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); } - pTerm->prereqAll = prereqAll; - pTerm->leftCursor = -1; - pTerm->iParent = -1; - pTerm->eOperator = 0; - if( allowedOp(op) ){ - int iCur, iColumn; - Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); - Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); - u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; - - if( pTerm->iField>0 ){ - assert( op==TK_IN ); - assert( pLeft->op==TK_VECTOR ); - pLeft = pLeft->x.pList->a[pTerm->iField-1].pExpr; - } - - if( exprMightBeIndexed(pSrc, op, prereqLeft, pLeft, &iCur, &iColumn) ){ - pTerm->leftCursor = iCur; - pTerm->u.leftColumn = iColumn; - pTerm->eOperator = operatorMask(op) & opMask; - } - if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; - if( pRight - && exprMightBeIndexed(pSrc, op, pTerm->prereqRight, pRight, &iCur,&iColumn) - ){ - WhereTerm *pNew; - Expr *pDup; - u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ - assert( pTerm->iField==0 ); - if( pTerm->leftCursor>=0 ){ - int idxNew; - pDup = sqlite3ExprDup(db, pExpr, 0); - if( db->mallocFailed ){ - sqlite3ExprDelete(db, pDup); - return; - } - idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); - if( idxNew==0 ) return; - pNew = &pWC->a[idxNew]; - markTermAsChild(pWC, idxNew, idxTerm); - if( op==TK_IS ) pNew->wtFlags |= TERM_IS; - pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags |= TERM_COPIED; - - if( termIsEquivalence(pParse, pDup) ){ - pTerm->eOperator |= WO_EQUIV; - eExtraOp = WO_EQUIV; - } - }else{ - pDup = pExpr; - pNew = pTerm; - } - exprCommute(pParse, pDup); - pNew->leftCursor = iCur; - pNew->u.leftColumn = iColumn; - testcase( (prereqLeft | extraRight) != prereqLeft ); - pNew->prereqRight = prereqLeft | extraRight; - pNew->prereqAll = prereqAll; - pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; - } - } - -#ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION - /* If a term is the BETWEEN operator, create two new virtual terms - ** that define the range that the BETWEEN implements. For example: - ** - ** a BETWEEN b AND c - ** - ** is converted into: - ** - ** (a BETWEEN b AND c) AND (a>=b) AND (a<=c) - ** - ** The two new terms are added onto the end of the WhereClause object. - ** The new terms are "dynamic" and are children of the original BETWEEN - ** term. That means that if the BETWEEN term is coded, the children are - ** skipped. Or, if the children are satisfied by an index, the original - ** BETWEEN term is skipped. - */ - else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){ - ExprList *pList = pExpr->x.pList; - int i; - static const u8 ops[] = {TK_GE, TK_LE}; - assert( pList!=0 ); - assert( pList->nExpr==2 ); - for(i=0; i<2; i++){ - Expr *pNewExpr; - int idxNew; - pNewExpr = sqlite3PExpr(pParse, ops[i], - sqlite3ExprDup(db, pExpr->pLeft, 0), - sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); - transferJoinMarkings(pNewExpr, pExpr); - idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - exprAnalyze(pSrc, pWC, idxNew); - pTerm = &pWC->a[idxTerm]; - markTermAsChild(pWC, idxNew, idxTerm); - } - } -#endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ - -#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) - /* Analyze a term that is composed of two or more subterms connected by - ** an OR operator. - */ - else if( pExpr->op==TK_OR ){ - assert( pWC->op==TK_AND ); - exprAnalyzeOrTerm(pSrc, pWC, idxTerm); - pTerm = &pWC->a[idxTerm]; - } -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ - -#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION - /* Add constraints to reduce the search space on a LIKE or GLOB - ** operator. - ** - ** A like pattern of the form "x LIKE 'aBc%'" is changed into constraints - ** - ** x>='ABC' AND x<'abd' AND x LIKE 'aBc%' - ** - ** The last character of the prefix "abc" is incremented to form the - ** termination condition "abd". If case is not significant (the default - ** for LIKE) then the lower-bound is made all uppercase and the upper- - ** bound is made all lowercase so that the bounds also work when comparing - ** BLOBs. - */ - if( pWC->op==TK_AND - && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) - ){ - Expr *pLeft; /* LHS of LIKE/GLOB operator */ - Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ - Expr *pNewExpr1; - Expr *pNewExpr2; - int idxNew1; - int idxNew2; - const char *zCollSeqName; /* Name of collating sequence */ - const u16 wtFlags = TERM_LIKEOPT | TERM_VIRTUAL | TERM_DYNAMIC; - - pLeft = pExpr->x.pList->a[1].pExpr; - pStr2 = sqlite3ExprDup(db, pStr1, 0); - - /* Convert the lower bound to upper-case and the upper bound to - ** lower-case (upper-case is less than lower-case in ASCII) so that - ** the range constraints also work for BLOBs - */ - if( noCase && !pParse->db->mallocFailed ){ - int i; - char c; - pTerm->wtFlags |= TERM_LIKE; - for(i=0; (c = pStr1->u.zToken[i])!=0; i++){ - pStr1->u.zToken[i] = sqlite3Toupper(c); - pStr2->u.zToken[i] = sqlite3Tolower(c); - } - } - - if( !db->mallocFailed ){ - u8 c, *pC; /* Last character before the first wildcard */ - pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; - c = *pC; - if( noCase ){ - /* The point is to increment the last character before the first - ** wildcard. But if we increment '@', that will push it into the - ** alphabetic range where case conversions will mess up the - ** inequality. To avoid this, make sure to also run the full - ** LIKE on all candidate expressions by clearing the isComplete flag - */ - if( c=='A'-1 ) isComplete = 0; - c = sqlite3UpperToLower[c]; - } - *pC = c + 1; - } - zCollSeqName = noCase ? "NOCASE" : "BINARY"; - pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); - pNewExpr1 = sqlite3PExpr(pParse, TK_GE, - sqlite3ExprAddCollateString(pParse,pNewExpr1,zCollSeqName), - pStr1, 0); - transferJoinMarkings(pNewExpr1, pExpr); - idxNew1 = whereClauseInsert(pWC, pNewExpr1, wtFlags); - testcase( idxNew1==0 ); - exprAnalyze(pSrc, pWC, idxNew1); - pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); - pNewExpr2 = sqlite3PExpr(pParse, TK_LT, - sqlite3ExprAddCollateString(pParse,pNewExpr2,zCollSeqName), - pStr2, 0); - transferJoinMarkings(pNewExpr2, pExpr); - idxNew2 = whereClauseInsert(pWC, pNewExpr2, wtFlags); - testcase( idxNew2==0 ); - exprAnalyze(pSrc, pWC, idxNew2); - pTerm = &pWC->a[idxTerm]; - if( isComplete ){ - markTermAsChild(pWC, idxNew1, idxTerm); - markTermAsChild(pWC, idxNew2, idxTerm); - } - } -#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ - -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Add a WO_MATCH auxiliary term to the constraint set if the - ** current expression is of the form: column MATCH expr. - ** This information is used by the xBestIndex methods of - ** virtual tables. The native query optimizer does not attempt - ** to do anything with MATCH functions. - */ - if( pWC->op==TK_AND && isMatchOfColumn(pExpr, &eOp2) ){ - int idxNew; - Expr *pRight, *pLeft; - WhereTerm *pNewTerm; - Bitmask prereqColumn, prereqExpr; - - pRight = pExpr->x.pList->a[0].pExpr; - pLeft = pExpr->x.pList->a[1].pExpr; - prereqExpr = sqlite3WhereExprUsage(pMaskSet, pRight); - prereqColumn = sqlite3WhereExprUsage(pMaskSet, pLeft); - if( (prereqExpr & prereqColumn)==0 ){ - Expr *pNewExpr; - pNewExpr = sqlite3PExpr(pParse, TK_MATCH, - 0, sqlite3ExprDup(db, pRight, 0), 0); - idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = prereqExpr; - pNewTerm->leftCursor = pLeft->iTable; - pNewTerm->u.leftColumn = pLeft->iColumn; - pNewTerm->eOperator = WO_MATCH; - pNewTerm->eMatchOp = eOp2; - markTermAsChild(pWC, idxNew, idxTerm); - pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags |= TERM_COPIED; - pNewTerm->prereqAll = pTerm->prereqAll; - } - } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - - /* If there is a vector == or IS term - e.g. "(a, b) == (?, ?)" - create - ** new terms for each component comparison - "a = ?" and "b = ?". The - ** new terms completely replace the original vector comparison, which is - ** no longer used. - ** - ** This is only required if at least one side of the comparison operation - ** is not a sub-select. */ - if( pWC->op==TK_AND - && (pExpr->op==TK_EQ || pExpr->op==TK_IS) - && sqlite3ExprIsVector(pExpr->pLeft) - && ( (pExpr->pLeft->flags & EP_xIsSelect)==0 - || (pExpr->pRight->flags & EP_xIsSelect)==0 - )){ - int nLeft = sqlite3ExprVectorSize(pExpr->pLeft); - int i; - assert( nLeft==sqlite3ExprVectorSize(pExpr->pRight) ); - for(i=0; ipLeft, i); - Expr *pRight = sqlite3ExprForVectorField(pParse, pExpr->pRight, i); - - pNew = sqlite3PExpr(pParse, pExpr->op, pLeft, pRight, 0); - transferJoinMarkings(pNew, pExpr); - idxNew = whereClauseInsert(pWC, pNew, TERM_DYNAMIC); - exprAnalyze(pSrc, pWC, idxNew); - } - pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags = TERM_CODED|TERM_VIRTUAL; /* Disable the original */ - pTerm->eOperator = 0; - } - - /* If there is a vector IN term - e.g. "(a, b) IN (SELECT ...)" - create - ** a virtual term for each vector component. The expression object - ** used by each such virtual term is pExpr (the full vector IN(...) - ** expression). The WhereTerm.iField variable identifies the index within - ** the vector on the LHS that the virtual term represents. - ** - ** This only works if the RHS is a simple SELECT, not a compound - */ - if( pWC->op==TK_AND && pExpr->op==TK_IN && pTerm->iField==0 - && pExpr->pLeft->op==TK_VECTOR - && pExpr->x.pSelect->pPrior==0 - ){ - int i; - for(i=0; ipLeft); i++){ - int idxNew; - idxNew = whereClauseInsert(pWC, pExpr, TERM_VIRTUAL); - pWC->a[idxNew].iField = i+1; - exprAnalyze(pSrc, pWC, idxNew); - markTermAsChild(pWC, idxNew, idxTerm); - } - } - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - /* When sqlite_stat3 histogram data is available an operator of the - ** form "x IS NOT NULL" can sometimes be evaluated more efficiently - ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a - ** virtual term of that form. - ** - ** Note that the virtual term must be tagged with TERM_VNULL. - */ - if( pExpr->op==TK_NOTNULL - && pExpr->pLeft->op==TK_COLUMN - && pExpr->pLeft->iColumn>=0 - && OptimizationEnabled(db, SQLITE_Stat34) - ){ - Expr *pNewExpr; - Expr *pLeft = pExpr->pLeft; - int idxNew; - WhereTerm *pNewTerm; - - pNewExpr = sqlite3PExpr(pParse, TK_GT, - sqlite3ExprDup(db, pLeft, 0), - sqlite3ExprAlloc(db, TK_NULL, 0, 0), 0); - - idxNew = whereClauseInsert(pWC, pNewExpr, - TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); - if( idxNew ){ - pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = 0; - pNewTerm->leftCursor = pLeft->iTable; - pNewTerm->u.leftColumn = pLeft->iColumn; - pNewTerm->eOperator = WO_GT; - markTermAsChild(pWC, idxNew, idxTerm); - pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags |= TERM_COPIED; - pNewTerm->prereqAll = pTerm->prereqAll; - } - } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ - - /* Prevent ON clause terms of a LEFT JOIN from being used to drive - ** an index for tables to the left of the join. - */ - testcase( pTerm!=&pWC->a[idxTerm] ); - pTerm = &pWC->a[idxTerm]; - pTerm->prereqRight |= extraRight; -} - -/*************************************************************************** -** Routines with file scope above. Interface to the rest of the where.c -** subsystem follows. -***************************************************************************/ + assert( pBuilder->nRecValid==nRecValid ); + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* -** This routine identifies subexpressions in the WHERE clause where -** each subexpression is separated by the AND operator or some other -** operator specified in the op parameter. The WhereClause structure -** is filled with pointers to subexpressions. For example: +** Disable a term in the WHERE clause. Except, do not disable the term +** if it controls a LEFT OUTER JOIN and it did not originate in the ON +** or USING clause of that join. ** -** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) -** \________/ \_______________/ \________________/ -** slot[0] slot[1] slot[2] +** Consider the term t2.z='ok' in the following queries: ** -** The original WHERE clause in pExpr is unaltered. All this routine -** does is make slot[] entries point to substructure within pExpr. +** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' +** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' +** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' ** -** In the previous sentence and in the diagram, "slot[]" refers to -** the WhereClause.a[] array. The slot[] array grows as needed to contain -** all terms of the WHERE clause. +** The t2.z='ok' is disabled in the in (2) because it originates +** in the ON clause. The term is disabled in (3) because it is not part +** of a LEFT OUTER JOIN. In (1), the term is not disabled. +** +** Disabling a term causes that term to not be tested in the inner loop +** of the join. Disabling is an optimization. When terms are satisfied +** by indices, we disable them to prevent redundant tests in the inner +** loop. We would get the correct results if nothing were ever disabled, +** but joins might run a little slower. The trick is to disable as much +** as we can without disabling too much. If we disabled in (1), we'd get +** the wrong answer. See ticket #813. +** +** If all the children of a term are disabled, then that term is also +** automatically disabled. In this way, terms get disabled if derived +** virtual terms are tested first. For example: +** +** x GLOB 'abc*' AND x>='abc' AND x<'acd' +** \___________/ \______/ \_____/ +** parent child1 child2 +** +** Only the parent term was in the original WHERE clause. The child1 +** and child2 terms were added by the LIKE optimization. If both of +** the virtual child terms are valid, then testing of the parent can be +** skipped. +** +** Usually the parent term is marked as TERM_CODED. But if the parent +** term was originally TERM_LIKE, then the parent gets TERM_LIKECOND instead. +** The TERM_LIKECOND marking indicates that the term should be coded inside +** a conditional such that is only evaluated on the second pass of a +** LIKE-optimization loop, when scanning BLOBs instead of strings. */ -SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ - Expr *pE2 = sqlite3ExprSkipCollate(pExpr); - pWC->op = op; - if( pE2==0 ) return; - if( pE2->op!=op ){ - whereClauseInsert(pWC, pExpr, 0); - }else{ - sqlite3WhereSplit(pWC, pE2->pLeft, op); - sqlite3WhereSplit(pWC, pE2->pRight, op); +static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ + int nLoop = 0; + while( pTerm + && (pTerm->wtFlags & TERM_CODED)==0 + && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + && (pLevel->notReady & pTerm->prereqAll)==0 + ){ + if( nLoop && (pTerm->wtFlags & TERM_LIKE)!=0 ){ + pTerm->wtFlags |= TERM_LIKECOND; + }else{ + pTerm->wtFlags |= TERM_CODED; + } + if( pTerm->iParent<0 ) break; + pTerm = &pTerm->pWC->a[pTerm->iParent]; + pTerm->nChild--; + if( pTerm->nChild!=0 ) break; + nLoop++; } } /* -** Initialize a preallocated WhereClause structure. -*/ -SQLITE_PRIVATE void sqlite3WhereClauseInit( - WhereClause *pWC, /* The WhereClause to be initialized */ - WhereInfo *pWInfo /* The WHERE processing context */ -){ - pWC->pWInfo = pWInfo; - pWC->pOuter = 0; - pWC->nTerm = 0; - pWC->nSlot = ArraySize(pWC->aStatic); - pWC->a = pWC->aStatic; -} - -/* -** Deallocate a WhereClause structure. The WhereClause structure -** itself is not freed. This routine is the inverse of -** sqlite3WhereClauseInit(). +** Code an OP_Affinity opcode to apply the column affinity string zAff +** to the n registers starting at base. +** +** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the +** beginning and end of zAff are ignored. If all entries in zAff are +** SQLITE_AFF_NONE, then no code gets generated. +** +** This routine makes its own copy of zAff so that the caller is free +** to modify zAff after this routine returns. */ -SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ - int i; - WhereTerm *a; - sqlite3 *db = pWC->pWInfo->pParse->db; - for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ - if( a->wtFlags & TERM_DYNAMIC ){ - sqlite3ExprDelete(db, a->pExpr); - } - if( a->wtFlags & TERM_ORINFO ){ - whereOrInfoDelete(db, a->u.pOrInfo); - }else if( a->wtFlags & TERM_ANDINFO ){ - whereAndInfoDelete(db, a->u.pAndInfo); - } - } - if( pWC->a!=pWC->aStatic ){ - sqlite3DbFree(db, pWC->a); +static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ + Vdbe *v = pParse->pVdbe; + if( zAff==0 ){ + assert( pParse->db->mallocFailed ); + return; } -} - + assert( v!=0 ); -/* -** These routines walk (recursively) an expression tree and generate -** a bitmask indicating which tables are used in that expression -** tree. -*/ -SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ - Bitmask mask; - if( p==0 ) return 0; - if( p->op==TK_COLUMN ){ - mask = sqlite3WhereGetMask(pMaskSet, p->iTable); - return mask; + /* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning + ** and end of the affinity string. + */ + while( n>0 && zAff[0]==SQLITE_AFF_NONE ){ + n--; + base++; + zAff++; } - assert( !ExprHasProperty(p, EP_TokenOnly) ); - mask = p->pRight ? sqlite3WhereExprUsage(pMaskSet, p->pRight) : 0; - if( p->pLeft ) mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft); - if( ExprHasProperty(p, EP_xIsSelect) ){ - mask |= exprSelectUsage(pMaskSet, p->x.pSelect); - }else if( p->x.pList ){ - mask |= sqlite3WhereExprListUsage(pMaskSet, p->x.pList); + while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){ + n--; } - return mask; -} -SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet *pMaskSet, ExprList *pList){ - int i; - Bitmask mask = 0; - if( pList ){ - for(i=0; inExpr; i++){ - mask |= sqlite3WhereExprUsage(pMaskSet, pList->a[i].pExpr); - } + + /* Code the OP_Affinity opcode if there is anything left to do. */ + if( n>0 ){ + sqlite3VdbeAddOp2(v, OP_Affinity, base, n); + sqlite3VdbeChangeP4(v, -1, zAff, n); + sqlite3ExprCacheAffinityChange(pParse, base, n); } - return mask; } /* -** Call exprAnalyze on all terms in a WHERE clause. +** Generate code for a single equality term of the WHERE clause. An equality +** term can be either X=expr or X IN (...). pTerm is the term to be +** coded. ** -** Note that exprAnalyze() might add new virtual terms onto the -** end of the WHERE clause. We do not want to analyze these new -** virtual terms, so start analyzing at the end and work forward -** so that the added virtual terms are never processed. -*/ -SQLITE_PRIVATE void sqlite3WhereExprAnalyze( - SrcList *pTabList, /* the FROM clause */ - WhereClause *pWC /* the WHERE clause to be analyzed */ -){ - int i; - for(i=pWC->nTerm-1; i>=0; i--){ - exprAnalyze(pTabList, pWC, i); - } -} - -/* -** For table-valued-functions, transform the function arguments into -** new WHERE clause terms. +** The current value for the constraint is left in register iReg. ** -** Each function argument translates into an equality constraint against -** a HIDDEN column in the table. +** For a constraint of the form X=expr, the expression is evaluated and its +** result is left on the stack. For constraints of the form X IN (...) +** this routine sets up a loop that will iterate over all values of X. */ -SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( - Parse *pParse, /* Parsing context */ - struct SrcList_item *pItem, /* The FROM clause term to process */ - WhereClause *pWC /* Xfer function arguments to here */ +static int codeEqualityTerm( + Parse *pParse, /* The parsing context */ + WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ + int bRev, /* True for reverse-order IN operations */ + int iTarget /* Attempt to leave results in this register */ ){ - Table *pTab; - int j, k; - ExprList *pArgs; - Expr *pColRef; - Expr *pTerm; - if( pItem->fg.isTabFunc==0 ) return; - pTab = pItem->pTab; - assert( pTab!=0 ); - pArgs = pItem->u1.pFuncArg; - if( pArgs==0 ) return; - for(j=k=0; jnExpr; j++){ - while( knCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){k++;} - if( k>=pTab->nCol ){ - sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d", - pTab->zName, j); - return; + Expr *pX = pTerm->pExpr; + Vdbe *v = pParse->pVdbe; + int iReg; /* Register holding results */ + + assert( iTarget>0 ); + if( pX->op==TK_EQ ){ + iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); + }else if( pX->op==TK_ISNULL ){ + iReg = iTarget; + sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); +#ifndef SQLITE_OMIT_SUBQUERY + }else{ + int eType; + int iTab; + struct InLoop *pIn; + WhereLoop *pLoop = pLevel->pWLoop; + + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( bRev ); + bRev = !bRev; + } + assert( pX->op==TK_IN ); + iReg = iTarget; + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0); + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } + iTab = pX->iTable; + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + VdbeCoverageIf(v, bRev); + VdbeCoverageIf(v, !bRev); + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + pLoop->wsFlags |= WHERE_IN_ABLE; + if( pLevel->u.in.nIn==0 ){ + pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + } + pLevel->u.in.nIn++; + pLevel->u.in.aInLoop = + sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); + pIn = pLevel->u.in.aInLoop; + if( pIn ){ + pIn += pLevel->u.in.nIn - 1; + pIn->iCur = iTab; + if( eType==IN_INDEX_ROWID ){ + pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg); + }else{ + pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); + } + pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; + sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v); + }else{ + pLevel->u.in.nIn = 0; } - pColRef = sqlite3ExprAlloc(pParse->db, TK_COLUMN, 0, 0); - if( pColRef==0 ) return; - pColRef->iTable = pItem->iCursor; - pColRef->iColumn = k++; - pColRef->pTab = pTab; - pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, - sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); - whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); +#endif } + disableTerm(pLevel, pTerm); + return iReg; } -/************** End of whereexpr.c *******************************************/ -/************** Begin file where.c *******************************************/ /* -** 2001 September 15 +** Generate code that will evaluate all == and IN constraints for an +** index scan. ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). +** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 +** The index has as many as three equality constraints, but in this +** example, the third "c" value is an inequality. So only two +** constraints are coded. This routine will generate code to evaluate +** a==5 and b IN (1,2,3). The current values for a and b will be stored +** in consecutive registers and the index of the first register is returned. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** In the example above nEq==2. But this subroutine works for any value +** of nEq including 0. If nEq==0, this routine is nearly a no-op. +** The only thing it does is allocate the pLevel->iMem memory cell and +** compute the affinity string. ** -************************************************************************* -** This module contains C code that generates VDBE code used to process -** the WHERE clause of SQL statements. This module is responsible for -** generating the code that loops through a table looking for applicable -** rows. Indices are selected and used to speed the search when doing -** so is applicable. Because this module is responsible for selecting -** indices, you might also think of this module as the "query optimizer". +** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints +** are == or IN and are covered by the nEq. nExtraReg is 1 if there is +** an inequality constraint (such as the "c>=5 AND c<10" in the example) that +** occurs after the nEq quality constraints. +** +** This routine allocates a range of nEq+nExtraReg memory cells and returns +** the index of the first memory cell in that range. The code that +** calls this routine will use that memory range to store keys for +** start and termination conditions of the loop. +** key value of the loop. If one or more IN operators appear, then +** this routine allocates an additional nEq memory cells for internal +** use. +** +** Before returning, *pzAff is set to point to a buffer containing a +** copy of the column affinity string of the index allocated using +** sqlite3DbMalloc(). Except, entries in the copy of the string associated +** with equality constraints that use NONE affinity are set to +** SQLITE_AFF_NONE. This is to deal with SQL such as the following: +** +** CREATE TABLE t1(a TEXT PRIMARY KEY, b); +** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; +** +** In the example above, the index on t1(a) has TEXT affinity. But since +** the right hand side of the equality constraint (t2.b) has NONE affinity, +** no conversion should be attempted before using a t2.b value as part of +** a key to search the index. Hence the first byte in the returned affinity +** string in this example would be set to SQLITE_AFF_NONE. */ -/* #include "sqliteInt.h" */ -/* #include "whereInt.h" */ - -/* Forward declaration of methods */ -static int whereLoopResize(sqlite3*, WhereLoop*, int); +static int codeAllEqualityTerms( + Parse *pParse, /* Parsing context */ + WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ + int bRev, /* Reverse the order of IN operators */ + int nExtraReg, /* Number of extra registers to allocate */ + char **pzAff /* OUT: Set to point to affinity string */ +){ + u16 nEq; /* The number of == or IN constraints to code */ + u16 nSkip; /* Number of left-most columns to skip */ + Vdbe *v = pParse->pVdbe; /* The vm under construction */ + Index *pIdx; /* The index being used for this loop */ + WhereTerm *pTerm; /* A single constraint term */ + WhereLoop *pLoop; /* The WhereLoop object */ + int j; /* Loop counter */ + int regBase; /* Base register */ + int nReg; /* Number of registers to allocate */ + char *zAff; /* Affinity string to return */ -/* Test variable that can be set to enable WHERE tracing */ -#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -/***/ int sqlite3WhereTrace = 0; -#endif + /* This module is only called on query plans that use an index. */ + pLoop = pLevel->pWLoop; + assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + nEq = pLoop->u.btree.nEq; + nSkip = pLoop->nSkip; + pIdx = pLoop->u.btree.pIndex; + assert( pIdx!=0 ); + /* Figure out how many memory cells we will need then allocate them. + */ + regBase = pParse->nMem + 1; + nReg = pLoop->u.btree.nEq + nExtraReg; + pParse->nMem += nReg; -/* -** Return the estimated number of output rows from a WHERE clause -*/ -SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ - return pWInfo->nRowOut; -} + zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx)); + if( !zAff ){ + pParse->db->mallocFailed = 1; + } -/* -** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this -** WHERE clause returns outputs for DISTINCT processing. -*/ -SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ - return pWInfo->eDistinct; -} + if( nSkip ){ + int iIdxCur = pLevel->iIdxCur; + sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); + j = sqlite3VdbeAddOp0(v, OP_Goto); + pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), + iIdxCur, 0, regBase, nSkip); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + sqlite3VdbeJumpHere(v, j); + for(j=0; jaiColumn[j]>=0 ); + VdbeComment((v, "%s", pIdx->pTable->aCol[pIdx->aiColumn[j]].zName)); + } + } -/* -** Return TRUE if the WHERE clause returns rows in ORDER BY order. -** Return FALSE if the output needs to be sorted. -*/ -SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->nOBSat; + /* Evaluate the equality constraints + */ + assert( zAff==0 || (int)strlen(zAff)>=nEq ); + for(j=nSkip; jaLTerm[j]; + assert( pTerm!=0 ); + /* The following testcase is true for indices with redundant columns. + ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ + testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); + if( r1!=regBase+j ){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, regBase); + regBase = r1; + }else{ + sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); + } + } + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IN ); + if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ + Expr *pRight = pTerm->pExpr->pRight; + if( sqlite3ExprCanBeNull(pRight) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); + VdbeCoverage(v); + } + if( zAff ){ + if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){ + zAff[j] = SQLITE_AFF_NONE; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ + zAff[j] = SQLITE_AFF_NONE; + } + } + } + } + *pzAff = zAff; + return regBase; } +#ifndef SQLITE_OMIT_EXPLAIN /* -** Return TRUE if the innermost loop of the WHERE clause implementation -** returns rows in ORDER BY order for complete run of the inner loop. +** This routine is a helper for explainIndexRange() below ** -** Across multiple iterations of outer loops, the output rows need not be -** sorted. As long as rows are sorted for just the innermost loop, this -** routine can return TRUE. -*/ -SQLITE_PRIVATE int sqlite3WhereOrderedInnerLoop(WhereInfo *pWInfo){ - return pWInfo->bOrderedInnerLoop; -} - -/* -** Return the VDBE address or label to jump to in order to continue -** immediately with the next row of a WHERE clause. -*/ -SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ - assert( pWInfo->iContinue!=0 ); - return pWInfo->iContinue; -} - -/* -** Return the VDBE address or label to jump to in order to break -** out of a WHERE loop. +** pStr holds the text of an expression that we are building up one term +** at a time. This routine adds a new term to the end of the expression. +** Terms are separated by AND so add the "AND" text for second and subsequent +** terms only. */ -SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ - return pWInfo->iBreak; +static void explainAppendTerm( + StrAccum *pStr, /* The text expression being built */ + int iTerm, /* Index of this term. First is zero */ + const char *zColumn, /* Name of the column */ + const char *zOp /* Name of the operator */ +){ + if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3StrAccumAppendAll(pStr, zColumn); + sqlite3StrAccumAppend(pStr, zOp, 1); + sqlite3StrAccumAppend(pStr, "?", 1); } /* -** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to -** operate directly on the rowis returned by a WHERE clause. Return -** ONEPASS_SINGLE (1) if the statement can operation directly because only -** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass -** optimization can be used on multiple +** Argument pLevel describes a strategy for scanning table pTab. This +** function appends text to pStr that describes the subset of table +** rows scanned by the strategy in the form of an SQL expression. ** -** If the ONEPASS optimization is used (if this routine returns true) -** then also write the indices of open cursors used by ONEPASS -** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data -** table and iaCur[1] gets the cursor used by an auxiliary index. -** Either value may be -1, indicating that cursor is not used. -** Any cursors returned will have been opened for writing. +** For example, if the query: ** -** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is -** unable to use the ONEPASS optimization. -*/ -SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ - memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); -#ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){ - sqlite3DebugPrintf("%s cursors: %d %d\n", - pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI", - aiCur[0], aiCur[1]); - } -#endif - return pWInfo->eOnePass; -} - -/* -** Move the content of pSrc into pDest -*/ -static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ - pDest->n = pSrc->n; - memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); -} - -/* -** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. +** SELECT * FROM t1 WHERE a=1 AND b>2; ** -** The new entry might overwrite an existing entry, or it might be -** appended, or it might be discarded. Do whatever is the right thing -** so that pSet keeps the N_OR_COST best entries seen so far. +** is run and there is an index on (a, b), then this function returns a +** string similar to: +** +** "a=? AND b>?" */ -static int whereOrInsert( - WhereOrSet *pSet, /* The WhereOrSet to be updated */ - Bitmask prereq, /* Prerequisites of the new entry */ - LogEst rRun, /* Run-cost of the new entry */ - LogEst nOut /* Number of outputs for the new entry */ -){ - u16 i; - WhereOrCost *p; - for(i=pSet->n, p=pSet->a; i>0; i--, p++){ - if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ - goto whereOrInsert_done; - } - if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ - return 0; - } - } - if( pSet->na[pSet->n++]; - p->nOut = nOut; - }else{ - p = pSet->a; - for(i=1; in; i++){ - if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; +static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ + Index *pIndex = pLoop->u.btree.pIndex; + u16 nEq = pLoop->u.btree.nEq; + u16 nSkip = pLoop->nSkip; + int i, j; + Column *aCol = pTab->aCol; + i16 *aiColumn = pIndex->aiColumn; + + if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; + sqlite3StrAccumAppend(pStr, " (", 2); + for(i=0; i=nSkip ){ + explainAppendTerm(pStr, i, z, "="); + }else{ + if( i ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3XPrintf(pStr, 0, "ANY(%s)", z); } - if( p->rRun<=rRun ) return 0; } -whereOrInsert_done: - p->prereq = prereq; - p->rRun = rRun; - if( p->nOut>nOut ) p->nOut = nOut; - return 1; -} -/* -** Return the bitmask for the given cursor number. Return 0 if -** iCursor is not in the set. -*/ -SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){ - int i; - assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); - for(i=0; in; i++){ - if( pMaskSet->ix[i]==iCursor ){ - return MASKBIT(i); - } + j = i; + if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(pStr, i++, z, ">"); } - return 0; + if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(pStr, i, z, "<"); + } + sqlite3StrAccumAppend(pStr, ")", 1); } /* -** Create a new mask for cursor iCursor. +** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN +** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was +** defined at compile-time. If it is not a no-op, a single OP_Explain opcode +** is added to the output to describe the table scan strategy in pLevel. ** -** There is one cursor per table in the FROM clause. The number of -** tables in the FROM clause is limited by a test early in the -** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] -** array will never overflow. +** If an OP_Explain opcode is added to the VM, its address is returned. +** Otherwise, if no OP_Explain is coded, zero is returned. */ -static void createMask(WhereMaskSet *pMaskSet, int iCursor){ - assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); - pMaskSet->ix[pMaskSet->n++] = iCursor; -} +static int explainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +){ + int ret = 0; +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pParse->explain==2 ) +#endif + { + struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; + Vdbe *v = pParse->pVdbe; /* VM being constructed */ + sqlite3 *db = pParse->db; /* Database handle */ + int iId = pParse->iSelectId; /* Select id (left-most output column) */ + int isSearch; /* True for a SEARCH. False for SCAN. */ + WhereLoop *pLoop; /* The controlling WhereLoop object */ + u32 flags; /* Flags that describe this loop */ + char *zMsg; /* Text to add to EQP output */ + StrAccum str; /* EQP output string */ + char zBuf[100]; /* Initial space for EQP output string */ -/* -** Advance to the next WhereTerm that matches according to the criteria -** established when the pScan object was initialized by whereScanInit(). -** Return NULL if there are no more matching WhereTerms. -*/ -static WhereTerm *whereScanNext(WhereScan *pScan){ - int iCur; /* The cursor on the LHS of the term */ - i16 iColumn; /* The column on the LHS of the term. -1 for IPK */ - Expr *pX; /* An expression being tested */ - WhereClause *pWC; /* Shorthand for pScan->pWC */ - WhereTerm *pTerm; /* The term being tested */ - int k = pScan->k; /* Where to start scanning */ + pLoop = pLevel->pWLoop; + flags = pLoop->wsFlags; + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0; - while( pScan->iEquiv<=pScan->nEquiv ){ - iCur = pScan->aiCur[pScan->iEquiv-1]; - iColumn = pScan->aiColumn[pScan->iEquiv-1]; - if( iColumn==XN_EXPR && pScan->pIdxExpr==0 ) return 0; - while( (pWC = pScan->pWC)!=0 ){ - for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ - if( pTerm->leftCursor==iCur - && pTerm->u.leftColumn==iColumn - && (iColumn!=XN_EXPR - || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0) - && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) - ){ - if( (pTerm->eOperator & WO_EQUIV)!=0 - && pScan->nEquivaiCur) - && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN - ){ - int j; - for(j=0; jnEquiv; j++){ - if( pScan->aiCur[j]==pX->iTable - && pScan->aiColumn[j]==pX->iColumn ){ - break; - } - } - if( j==pScan->nEquiv ){ - pScan->aiCur[j] = pX->iTable; - pScan->aiColumn[j] = pX->iColumn; - pScan->nEquiv++; - } - } - if( (pTerm->eOperator & pScan->opMask)!=0 ){ - /* Verify the affinity and collating sequence match */ - if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; - Parse *pParse = pWC->pWInfo->pParse; - pX = pTerm->pExpr; - if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ - continue; - } - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, - pX->pLeft, pX->pRight); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ - continue; - } - } - if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0 - && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN - && pX->iTable==pScan->aiCur[0] - && pX->iColumn==pScan->aiColumn[0] - ){ - testcase( pTerm->eOperator & WO_IS ); - continue; - } - pScan->k = k+1; - return pTerm; - } + isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + + sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); + if( pItem->pSelect ){ + sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); + }else{ + sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); + } + + if( pItem->zAlias ){ + sqlite3XPrintf(&str, 0, " AS %s", pItem->zAlias); + } + if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + const char *zFmt = 0; + Index *pIdx; + + assert( pLoop->u.btree.pIndex!=0 ); + pIdx = pLoop->u.btree.pIndex; + assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); + if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + if( isSearch ){ + zFmt = "PRIMARY KEY"; } + }else if( flags & WHERE_PARTIALIDX ){ + zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; + }else if( flags & WHERE_AUTO_INDEX ){ + zFmt = "AUTOMATIC COVERING INDEX"; + }else if( flags & WHERE_IDX_ONLY ){ + zFmt = "COVERING INDEX %s"; + }else{ + zFmt = "INDEX %s"; } - pScan->pWC = pScan->pWC->pOuter; - k = 0; + if( zFmt ){ + sqlite3StrAccumAppend(&str, " USING ", 7); + sqlite3XPrintf(&str, 0, zFmt, pIdx->zName); + explainIndexRange(&str, pLoop, pItem->pTab); + } + }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ + const char *zRange; + if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ + zRange = "(rowid=?)"; + }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ + zRange = "(rowid>? AND rowid?)"; + }else{ + assert( flags&WHERE_TOP_LIMIT); + zRange = "(rowidpWC = pScan->pOrigWC; - k = 0; - pScan->iEquiv++; +#ifndef SQLITE_OMIT_VIRTUALTABLE + else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ + sqlite3XPrintf(&str, 0, " VIRTUAL TABLE INDEX %d:%s", + pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); + } +#endif +#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS + if( pLoop->nOut>=10 ){ + sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); + }else{ + sqlite3StrAccumAppend(&str, " (~1 row)", 9); + } +#endif + zMsg = sqlite3StrAccumFinish(&str); + ret = sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg,P4_DYNAMIC); } - return 0; + return ret; } +#else +# define explainOneScan(u,v,w,x,y,z) 0 +#endif /* SQLITE_OMIT_EXPLAIN */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* -** Initialize a WHERE clause scanner object. Return a pointer to the -** first match. Return NULL if there are no matches. -** -** The scanner will be searching the WHERE clause pWC. It will look -** for terms of the form "X " where X is column iColumn of table -** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx -** must be one of the indexes of table iCur. -** -** The must be one of the operators described by opMask. -** -** If the search is for X and the WHERE clause contains terms of the -** form X=Y then this routine might also return terms of the form -** "Y ". The number of levels of transitivity is limited, -** but is enough to handle most commonly occurring SQL statements. +** Configure the VM passed as the first argument with an +** sqlite3_stmt_scanstatus() entry corresponding to the scan used to +** implement level pLvl. Argument pSrclist is a pointer to the FROM +** clause that the scan reads data from. ** -** If X is not the INTEGER PRIMARY KEY then X must be compatible with -** index pIdx. +** If argument addrExplain is not 0, it must be the address of an +** OP_Explain instruction that describes the same loop. */ -static WhereTerm *whereScanInit( - WhereScan *pScan, /* The WhereScan object being initialized */ - WhereClause *pWC, /* The WHERE clause to be scanned */ - int iCur, /* Cursor to scan for */ - int iColumn, /* Column to scan for */ - u32 opMask, /* Operator(s) to scan for */ - Index *pIdx /* Must be compatible with this index */ +static void addScanStatus( + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ ){ - int j = 0; - - /* memset(pScan, 0, sizeof(*pScan)); */ - pScan->pOrigWC = pWC; - pScan->pWC = pWC; - pScan->pIdxExpr = 0; - if( pIdx ){ - j = iColumn; - iColumn = pIdx->aiColumn[j]; - if( iColumn==XN_EXPR ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; - if( iColumn==pIdx->pTable->iPKey ) iColumn = XN_ROWID; - } - if( pIdx && iColumn>=0 ){ - pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; - pScan->zCollName = pIdx->azColl[j]; + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; }else{ - pScan->idxaff = 0; - pScan->zCollName = 0; + zObj = pSrclist->a[pLvl->iFrom].zName; } - pScan->opMask = opMask; - pScan->k = 0; - pScan->aiCur[0] = iCur; - pScan->aiColumn[0] = iColumn; - pScan->nEquiv = 1; - pScan->iEquiv = 1; - return whereScanNext(pScan); + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); } +#else +# define addScanStatus(a, b, c, d) ((void)d) +#endif /* -** Search for a term in the WHERE clause that is of the form "X " -** where X is a reference to the iColumn of table iCur or of index pIdx -** if pIdx!=0 and is one of the WO_xx operator codes specified by -** the op parameter. Return a pointer to the term. Return 0 if not found. -** -** If pIdx!=0 then it must be one of the indexes of table iCur. -** Search for terms matching the iColumn-th column of pIdx -** rather than the iColumn-th column of table iCur. -** -** The term returned might by Y= if there is another constraint in -** the WHERE clause that specifies that X=Y. Any such constraints will be -** identified by the WO_EQUIV bit in the pTerm->eOperator field. The -** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11 -** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10 -** other equivalent values. Hence a search for X will return if X=A1 -** and A1=A2 and A2=A3 and ... and A9=A10 and A10=. +** If the most recently coded instruction is a constant range contraint +** that originated from the LIKE optimization, then change the P3 to be +** pLoop->iLikeRepCntr and set P5. ** -** If there are multiple terms in the WHERE clause of the form "X " -** then try for the one with no dependencies on - in other words where -** is a constant expression of some kind. Only return entries of -** the form "X Y" where Y is a column in another table if no terms of -** the form "X " exist. If no terms with a constant RHS -** exist, try to return a term that does not use WO_EQUIV. +** The LIKE optimization trys to evaluate "x LIKE 'abc%'" as a range +** expression: "x>='ABC' AND x<'abd'". But this requires that the range +** scan loop run twice, once for strings and a second time for BLOBs. +** The OP_String opcodes on the second pass convert the upper and lower +** bound string contants to blobs. This routine makes the necessary changes +** to the OP_String opcodes for that to happen. */ -SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( - WhereClause *pWC, /* The WHERE clause to be searched */ - int iCur, /* Cursor number of LHS */ - int iColumn, /* Column number of LHS */ - Bitmask notReady, /* RHS must not overlap with this mask */ - u32 op, /* Mask of WO_xx values describing operator */ - Index *pIdx /* Must be compatible with this index, if not NULL */ +static void whereLikeOptimizationStringFixup( + Vdbe *v, /* prepared statement under construction */ + WhereLevel *pLevel, /* The loop that contains the LIKE operator */ + WhereTerm *pTerm /* The upper or lower bound just coded */ ){ - WhereTerm *pResult = 0; - WhereTerm *p; - WhereScan scan; - - p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); - op &= WO_EQ|WO_IS; - while( p ){ - if( (p->prereqRight & notReady)==0 ){ - if( p->prereqRight==0 && (p->eOperator&op)!=0 ){ - testcase( p->eOperator & WO_IS ); - return p; - } - if( pResult==0 ) pResult = p; - } - p = whereScanNext(&scan); + if( pTerm->wtFlags & TERM_LIKEOPT ){ + VdbeOp *pOp; + assert( pLevel->iLikeRepCntr>0 ); + pOp = sqlite3VdbeGetOp(v, -1); + assert( pOp!=0 ); + assert( pOp->opcode==OP_String8 + || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); + pOp->p3 = pLevel->iLikeRepCntr; + pOp->p5 = 1; } - return pResult; } /* -** This function searches pList for an entry that matches the iCol-th column -** of index pIdx. -** -** If such an expression is found, its index in pList->a[] is returned. If -** no expression is found, -1 is returned. -*/ -static int findIndexCol( - Parse *pParse, /* Parse context */ - ExprList *pList, /* Expression list to search */ - int iBase, /* Cursor for table associated with pIdx */ - Index *pIdx, /* Index to match column of */ - int iCol /* Column of index to match */ -){ - int i; - const char *zColl = pIdx->azColl[iCol]; - - for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); - if( p->op==TK_COLUMN - && p->iColumn==pIdx->aiColumn[iCol] - && p->iTable==iBase - ){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); - if( pColl && 0==sqlite3StrICmp(pColl->zName, zColl) ){ - return i; - } - } - } - - return -1; -} - -/* -** Return TRUE if the iCol-th column of index pIdx is NOT NULL -*/ -static int indexColumnNotNull(Index *pIdx, int iCol){ - int j; - assert( pIdx!=0 ); - assert( iCol>=0 && iColnColumn ); - j = pIdx->aiColumn[iCol]; - if( j>=0 ){ - return pIdx->pTable->aCol[j].notNull; - }else if( j==(-1) ){ - return 1; - }else{ - assert( j==(-2) ); - return 0; /* Assume an indexed expression can always yield a NULL */ - - } -} - -/* -** Return true if the DISTINCT expression-list passed as the third argument -** is redundant. -** -** A DISTINCT list is redundant if any subset of the columns in the -** DISTINCT list are collectively unique and individually non-null. -*/ -static int isDistinctRedundant( - Parse *pParse, /* Parsing context */ - SrcList *pTabList, /* The FROM clause */ - WhereClause *pWC, /* The WHERE clause */ - ExprList *pDistinct /* The result set that needs to be DISTINCT */ -){ - Table *pTab; - Index *pIdx; - int i; - int iBase; - - /* If there is more than one table or sub-select in the FROM clause of - ** this query, then it will not be possible to show that the DISTINCT - ** clause is redundant. */ - if( pTabList->nSrc!=1 ) return 0; - iBase = pTabList->a[0].iCursor; - pTab = pTabList->a[0].pTab; - - /* If any of the expressions is an IPK column on table iBase, then return - ** true. Note: The (p->iTable==iBase) part of this test may be false if the - ** current SELECT is a correlated sub-query. - */ - for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); - if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; - } - - /* Loop through all indices on the table, checking each to see if it makes - ** the DISTINCT qualifier redundant. It does so if: - ** - ** 1. The index is itself UNIQUE, and - ** - ** 2. All of the columns in the index are either part of the pDistinct - ** list, or else the WHERE clause contains a term of the form "col=X", - ** where X is a constant value. The collation sequences of the - ** comparison and select-list expressions must match those of the index. - ** - ** 3. All of those index columns for which the WHERE clause does not - ** contain a "col=X" term are subject to a NOT NULL constraint. - */ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !IsUniqueIndex(pIdx) ) continue; - for(i=0; inKeyCol; i++){ - if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){ - if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break; - if( indexColumnNotNull(pIdx, i)==0 ) break; - } - } - if( i==pIdx->nKeyCol ){ - /* This index implies that the DISTINCT qualifier is redundant. */ - return 1; - } - } - - return 0; -} - - -/* -** Estimate the logarithm of the input value to base 2. -*/ -static LogEst estLog(LogEst N){ - return N<=10 ? 0 : sqlite3LogEst(N) - 33; -} - -/* -** Convert OP_Column opcodes to OP_Copy in previously generated code. -** -** This routine runs over generated VDBE code and translates OP_Column -** opcodes into OP_Copy when the table is being accessed via co-routine -** instead of via table lookup. -** -** If the bIncrRowid parameter is 0, then any OP_Rowid instructions on -** cursor iTabCur are transformed into OP_Null. Or, if bIncrRowid is non-zero, -** then each OP_Rowid is transformed into an instruction to increment the -** value stored in its output register. -*/ -static void translateColumnToCopy( - Vdbe *v, /* The VDBE containing code to translate */ - int iStart, /* Translate from this opcode to the end */ - int iTabCur, /* OP_Column/OP_Rowid references to this table */ - int iRegister, /* The first column is in this register */ - int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */ -){ - VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); - int iEnd = sqlite3VdbeCurrentAddr(v); - for(; iStartp1!=iTabCur ) continue; - if( pOp->opcode==OP_Column ){ - pOp->opcode = OP_Copy; - pOp->p1 = pOp->p2 + iRegister; - pOp->p2 = pOp->p3; - pOp->p3 = 0; - }else if( pOp->opcode==OP_Rowid ){ - if( bIncrRowid ){ - /* Increment the value stored in the P2 operand of the OP_Rowid. */ - pOp->opcode = OP_AddImm; - pOp->p1 = pOp->p2; - pOp->p2 = 1; - }else{ - pOp->opcode = OP_Null; - pOp->p1 = 0; - pOp->p3 = 0; - } - } - } -} - -/* -** Two routines for printing the content of an sqlite3_index_info -** structure. Used for testing and debugging only. If neither -** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines -** are no-ops. -*/ -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) -static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ - int i; - if( !sqlite3WhereTrace ) return; - for(i=0; inConstraint; i++){ - sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", - i, - p->aConstraint[i].iColumn, - p->aConstraint[i].iTermOffset, - p->aConstraint[i].op, - p->aConstraint[i].usable); - } - for(i=0; inOrderBy; i++){ - sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", - i, - p->aOrderBy[i].iColumn, - p->aOrderBy[i].desc); - } -} -static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ - int i; - if( !sqlite3WhereTrace ) return; - for(i=0; inConstraint; i++){ - sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", - i, - p->aConstraintUsage[i].argvIndex, - p->aConstraintUsage[i].omit); - } - sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum); - sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); - sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); - sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); - sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); -} -#else -#define TRACE_IDX_INPUTS(A) -#define TRACE_IDX_OUTPUTS(A) -#endif - -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX -/* -** Return TRUE if the WHERE clause term pTerm is of a form where it -** could be used with an index to access pSrc, assuming an appropriate -** index existed. -*/ -static int termCanDriveIndex( - WhereTerm *pTerm, /* WHERE clause term to check */ - struct SrcList_item *pSrc, /* Table we are trying to access */ - Bitmask notReady /* Tables in outer loops of the join */ -){ - char aff; - if( pTerm->leftCursor!=pSrc->iCursor ) return 0; - if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; - if( (pTerm->prereqRight & notReady)!=0 ) return 0; - if( pTerm->u.leftColumn<0 ) return 0; - aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; - if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; - testcase( pTerm->pExpr->op==TK_IS ); - return 1; -} -#endif - - -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX -/* -** Generate code to construct the Index object for an automatic index -** and to set up the WhereLevel object pLevel so that the code generator -** makes use of the automatic index. -*/ -static void constructAutomaticIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to get the next index */ - Bitmask notReady, /* Mask of cursors that are not available */ - WhereLevel *pLevel /* Write new index here */ -){ - int nKeyCol; /* Number of columns in the constructed index */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - WhereTerm *pWCEnd; /* End of pWC->a[] */ - Index *pIdx; /* Object describing the transient index */ - Vdbe *v; /* Prepared statement under construction */ - int addrInit; /* Address of the initialization bypass jump */ - Table *pTable; /* The table being indexed */ - int addrTop; /* Top of the index fill loop */ - int regRecord; /* Register holding an index record */ - int n; /* Column counter */ - int i; /* Loop counter */ - int mxBitCol; /* Maximum column in pSrc->colUsed */ - CollSeq *pColl; /* Collating sequence to on a column */ - WhereLoop *pLoop; /* The Loop object */ - char *zNotUsed; /* Extra space on the end of pIdx */ - Bitmask idxCols; /* Bitmap of columns used for indexing */ - Bitmask extraCols; /* Bitmap of additional columns */ - u8 sentWarning = 0; /* True if a warnning has been issued */ - Expr *pPartial = 0; /* Partial Index Expression */ - int iContinue = 0; /* Jump here to skip excluded rows */ - struct SrcList_item *pTabItem; /* FROM clause term being indexed */ - int addrCounter = 0; /* Address where integer counter is initialized */ - int regBase; /* Array of registers where record is assembled */ - - /* Generate code to skip over the creation and initialization of the - ** transient index on 2nd and subsequent iterations of the loop. */ - v = pParse->pVdbe; - assert( v!=0 ); - addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); - - /* Count the number of columns that will be added to the index - ** and used to match WHERE clause constraints */ - nKeyCol = 0; - pTable = pSrc->pTab; - pWCEnd = &pWC->a[pWC->nTerm]; - pLoop = pLevel->pWLoop; - idxCols = 0; - for(pTerm=pWC->a; pTermpExpr; - assert( !ExprHasProperty(pExpr, EP_FromJoin) /* prereq always non-zero */ - || pExpr->iRightJoinTable!=pSrc->iCursor /* for the right-hand */ - || pLoop->prereq!=0 ); /* table of a LEFT JOIN */ - if( pLoop->prereq==0 - && (pTerm->wtFlags & TERM_VIRTUAL)==0 - && !ExprHasProperty(pExpr, EP_FromJoin) - && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){ - pPartial = sqlite3ExprAnd(pParse->db, pPartial, - sqlite3ExprDup(pParse->db, pExpr, 0)); - } - if( termCanDriveIndex(pTerm, pSrc, notReady) ){ - int iCol = pTerm->u.leftColumn; - Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); - testcase( iCol==BMS ); - testcase( iCol==BMS-1 ); - if( !sentWarning ){ - sqlite3_log(SQLITE_WARNING_AUTOINDEX, - "automatic index on %s(%s)", pTable->zName, - pTable->aCol[iCol].zName); - sentWarning = 1; - } - if( (idxCols & cMask)==0 ){ - if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ - goto end_auto_index_create; - } - pLoop->aLTerm[nKeyCol++] = pTerm; - idxCols |= cMask; - } - } - } - assert( nKeyCol>0 ); - pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; - pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED - | WHERE_AUTO_INDEX; - - /* Count the number of additional columns needed to create a - ** covering index. A "covering index" is an index that contains all - ** columns that are needed by the query. With a covering index, the - ** original table never needs to be accessed. Automatic indices must - ** be a covering index because the index will not be updated if the - ** original table changes and the index and table cannot both be used - ** if they go out of sync. - */ - extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); - mxBitCol = MIN(BMS-1,pTable->nCol); - testcase( pTable->nCol==BMS-1 ); - testcase( pTable->nCol==BMS-2 ); - for(i=0; icolUsed & MASKBIT(BMS-1) ){ - nKeyCol += pTable->nCol - BMS + 1; - } - - /* Construct the Index object to describe this index */ - pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); - if( pIdx==0 ) goto end_auto_index_create; - pLoop->u.btree.pIndex = pIdx; - pIdx->zName = "auto-index"; - pIdx->pTable = pTable; - n = 0; - idxCols = 0; - for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); - testcase( iCol==BMS-1 ); - testcase( iCol==BMS ); - if( (idxCols & cMask)==0 ){ - Expr *pX = pTerm->pExpr; - idxCols |= cMask; - pIdx->aiColumn[n] = pTerm->u.leftColumn; - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; - n++; - } - } - } - assert( (u32)n==pLoop->u.btree.nEq ); - - /* Add additional columns needed to make the automatic index into - ** a covering index */ - for(i=0; iaiColumn[n] = i; - pIdx->azColl[n] = sqlite3StrBINARY; - n++; - } - } - if( pSrc->colUsed & MASKBIT(BMS-1) ){ - for(i=BMS-1; inCol; i++){ - pIdx->aiColumn[n] = i; - pIdx->azColl[n] = sqlite3StrBINARY; - n++; - } - } - assert( n==nKeyCol ); - pIdx->aiColumn[n] = XN_ROWID; - pIdx->azColl[n] = sqlite3StrBINARY; - - /* Create the automatic index */ - assert( pLevel->iIdxCur>=0 ); - pLevel->iIdxCur = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); - sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - VdbeComment((v, "for %s", pTable->zName)); - - /* Fill the automatic index with content */ - sqlite3ExprCachePush(pParse); - pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; - if( pTabItem->fg.viaCoroutine ){ - int regYield = pTabItem->regReturn; - addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); - addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); - VdbeCoverage(v); - VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); - }else{ - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); - } - if( pPartial ){ - iContinue = sqlite3VdbeMakeLabel(v); - sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); - pLoop->wsFlags |= WHERE_PARTIALIDX; - } - regRecord = sqlite3GetTempReg(pParse); - regBase = sqlite3GenerateIndexKey( - pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0 - ); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); - if( pTabItem->fg.viaCoroutine ){ - sqlite3VdbeChangeP2(v, addrCounter, regBase+n); - translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1); - sqlite3VdbeGoto(v, addrTop); - pTabItem->fg.viaCoroutine = 0; - }else{ - sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); - } - sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); - sqlite3VdbeJumpHere(v, addrTop); - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ExprCachePop(pParse); - - /* Jump here when skipping the initialization */ - sqlite3VdbeJumpHere(v, addrInit); - -end_auto_index_create: - sqlite3ExprDelete(pParse->db, pPartial); -} -#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ - -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* -** Allocate and populate an sqlite3_index_info structure. It is the -** responsibility of the caller to eventually release the structure -** by passing the pointer returned by this function to sqlite3_free(). -*/ -static sqlite3_index_info *allocateIndexInfo( - Parse *pParse, - WhereClause *pWC, - Bitmask mUnusable, /* Ignore terms with these prereqs */ - struct SrcList_item *pSrc, - ExprList *pOrderBy, - u16 *pmNoOmit /* Mask of terms not to omit */ -){ - int i, j; - int nTerm; - struct sqlite3_index_constraint *pIdxCons; - struct sqlite3_index_orderby *pIdxOrderBy; - struct sqlite3_index_constraint_usage *pUsage; - WhereTerm *pTerm; - int nOrderBy; - sqlite3_index_info *pIdxInfo; - u16 mNoOmit = 0; - - /* Count the number of possible WHERE clause constraints referring - ** to this virtual table */ - for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - if( pTerm->leftCursor != pSrc->iCursor ) continue; - if( pTerm->prereqRight & mUnusable ) continue; - assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); - testcase( pTerm->eOperator & WO_IN ); - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_IS ); - testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; - if( pTerm->wtFlags & TERM_VNULL ) continue; - assert( pTerm->u.leftColumn>=(-1) ); - nTerm++; - } - - /* If the ORDER BY clause contains only columns in the current - ** virtual table then allocate space for the aOrderBy part of - ** the sqlite3_index_info structure. - */ - nOrderBy = 0; - if( pOrderBy ){ - int n = pOrderBy->nExpr; - for(i=0; ia[i].pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; - } - if( i==n){ - nOrderBy = n; - } - } - - /* Allocate the sqlite3_index_info structure - */ - pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) - + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm - + sizeof(*pIdxOrderBy)*nOrderBy ); - if( pIdxInfo==0 ){ - sqlite3ErrorMsg(pParse, "out of memory"); - return 0; - } - - /* Initialize the structure. The sqlite3_index_info structure contains - ** many fields that are declared "const" to prevent xBestIndex from - ** changing them. We have to do some funky casting in order to - ** initialize those fields. - */ - pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1]; - pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; - pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; - *(int*)&pIdxInfo->nConstraint = nTerm; - *(int*)&pIdxInfo->nOrderBy = nOrderBy; - *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons; - *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; - *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage = - pUsage; - - for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - u8 op; - if( pTerm->leftCursor != pSrc->iCursor ) continue; - if( pTerm->prereqRight & mUnusable ) continue; - assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); - testcase( pTerm->eOperator & WO_IN ); - testcase( pTerm->eOperator & WO_IS ); - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; - if( pTerm->wtFlags & TERM_VNULL ) continue; - assert( pTerm->u.leftColumn>=(-1) ); - pIdxCons[j].iColumn = pTerm->u.leftColumn; - pIdxCons[j].iTermOffset = i; - op = (u8)pTerm->eOperator & WO_ALL; - if( op==WO_IN ) op = WO_EQ; - if( op==WO_MATCH ){ - op = pTerm->eMatchOp; - } - pIdxCons[j].op = op; - /* The direct assignment in the previous line is possible only because - ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The - ** following asserts verify this fact. */ - assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); - assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); - assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); - assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); - assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); - assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); - assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); - - if( op & (WO_LT|WO_LE|WO_GT|WO_GE) - && sqlite3ExprIsVector(pTerm->pExpr->pRight) - ){ - if( i<16 ) mNoOmit |= (1 << i); - if( op==WO_LT ) pIdxCons[j].op = WO_LE; - if( op==WO_GT ) pIdxCons[j].op = WO_GE; - } - - j++; - } - for(i=0; ia[i].pExpr; - pIdxOrderBy[i].iColumn = pExpr->iColumn; - pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; - } - - *pmNoOmit = mNoOmit; - return pIdxInfo; -} - -/* -** The table object reference passed as the second argument to this function -** must represent a virtual table. This function invokes the xBestIndex() -** method of the virtual table with the sqlite3_index_info object that -** comes in as the 3rd argument to this function. -** -** If an error occurs, pParse is populated with an error message and a -** non-zero value is returned. Otherwise, 0 is returned and the output -** part of the sqlite3_index_info structure is left populated. -** -** Whether or not an error is returned, it is the responsibility of the -** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates -** that this is required. -*/ -static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ - sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; - int rc; - - TRACE_IDX_INPUTS(p); - rc = pVtab->pModule->xBestIndex(pVtab, p); - TRACE_IDX_OUTPUTS(p); - - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ){ - sqlite3OomFault(pParse->db); - }else if( !pVtab->zErrMsg ){ - sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); - }else{ - sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); - } - } - sqlite3_free(pVtab->zErrMsg); - pVtab->zErrMsg = 0; - -#if 0 - /* This error is now caught by the caller. - ** Search for "xBestIndex malfunction" below */ - for(i=0; inConstraint; i++){ - if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ - sqlite3ErrorMsg(pParse, - "table %s: xBestIndex returned an invalid plan", pTab->zName); - } - } -#endif - - return pParse->nErr; -} -#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** Estimate the location of a particular key among all keys in an -** index. Store the results in aStat as follows: -** -** aStat[0] Est. number of rows less than pRec -** aStat[1] Est. number of rows equal to pRec -** -** Return the index of the sample that is the smallest sample that -** is greater than or equal to pRec. Note that this index is not an index -** into the aSample[] array - it is an index into a virtual set of samples -** based on the contents of aSample[] and the number of fields in record -** pRec. -*/ -static int whereKeyStats( - Parse *pParse, /* Database connection */ - Index *pIdx, /* Index to consider domain of */ - UnpackedRecord *pRec, /* Vector of values to consider */ - int roundUp, /* Round up if true. Round down if false */ - tRowcnt *aStat /* OUT: stats written here */ -){ - IndexSample *aSample = pIdx->aSample; - int iCol; /* Index of required stats in anEq[] etc. */ - int i; /* Index of first sample >= pRec */ - int iSample; /* Smallest sample larger than or equal to pRec */ - int iMin = 0; /* Smallest sample not yet tested */ - int iTest; /* Next sample to test */ - int res; /* Result of comparison operation */ - int nField; /* Number of fields in pRec */ - tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */ - -#ifndef SQLITE_DEBUG - UNUSED_PARAMETER( pParse ); -#endif - assert( pRec!=0 ); - assert( pIdx->nSample>0 ); - assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol ); - - /* Do a binary search to find the first sample greater than or equal - ** to pRec. If pRec contains a single field, the set of samples to search - ** is simply the aSample[] array. If the samples in aSample[] contain more - ** than one fields, all fields following the first are ignored. - ** - ** If pRec contains N fields, where N is more than one, then as well as the - ** samples in aSample[] (truncated to N fields), the search also has to - ** consider prefixes of those samples. For example, if the set of samples - ** in aSample is: - ** - ** aSample[0] = (a, 5) - ** aSample[1] = (a, 10) - ** aSample[2] = (b, 5) - ** aSample[3] = (c, 100) - ** aSample[4] = (c, 105) - ** - ** Then the search space should ideally be the samples above and the - ** unique prefixes [a], [b] and [c]. But since that is hard to organize, - ** the code actually searches this set: - ** - ** 0: (a) - ** 1: (a, 5) - ** 2: (a, 10) - ** 3: (a, 10) - ** 4: (b) - ** 5: (b, 5) - ** 6: (c) - ** 7: (c, 100) - ** 8: (c, 105) - ** 9: (c, 105) - ** - ** For each sample in the aSample[] array, N samples are present in the - ** effective sample array. In the above, samples 0 and 1 are based on - ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc. - ** - ** Often, sample i of each block of N effective samples has (i+1) fields. - ** Except, each sample may be extended to ensure that it is greater than or - ** equal to the previous sample in the array. For example, in the above, - ** sample 2 is the first sample of a block of N samples, so at first it - ** appears that it should be 1 field in size. However, that would make it - ** smaller than sample 1, so the binary search would not work. As a result, - ** it is extended to two fields. The duplicates that this creates do not - ** cause any problems. - */ - nField = pRec->nField; - iCol = 0; - iSample = pIdx->nSample * nField; - do{ - int iSamp; /* Index in aSample[] of test sample */ - int n; /* Number of fields in test sample */ - - iTest = (iMin+iSample)/2; - iSamp = iTest / nField; - if( iSamp>0 ){ - /* The proposed effective sample is a prefix of sample aSample[iSamp]. - ** Specifically, the shortest prefix of at least (1 + iTest%nField) - ** fields that is greater than the previous effective sample. */ - for(n=(iTest % nField) + 1; nnField = n; - res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec); - if( res<0 ){ - iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1]; - iMin = iTest+1; - }else if( res==0 && ndb->mallocFailed==0 ){ - if( res==0 ){ - /* If (res==0) is true, then pRec must be equal to sample i. */ - assert( inSample ); - assert( iCol==nField-1 ); - pRec->nField = nField; - assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) - || pParse->db->mallocFailed - ); - }else{ - /* Unless i==pIdx->nSample, indicating that pRec is larger than - ** all samples in the aSample[] array, pRec must be smaller than the - ** (iCol+1) field prefix of sample i. */ - assert( i<=pIdx->nSample && i>=0 ); - pRec->nField = iCol+1; - assert( i==pIdx->nSample - || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 - || pParse->db->mallocFailed ); - - /* if i==0 and iCol==0, then record pRec is smaller than all samples - ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must - ** be greater than or equal to the (iCol) field prefix of sample i. - ** If (i>0), then pRec must also be greater than sample (i-1). */ - if( iCol>0 ){ - pRec->nField = iCol; - assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 - || pParse->db->mallocFailed ); - } - if( i>0 ){ - pRec->nField = nField; - assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 - || pParse->db->mallocFailed ); - } - } - } -#endif /* ifdef SQLITE_DEBUG */ - - if( res==0 ){ - /* Record pRec is equal to sample i */ - assert( iCol==nField-1 ); - aStat[0] = aSample[i].anLt[iCol]; - aStat[1] = aSample[i].anEq[iCol]; - }else{ - /* At this point, the (iCol+1) field prefix of aSample[i] is the first - ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec - ** is larger than all samples in the array. */ - tRowcnt iUpper, iGap; - if( i>=pIdx->nSample ){ - iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); - }else{ - iUpper = aSample[i].anLt[iCol]; - } - - if( iLower>=iUpper ){ - iGap = 0; - }else{ - iGap = iUpper - iLower; - } - if( roundUp ){ - iGap = (iGap*2)/3; - }else{ - iGap = iGap/3; - } - aStat[0] = iLower + iGap; - aStat[1] = pIdx->aAvgEq[iCol]; - } - - /* Restore the pRec->nField value before returning. */ - pRec->nField = nField; - return i; -} -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ - -/* -** If it is not NULL, pTerm is a term that provides an upper or lower -** bound on a range scan. Without considering pTerm, it is estimated -** that the scan will visit nNew rows. This function returns the number -** estimated to be visited after taking pTerm into account. -** -** If the user explicitly specified a likelihood() value for this term, -** then the return value is the likelihood multiplied by the number of -** input rows. Otherwise, this function assumes that an "IS NOT NULL" term -** has a likelihood of 0.50, and any other term a likelihood of 0.25. -*/ -static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ - LogEst nRet = nNew; - if( pTerm ){ - if( pTerm->truthProb<=0 ){ - nRet += pTerm->truthProb; - }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ - nRet -= 20; assert( 20==sqlite3LogEst(4) ); - } - } - return nRet; -} - - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** Return the affinity for a single column of an index. -*/ -SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ - assert( iCol>=0 && iColnColumn ); - if( !pIdx->zColAff ){ - if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; - } - return pIdx->zColAff[iCol]; -} -#endif - - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** This function is called to estimate the number of rows visited by a -** range-scan on a skip-scan index. For example: -** -** CREATE INDEX i1 ON t1(a, b, c); -** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; -** -** Value pLoop->nOut is currently set to the estimated number of rows -** visited for scanning (a=? AND b=?). This function reduces that estimate -** by some factor to account for the (c BETWEEN ? AND ?) expression based -** on the stat4 data for the index. this scan will be peformed multiple -** times (once for each (a,b) combination that matches a=?) is dealt with -** by the caller. -** -** It does this by scanning through all stat4 samples, comparing values -** extracted from pLower and pUpper with the corresponding column in each -** sample. If L and U are the number of samples found to be less than or -** equal to the values extracted from pLower and pUpper respectively, and -** N is the total number of samples, the pLoop->nOut value is adjusted -** as follows: -** -** nOut = nOut * ( min(U - L, 1) / N ) -** -** If pLower is NULL, or a value cannot be extracted from the term, L is -** set to zero. If pUpper is NULL, or a value cannot be extracted from it, -** U is set to N. -** -** Normally, this function sets *pbDone to 1 before returning. However, -** if no value can be extracted from either pLower or pUpper (and so the -** estimate of the number of rows delivered remains unchanged), *pbDone -** is left as is. -** -** If an error occurs, an SQLite error code is returned. Otherwise, -** SQLITE_OK. -*/ -static int whereRangeSkipScanEst( - Parse *pParse, /* Parsing & code generating context */ - WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ - WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - WhereLoop *pLoop, /* Update the .nOut value of this loop */ - int *pbDone /* Set to true if at least one expr. value extracted */ -){ - Index *p = pLoop->u.btree.pIndex; - int nEq = pLoop->u.btree.nEq; - sqlite3 *db = pParse->db; - int nLower = -1; - int nUpper = p->nSample+1; - int rc = SQLITE_OK; - u8 aff = sqlite3IndexColumnAffinity(db, p, nEq); - CollSeq *pColl; - - sqlite3_value *p1 = 0; /* Value extracted from pLower */ - sqlite3_value *p2 = 0; /* Value extracted from pUpper */ - sqlite3_value *pVal = 0; /* Value extracted from record */ - - pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); - if( pLower ){ - rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); - nLower = 0; - } - if( pUpper && rc==SQLITE_OK ){ - rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); - nUpper = p2 ? 0 : p->nSample; - } - - if( p1 || p2 ){ - int i; - int nDiff; - for(i=0; rc==SQLITE_OK && inSample; i++){ - rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); - if( rc==SQLITE_OK && p1 ){ - int res = sqlite3MemCompare(p1, pVal, pColl); - if( res>=0 ) nLower++; - } - if( rc==SQLITE_OK && p2 ){ - int res = sqlite3MemCompare(p2, pVal, pColl); - if( res>=0 ) nUpper++; - } - } - nDiff = (nUpper - nLower); - if( nDiff<=0 ) nDiff = 1; - - /* If there is both an upper and lower bound specified, and the - ** comparisons indicate that they are close together, use the fallback - ** method (assume that the scan visits 1/64 of the rows) for estimating - ** the number of rows visited. Otherwise, estimate the number of rows - ** using the method described in the header comment for this function. */ - if( nDiff!=1 || pUpper==0 || pLower==0 ){ - int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); - pLoop->nOut -= nAdjust; - *pbDone = 1; - WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", - nLower, nUpper, nAdjust*-1, pLoop->nOut)); - } - - }else{ - assert( *pbDone==0 ); - } - - sqlite3ValueFree(p1); - sqlite3ValueFree(p2); - sqlite3ValueFree(pVal); - - return rc; -} -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ - -/* -** This function is used to estimate the number of rows that will be visited -** by scanning an index for a range of values. The range may have an upper -** bound, a lower bound, or both. The WHERE clause terms that set the upper -** and lower bounds are represented by pLower and pUpper respectively. For -** example, assuming that index p is on t1(a): -** -** ... FROM t1 WHERE a > ? AND a < ? ... -** |_____| |_____| -** | | -** pLower pUpper -** -** If either of the upper or lower bound is not present, then NULL is passed in -** place of the corresponding WhereTerm. -** -** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index -** column subject to the range constraint. Or, equivalently, the number of -** equality constraints optimized by the proposed index scan. For example, -** assuming index p is on t1(a, b), and the SQL query is: -** -** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... -** -** then nEq is set to 1 (as the range restricted column, b, is the second -** left-most column of the index). Or, if the query is: -** -** ... FROM t1 WHERE a > ? AND a < ? ... -** -** then nEq is set to 0. -** -** When this function is called, *pnOut is set to the sqlite3LogEst() of the -** number of rows that the index scan is expected to visit without -** considering the range constraints. If nEq is 0, then *pnOut is the number of -** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) -** to account for the range constraints pLower and pUpper. -** -** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be -** used, a single range inequality reduces the search space by a factor of 4. -** and a pair of constraints (x>? AND x123" Might be NULL */ - WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ -){ - int rc = SQLITE_OK; - int nOut = pLoop->nOut; - LogEst nNew; - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - Index *p = pLoop->u.btree.pIndex; - int nEq = pLoop->u.btree.nEq; - - if( p->nSample>0 && nEqnSampleCol ){ - if( nEq==pBuilder->nRecValid ){ - UnpackedRecord *pRec = pBuilder->pRec; - tRowcnt a[2]; - int nBtm = pLoop->u.btree.nBtm; - int nTop = pLoop->u.btree.nTop; - - /* Variable iLower will be set to the estimate of the number of rows in - ** the index that are less than the lower bound of the range query. The - ** lower bound being the concatenation of $P and $L, where $P is the - ** key-prefix formed by the nEq values matched against the nEq left-most - ** columns of the index, and $L is the value in pLower. - ** - ** Or, if pLower is NULL or $L cannot be extracted from it (because it - ** is not a simple variable or literal value), the lower bound of the - ** range is $P. Due to a quirk in the way whereKeyStats() works, even - ** if $L is available, whereKeyStats() is called for both ($P) and - ** ($P:$L) and the larger of the two returned values is used. - ** - ** Similarly, iUpper is to be set to the estimate of the number of rows - ** less than the upper bound of the range query. Where the upper bound - ** is either ($P) or ($P:$U). Again, even if $U is available, both values - ** of iUpper are requested of whereKeyStats() and the smaller used. - ** - ** The number of rows between the two bounds is then just iUpper-iLower. - */ - tRowcnt iLower; /* Rows less than the lower bound */ - tRowcnt iUpper; /* Rows less than the upper bound */ - int iLwrIdx = -2; /* aSample[] for the lower bound */ - int iUprIdx = -1; /* aSample[] for the upper bound */ - - if( pRec ){ - testcase( pRec->nField!=pBuilder->nRecValid ); - pRec->nField = pBuilder->nRecValid; - } - /* Determine iLower and iUpper using ($P) only. */ - if( nEq==0 ){ - iLower = 0; - iUpper = p->nRowEst0; - }else{ - /* Note: this call could be optimized away - since the same values must - ** have been requested when testing key $P in whereEqualScanEst(). */ - whereKeyStats(pParse, p, pRec, 0, a); - iLower = a[0]; - iUpper = a[0] + a[1]; - } - - assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); - assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - assert( p->aSortOrder!=0 ); - if( p->aSortOrder[nEq] ){ - /* The roles of pLower and pUpper are swapped for a DESC index */ - SWAP(WhereTerm*, pLower, pUpper); - SWAP(int, nBtm, nTop); - } - - /* If possible, improve on the iLower estimate using ($P:$L). */ - if( pLower ){ - int n; /* Values extracted from pExpr */ - Expr *pExpr = pLower->pExpr->pRight; - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n); - if( rc==SQLITE_OK && n ){ - tRowcnt iNew; - u16 mask = WO_GT|WO_LE; - if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); - iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); - iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0); - if( iNew>iLower ) iLower = iNew; - nOut--; - pLower = 0; - } - } - - /* If possible, improve on the iUpper estimate using ($P:$U). */ - if( pUpper ){ - int n; /* Values extracted from pExpr */ - Expr *pExpr = pUpper->pExpr->pRight; - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n); - if( rc==SQLITE_OK && n ){ - tRowcnt iNew; - u16 mask = WO_GT|WO_LE; - if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); - iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); - iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0); - if( iNewpRec = pRec; - if( rc==SQLITE_OK ){ - if( iUpper>iLower ){ - nNew = sqlite3LogEst(iUpper - iLower); - /* TUNING: If both iUpper and iLower are derived from the same - ** sample, then assume they are 4x more selective. This brings - ** the estimated selectivity more in line with what it would be - ** if estimated without the use of STAT3/4 tables. */ - if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); - }else{ - nNew = 10; assert( 10==sqlite3LogEst(2) ); - } - if( nNewwtFlags & TERM_VNULL)==0 ); - nNew = whereRangeAdjust(pLower, nOut); - nNew = whereRangeAdjust(pUpper, nNew); - - /* TUNING: If there is both an upper and lower limit and neither limit - ** has an application-defined likelihood(), assume the range is - ** reduced by an additional 75%. This means that, by default, an open-ended - ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the - ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to - ** match 1/64 of the index. */ - if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ - nNew -= 20; - } - - nOut -= (pLower!=0) + (pUpper!=0); - if( nNew<10 ) nNew = 10; - if( nNewnOut>nOut ){ - WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", - pLoop->nOut, nOut)); - } -#endif - pLoop->nOut = (LogEst)nOut; - return rc; -} - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** Estimate the number of rows that will be returned based on -** an equality constraint x=VALUE and where that VALUE occurs in -** the histogram data. This only works when x is the left-most -** column of an index and sqlite_stat3 histogram data is available -** for that index. When pExpr==NULL that means the constraint is -** "x IS NULL" instead of "x=VALUE". -** -** Write the estimated row count into *pnRow and return SQLITE_OK. -** If unable to make an estimate, leave *pnRow unchanged and return -** non-zero. -** -** This routine can fail if it is unable to load a collating sequence -** required for string comparison, or if unable to allocate memory -** for a UTF conversion required for comparison. The error is stored -** in the pParse structure. -*/ -static int whereEqualScanEst( - Parse *pParse, /* Parsing & code generating context */ - WhereLoopBuilder *pBuilder, - Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ - tRowcnt *pnRow /* Write the revised row estimate here */ -){ - Index *p = pBuilder->pNew->u.btree.pIndex; - int nEq = pBuilder->pNew->u.btree.nEq; - UnpackedRecord *pRec = pBuilder->pRec; - int rc; /* Subfunction return code */ - tRowcnt a[2]; /* Statistics */ - int bOk; - - assert( nEq>=1 ); - assert( nEq<=p->nColumn ); - assert( p->aSample!=0 ); - assert( p->nSample>0 ); - assert( pBuilder->nRecValidnRecValid<(nEq-1) ){ - return SQLITE_NOTFOUND; - } - - /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() - ** below would return the same value. */ - if( nEq>=p->nColumn ){ - *pnRow = 1; - return SQLITE_OK; - } - - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk); - pBuilder->pRec = pRec; - if( rc!=SQLITE_OK ) return rc; - if( bOk==0 ) return SQLITE_NOTFOUND; - pBuilder->nRecValid = nEq; - - whereKeyStats(pParse, p, pRec, 0, a); - WHERETRACE(0x10,("equality scan regions %s(%d): %d\n", - p->zName, nEq-1, (int)a[1])); - *pnRow = a[1]; - - return rc; -} -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** Estimate the number of rows that will be returned based on -** an IN constraint where the right-hand side of the IN operator -** is a list of values. Example: -** -** WHERE x IN (1,2,3,4) -** -** Write the estimated row count into *pnRow and return SQLITE_OK. -** If unable to make an estimate, leave *pnRow unchanged and return -** non-zero. -** -** This routine can fail if it is unable to load a collating sequence -** required for string comparison, or if unable to allocate memory -** for a UTF conversion required for comparison. The error is stored -** in the pParse structure. -*/ -static int whereInScanEst( - Parse *pParse, /* Parsing & code generating context */ - WhereLoopBuilder *pBuilder, - ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ - tRowcnt *pnRow /* Write the revised row estimate here */ -){ - Index *p = pBuilder->pNew->u.btree.pIndex; - i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); - int nRecValid = pBuilder->nRecValid; - int rc = SQLITE_OK; /* Subfunction return code */ - tRowcnt nEst; /* Number of rows for a single term */ - tRowcnt nRowEst = 0; /* New estimate of the number of rows */ - int i; /* Loop counter */ - - assert( p->aSample!=0 ); - for(i=0; rc==SQLITE_OK && inExpr; i++){ - nEst = nRow0; - rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); - nRowEst += nEst; - pBuilder->nRecValid = nRecValid; - } - - if( rc==SQLITE_OK ){ - if( nRowEst > nRow0 ) nRowEst = nRow0; - *pnRow = nRowEst; - WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); - } - assert( pBuilder->nRecValid==nRecValid ); - return rc; -} -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ - - -#ifdef WHERETRACE_ENABLED -/* -** Print the content of a WhereTerm object -*/ -static void whereTermPrint(WhereTerm *pTerm, int iTerm){ - if( pTerm==0 ){ - sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); - }else{ - char zType[4]; - char zLeft[50]; - memcpy(zType, "...", 4); - if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; - if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; - if( pTerm->eOperator & WO_SINGLE ){ - sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}", - pTerm->leftCursor, pTerm->u.leftColumn); - }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){ - sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%lld", - pTerm->u.pOrInfo->indexable); - }else{ - sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor); - } - sqlite3DebugPrintf( - "TERM-%-3d %p %s %-12s prob=%-3d op=0x%03x wtFlags=0x%04x", - iTerm, pTerm, zType, zLeft, pTerm->truthProb, - pTerm->eOperator, pTerm->wtFlags); - if( pTerm->iField ){ - sqlite3DebugPrintf(" iField=%d\n", pTerm->iField); - }else{ - sqlite3DebugPrintf("\n"); - } - sqlite3TreeViewExpr(0, pTerm->pExpr, 0); - } -} -#endif - -#ifdef WHERETRACE_ENABLED -/* -** Show the complete content of a WhereClause -*/ -SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC){ - int i; - for(i=0; inTerm; i++){ - whereTermPrint(&pWC->a[i], i); - } -} -#endif - -#ifdef WHERETRACE_ENABLED -/* -** Print a WhereLoop object for debugging purposes -*/ -static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ - WhereInfo *pWInfo = pWC->pWInfo; - int nb = 1+(pWInfo->pTabList->nSrc+3)/4; - struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pTab; - Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; - sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, - p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); - sqlite3DebugPrintf(" %12s", - pItem->zAlias ? pItem->zAlias : pTab->zName); - if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ - const char *zName; - if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ - if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ - int i = sqlite3Strlen30(zName) - 1; - while( zName[i]!='_' ) i--; - zName += i; - } - sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq); - }else{ - sqlite3DebugPrintf("%20s",""); - } - }else{ - char *z; - if( p->u.vtab.idxStr ){ - z = sqlite3_mprintf("(%d,\"%s\",%x)", - p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); - }else{ - z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); - } - sqlite3DebugPrintf(" %-19s", z); - sqlite3_free(z); - } - if( p->wsFlags & WHERE_SKIPSCAN ){ - sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); - }else{ - sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); - } - sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); - if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ - int i; - for(i=0; inLTerm; i++){ - whereTermPrint(p->aLTerm[i], i); - } - } -} -#endif - -/* -** Convert bulk memory into a valid WhereLoop that can be passed -** to whereLoopClear harmlessly. -*/ -static void whereLoopInit(WhereLoop *p){ - p->aLTerm = p->aLTermSpace; - p->nLTerm = 0; - p->nLSlot = ArraySize(p->aLTermSpace); - p->wsFlags = 0; -} - -/* -** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. -*/ -static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ - if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){ - if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){ - sqlite3_free(p->u.vtab.idxStr); - p->u.vtab.needFree = 0; - p->u.vtab.idxStr = 0; - }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ - sqlite3DbFree(db, p->u.btree.pIndex->zColAff); - sqlite3DbFree(db, p->u.btree.pIndex); - p->u.btree.pIndex = 0; - } - } -} - -/* -** Deallocate internal memory used by a WhereLoop object -*/ -static void whereLoopClear(sqlite3 *db, WhereLoop *p){ - if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); - whereLoopClearUnion(db, p); - whereLoopInit(p); -} - -/* -** Increase the memory allocation for pLoop->aLTerm[] to be at least n. -*/ -static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ - WhereTerm **paNew; - if( p->nLSlot>=n ) return SQLITE_OK; - n = (n+7)&~7; - paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n); - if( paNew==0 ) return SQLITE_NOMEM_BKPT; - memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); - if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); - p->aLTerm = paNew; - p->nLSlot = n; - return SQLITE_OK; -} - -/* -** Transfer content from the second pLoop into the first. -*/ -static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ - whereLoopClearUnion(db, pTo); - if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ - memset(&pTo->u, 0, sizeof(pTo->u)); - return SQLITE_NOMEM_BKPT; - } - memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); - memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); - if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ - pFrom->u.vtab.needFree = 0; - }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){ - pFrom->u.btree.pIndex = 0; - } - return SQLITE_OK; -} - -/* -** Delete a WhereLoop object -*/ -static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ - whereLoopClear(db, p); - sqlite3DbFree(db, p); -} - -/* -** Free a WhereInfo structure -*/ -static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ - if( ALWAYS(pWInfo) ){ - int i; - for(i=0; inLevel; i++){ - WhereLevel *pLevel = &pWInfo->a[i]; - if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){ - sqlite3DbFree(db, pLevel->u.in.aInLoop); - } - } - sqlite3WhereClauseClear(&pWInfo->sWC); - while( pWInfo->pLoops ){ - WhereLoop *p = pWInfo->pLoops; - pWInfo->pLoops = p->pNextLoop; - whereLoopDelete(db, p); - } - sqlite3DbFree(db, pWInfo); - } -} - -/* -** Return TRUE if all of the following are true: -** -** (1) X has the same or lower cost that Y -** (2) X is a proper subset of Y -** (3) X skips at least as many columns as Y -** -** By "proper subset" we mean that X uses fewer WHERE clause terms -** than Y and that every WHERE clause term used by X is also used -** by Y. -** -** If X is a proper subset of Y then Y is a better choice and ought -** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. The third rule -** was added because if X uses skip-scan less than Y it still might -** deserve a lower cost even if it is a proper subset of Y. -*/ -static int whereLoopCheaperProperSubset( - const WhereLoop *pX, /* First WhereLoop to compare */ - const WhereLoop *pY /* Compare against this WhereLoop */ -){ - int i, j; - if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ - return 0; /* X is not a subset of Y */ - } - if( pY->nSkip > pX->nSkip ) return 0; - if( pX->rRun >= pY->rRun ){ - if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ - if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ - } - for(i=pX->nLTerm-1; i>=0; i--){ - if( pX->aLTerm[i]==0 ) continue; - for(j=pY->nLTerm-1; j>=0; j--){ - if( pY->aLTerm[j]==pX->aLTerm[i] ) break; - } - if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ - } - return 1; /* All conditions meet */ -} - -/* -** Try to adjust the cost of WhereLoop pTemplate upwards or downwards so -** that: -** -** (1) pTemplate costs less than any other WhereLoops that are a proper -** subset of pTemplate -** -** (2) pTemplate costs more than any other WhereLoops for which pTemplate -** is a proper subset. -** -** To say "WhereLoop X is a proper subset of Y" means that X uses fewer -** WHERE clause terms than Y and that every WHERE clause term used by X is -** also used by Y. -*/ -static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ - if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; - for(; p; p=p->pNextLoop){ - if( p->iTab!=pTemplate->iTab ) continue; - if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; - if( whereLoopCheaperProperSubset(p, pTemplate) ){ - /* Adjust pTemplate cost downward so that it is cheaper than its - ** subset p. */ - WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", - pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); - pTemplate->rRun = p->rRun; - pTemplate->nOut = p->nOut - 1; - }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ - /* Adjust pTemplate cost upward so that it is costlier than p since - ** pTemplate is a proper subset of p */ - WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", - pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); - pTemplate->rRun = p->rRun; - pTemplate->nOut = p->nOut + 1; - } - } -} - -/* -** Search the list of WhereLoops in *ppPrev looking for one that can be -** supplanted by pTemplate. -** -** Return NULL if the WhereLoop list contains an entry that can supplant -** pTemplate, in other words if pTemplate does not belong on the list. -** -** If pX is a WhereLoop that pTemplate can supplant, then return the -** link that points to pX. -** -** If pTemplate cannot supplant any existing element of the list but needs -** to be added to the list, then return a pointer to the tail of the list. -*/ -static WhereLoop **whereLoopFindLesser( - WhereLoop **ppPrev, - const WhereLoop *pTemplate -){ - WhereLoop *p; - for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ - if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ - /* If either the iTab or iSortIdx values for two WhereLoop are different - ** then those WhereLoops need to be considered separately. Neither is - ** a candidate to replace the other. */ - continue; - } - /* In the current implementation, the rSetup value is either zero - ** or the cost of building an automatic index (NlogN) and the NlogN - ** is the same for compatible WhereLoops. */ - assert( p->rSetup==0 || pTemplate->rSetup==0 - || p->rSetup==pTemplate->rSetup ); - - /* whereLoopAddBtree() always generates and inserts the automatic index - ** case first. Hence compatible candidate WhereLoops never have a larger - ** rSetup. Call this SETUP-INVARIANT */ - assert( p->rSetup>=pTemplate->rSetup ); - - /* Any loop using an appliation-defined index (or PRIMARY KEY or - ** UNIQUE constraint) with one or more == constraints is better - ** than an automatic index. Unless it is a skip-scan. */ - if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 - && (pTemplate->nSkip)==0 - && (pTemplate->wsFlags & WHERE_INDEXED)!=0 - && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 - && (p->prereq & pTemplate->prereq)==pTemplate->prereq - ){ - break; - } - - /* If existing WhereLoop p is better than pTemplate, pTemplate can be - ** discarded. WhereLoop p is better if: - ** (1) p has no more dependencies than pTemplate, and - ** (2) p has an equal or lower cost than pTemplate - */ - if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */ - && p->rSetup<=pTemplate->rSetup /* (2a) */ - && p->rRun<=pTemplate->rRun /* (2b) */ - && p->nOut<=pTemplate->nOut /* (2c) */ - ){ - return 0; /* Discard pTemplate */ - } - - /* If pTemplate is always better than p, then cause p to be overwritten - ** with pTemplate. pTemplate is better than p if: - ** (1) pTemplate has no more dependences than p, and - ** (2) pTemplate has an equal or lower cost than p. - */ - if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ - && p->rRun>=pTemplate->rRun /* (2a) */ - && p->nOut>=pTemplate->nOut /* (2b) */ - ){ - assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ - break; /* Cause p to be overwritten by pTemplate */ - } - } - return ppPrev; -} - -/* -** Insert or replace a WhereLoop entry using the template supplied. -** -** An existing WhereLoop entry might be overwritten if the new template -** is better and has fewer dependencies. Or the template will be ignored -** and no insert will occur if an existing WhereLoop is faster and has -** fewer dependencies than the template. Otherwise a new WhereLoop is -** added based on the template. -** -** If pBuilder->pOrSet is not NULL then we care about only the -** prerequisites and rRun and nOut costs of the N best loops. That -** information is gathered in the pBuilder->pOrSet object. This special -** processing mode is used only for OR clause processing. -** -** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we -** still might overwrite similar loops with the new template if the -** new template is better. Loops may be overwritten if the following -** conditions are met: -** -** (1) They have the same iTab. -** (2) They have the same iSortIdx. -** (3) The template has same or fewer dependencies than the current loop -** (4) The template has the same or lower cost than the current loop -*/ -static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ - WhereLoop **ppPrev, *p; - WhereInfo *pWInfo = pBuilder->pWInfo; - sqlite3 *db = pWInfo->pParse->db; - int rc; - - /* If pBuilder->pOrSet is defined, then only keep track of the costs - ** and prereqs. - */ - if( pBuilder->pOrSet!=0 ){ - if( pTemplate->nLTerm ){ -#if WHERETRACE_ENABLED - u16 n = pBuilder->pOrSet->n; - int x = -#endif - whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, - pTemplate->nOut); -#if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); - whereLoopPrint(pTemplate, pBuilder->pWC); - } -#endif - } - return SQLITE_OK; - } - - /* Look for an existing WhereLoop to replace with pTemplate - */ - whereLoopAdjustCost(pWInfo->pLoops, pTemplate); - ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); - - if( ppPrev==0 ){ - /* There already exists a WhereLoop on the list that is better - ** than pTemplate, so just ignore pTemplate */ -#if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(" skip: "); - whereLoopPrint(pTemplate, pBuilder->pWC); - } -#endif - return SQLITE_OK; - }else{ - p = *ppPrev; - } - - /* If we reach this point it means that either p[] should be overwritten - ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new - ** WhereLoop and insert it. - */ -#if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - if( p!=0 ){ - sqlite3DebugPrintf("replace: "); - whereLoopPrint(p, pBuilder->pWC); - } - sqlite3DebugPrintf(" add: "); - whereLoopPrint(pTemplate, pBuilder->pWC); - } -#endif - if( p==0 ){ - /* Allocate a new WhereLoop to add to the end of the list */ - *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop)); - if( p==0 ) return SQLITE_NOMEM_BKPT; - whereLoopInit(p); - p->pNextLoop = 0; - }else{ - /* We will be overwriting WhereLoop p[]. But before we do, first - ** go through the rest of the list and delete any other entries besides - ** p[] that are also supplated by pTemplate */ - WhereLoop **ppTail = &p->pNextLoop; - WhereLoop *pToDel; - while( *ppTail ){ - ppTail = whereLoopFindLesser(ppTail, pTemplate); - if( ppTail==0 ) break; - pToDel = *ppTail; - if( pToDel==0 ) break; - *ppTail = pToDel->pNextLoop; -#if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(" delete: "); - whereLoopPrint(pToDel, pBuilder->pWC); - } -#endif - whereLoopDelete(db, pToDel); - } - } - rc = whereLoopXfer(db, p, pTemplate); - if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ - Index *pIndex = p->u.btree.pIndex; - if( pIndex && pIndex->tnum==0 ){ - p->u.btree.pIndex = 0; - } - } - return rc; -} - -/* -** Adjust the WhereLoop.nOut value downward to account for terms of the -** WHERE clause that reference the loop but which are not used by an -** index. -* -** For every WHERE clause term that is not used by the index -** and which has a truth probability assigned by one of the likelihood(), -** likely(), or unlikely() SQL functions, reduce the estimated number -** of output rows by the probability specified. -** -** TUNING: For every WHERE clause term that is not used by the index -** and which does not have an assigned truth probability, heuristics -** described below are used to try to estimate the truth probability. -** TODO --> Perhaps this is something that could be improved by better -** table statistics. -** -** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75% -** value corresponds to -1 in LogEst notation, so this means decrement -** the WhereLoop.nOut field for every such WHERE clause term. -** -** Heuristic 2: If there exists one or more WHERE clause terms of the -** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the -** final output row estimate is no greater than 1/4 of the total number -** of rows in the table. In other words, assume that x==EXPR will filter -** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the -** "x" column is boolean or else -1 or 0 or 1 is a common default value -** on the "x" column and so in that case only cap the output row estimate -** at 1/2 instead of 1/4. -*/ -static void whereLoopOutputAdjust( - WhereClause *pWC, /* The WHERE clause */ - WhereLoop *pLoop, /* The loop to adjust downward */ - LogEst nRow /* Number of rows in the entire table */ -){ - WhereTerm *pTerm, *pX; - Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); - int i, j, k; - LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ - - assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); - for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ - if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; - if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; - if( (pTerm->prereqAll & notAllowed)!=0 ) continue; - for(j=pLoop->nLTerm-1; j>=0; j--){ - pX = pLoop->aLTerm[j]; - if( pX==0 ) continue; - if( pX==pTerm ) break; - if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; - } - if( j<0 ){ - if( pTerm->truthProb<=0 ){ - /* If a truth probability is specified using the likelihood() hints, - ** then use the probability provided by the application. */ - pLoop->nOut += pTerm->truthProb; - }else{ - /* In the absence of explicit truth probabilities, use heuristics to - ** guess a reasonable truth probability. */ - pLoop->nOut--; - if( pTerm->eOperator&(WO_EQ|WO_IS) ){ - Expr *pRight = pTerm->pExpr->pRight; - testcase( pTerm->pExpr->op==TK_IS ); - if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ - k = 10; - }else{ - k = 20; - } - if( iReducenOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce; -} - -/* -** Term pTerm is a vector range comparison operation. The first comparison -** in the vector can be optimized using column nEq of the index. This -** function returns the total number of vector elements that can be used -** as part of the range comparison. -** -** For example, if the query is: -** -** WHERE a = ? AND (b, c, d) > (?, ?, ?) -** -** and the index: -** -** CREATE INDEX ... ON (a, b, c, d, e) -** -** then this function would be invoked with nEq=1. The value returned in -** this case is 3. -*/ -static int whereRangeVectorLen( - Parse *pParse, /* Parsing context */ - int iCur, /* Cursor open on pIdx */ - Index *pIdx, /* The index to be used for a inequality constraint */ - int nEq, /* Number of prior equality constraints on same index */ - WhereTerm *pTerm /* The vector inequality constraint */ -){ - int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft); - int i; - - nCmp = MIN(nCmp, (pIdx->nColumn - nEq)); - for(i=1; ipExpr->pLeft->x.pList->a[i].pExpr; - Expr *pRhs = pTerm->pExpr->pRight; - if( pRhs->flags & EP_xIsSelect ){ - pRhs = pRhs->x.pSelect->pEList->a[i].pExpr; - }else{ - pRhs = pRhs->x.pList->a[i].pExpr; - } - - /* Check that the LHS of the comparison is a column reference to - ** the right column of the right source table. And that the sort - ** order of the index column is the same as the sort order of the - ** leftmost index column. */ - if( pLhs->op!=TK_COLUMN - || pLhs->iTable!=iCur - || pLhs->iColumn!=pIdx->aiColumn[i+nEq] - || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq] - ){ - break; - } - - testcase( pLhs->iColumn==XN_ROWID ); - aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs)); - idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn); - if( aff!=idxaff ) break; - - pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); - if( pColl==0 ) break; - if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break; - } - return i; -} - -/* -** Adjust the cost C by the costMult facter T. This only occurs if -** compiled with -DSQLITE_ENABLE_COSTMULT -*/ -#ifdef SQLITE_ENABLE_COSTMULT -# define ApplyCostMultiplier(C,T) C += T -#else -# define ApplyCostMultiplier(C,T) -#endif - -/* -** We have so far matched pBuilder->pNew->u.btree.nEq terms of the -** index pIndex. Try to match one more. -** -** When this function is called, pBuilder->pNew->nOut contains the -** number of rows expected to be visited by filtering using the nEq -** terms only. If it is modified, this value is restored before this -** function returns. -** -** If pProbe->tnum==0, that means pIndex is a fake index used for the -** INTEGER PRIMARY KEY. -*/ -static int whereLoopAddBtreeIndex( - WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ - struct SrcList_item *pSrc, /* FROM clause term being analyzed */ - Index *pProbe, /* An index on pSrc */ - LogEst nInMul /* log(Number of iterations due to IN) */ -){ - WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ - Parse *pParse = pWInfo->pParse; /* Parsing context */ - sqlite3 *db = pParse->db; /* Database connection malloc context */ - WhereLoop *pNew; /* Template WhereLoop under construction */ - WhereTerm *pTerm; /* A WhereTerm under consideration */ - int opMask; /* Valid operators for constraints */ - WhereScan scan; /* Iterator for WHERE terms */ - Bitmask saved_prereq; /* Original value of pNew->prereq */ - u16 saved_nLTerm; /* Original value of pNew->nLTerm */ - u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ - u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */ - u16 saved_nTop; /* Original value of pNew->u.btree.nTop */ - u16 saved_nSkip; /* Original value of pNew->nSkip */ - u32 saved_wsFlags; /* Original value of pNew->wsFlags */ - LogEst saved_nOut; /* Original value of pNew->nOut */ - int rc = SQLITE_OK; /* Return code */ - LogEst rSize; /* Number of rows in the table */ - LogEst rLogSize; /* Logarithm of table size */ - WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ - - pNew = pBuilder->pNew; - if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; - WHERETRACE(0x800, ("BEGIN addBtreeIdx(%s), nEq=%d\n", - pProbe->zName, pNew->u.btree.nEq)); - - assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); - assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); - if( pNew->wsFlags & WHERE_BTM_LIMIT ){ - opMask = WO_LT|WO_LE; - }else{ - assert( pNew->u.btree.nBtm==0 ); - opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; - } - if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); - - assert( pNew->u.btree.nEqnColumn ); - - saved_nEq = pNew->u.btree.nEq; - saved_nBtm = pNew->u.btree.nBtm; - saved_nTop = pNew->u.btree.nTop; - saved_nSkip = pNew->nSkip; - saved_nLTerm = pNew->nLTerm; - saved_wsFlags = pNew->wsFlags; - saved_prereq = pNew->prereq; - saved_nOut = pNew->nOut; - pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq, - opMask, pProbe); - pNew->rSetup = 0; - rSize = pProbe->aiRowLogEst[0]; - rLogSize = estLog(rSize); - for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ - u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ - LogEst rCostIdx; - LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ - int nIn = 0; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - int nRecValid = pBuilder->nRecValid; -#endif - if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) - && indexColumnNotNull(pProbe, saved_nEq) - ){ - continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ - } - if( pTerm->prereqRight & pNew->maskSelf ) continue; - - /* Do not allow the upper bound of a LIKE optimization range constraint - ** to mix with a lower range bound from some other source */ - if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; - - /* Do not allow IS constraints from the WHERE clause to be used by the - ** right table of a LEFT JOIN. Only constraints in the ON clause are - ** allowed */ - if( (pSrc->fg.jointype & JT_LEFT)!=0 - && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) - && (eOp & (WO_IS|WO_ISNULL))!=0 - ){ - testcase( eOp & WO_IS ); - testcase( eOp & WO_ISNULL ); - continue; - } - - pNew->wsFlags = saved_wsFlags; - pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nBtm = saved_nBtm; - pNew->u.btree.nTop = saved_nTop; - pNew->nLTerm = saved_nLTerm; - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ - pNew->aLTerm[pNew->nLTerm++] = pTerm; - pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; - - assert( nInMul==0 - || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 - || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 - || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 - ); - - if( eOp & WO_IN ){ - Expr *pExpr = pTerm->pExpr; - pNew->wsFlags |= WHERE_COLUMN_IN; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ - int i; - nIn = 46; assert( 46==sqlite3LogEst(25) ); - - /* The expression may actually be of the form (x, y) IN (SELECT...). - ** In this case there is a separate term for each of (x) and (y). - ** However, the nIn multiplier should only be applied once, not once - ** for each such term. The following loop checks that pTerm is the - ** first such term in use, and sets nIn back to 0 if it is not. */ - for(i=0; inLTerm-1; i++){ - if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0; - } - }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ - /* "x IN (value, value, ...)" */ - nIn = sqlite3LogEst(pExpr->x.pList->nExpr); - assert( nIn>0 ); /* RHS always has 2 or more terms... The parser - ** changes "x IN (?)" into "x=?". */ - } - }else if( eOp & (WO_EQ|WO_IS) ){ - int iCol = pProbe->aiColumn[saved_nEq]; - pNew->wsFlags |= WHERE_COLUMN_EQ; - assert( saved_nEq==pNew->u.btree.nEq ); - if( iCol==XN_ROWID - || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) - ){ - if( iCol>=0 && pProbe->uniqNotNull==0 ){ - pNew->wsFlags |= WHERE_UNQ_WANTED; - }else{ - pNew->wsFlags |= WHERE_ONEROW; - } - } - }else if( eOp & WO_ISNULL ){ - pNew->wsFlags |= WHERE_COLUMN_NULL; - }else if( eOp & (WO_GT|WO_GE) ){ - testcase( eOp & WO_GT ); - testcase( eOp & WO_GE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; - pNew->u.btree.nBtm = whereRangeVectorLen( - pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm - ); - pBtm = pTerm; - pTop = 0; - if( pTerm->wtFlags & TERM_LIKEOPT ){ - /* Range contraints that come from the LIKE optimization are - ** always used in pairs. */ - pTop = &pTerm[1]; - assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); - assert( pTop->wtFlags & TERM_LIKEOPT ); - assert( pTop->eOperator==WO_LT ); - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ - pNew->aLTerm[pNew->nLTerm++] = pTop; - pNew->wsFlags |= WHERE_TOP_LIMIT; - pNew->u.btree.nTop = 1; - } - }else{ - assert( eOp & (WO_LT|WO_LE) ); - testcase( eOp & WO_LT ); - testcase( eOp & WO_LE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; - pNew->u.btree.nTop = whereRangeVectorLen( - pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm - ); - pTop = pTerm; - pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? - pNew->aLTerm[pNew->nLTerm-2] : 0; - } - - /* At this point pNew->nOut is set to the number of rows expected to - ** be visited by the index scan before considering term pTerm, or the - ** values of nIn and nInMul. In other words, assuming that all - ** "x IN(...)" terms are replaced with "x = ?". This block updates - ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ - assert( pNew->nOut==saved_nOut ); - if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ - /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 - ** data, using some other estimate. */ - whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); - }else{ - int nEq = ++pNew->u.btree.nEq; - assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) ); - - assert( pNew->nOut==saved_nOut ); - if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){ - assert( (eOp & WO_IN) || nIn==0 ); - testcase( eOp & WO_IN ); - pNew->nOut += pTerm->truthProb; - pNew->nOut -= nIn; - }else{ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - tRowcnt nOut = 0; - if( nInMul==0 - && pProbe->nSample - && pNew->u.btree.nEq<=pProbe->nSampleCol - && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) - ){ - Expr *pExpr = pTerm->pExpr; - if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ - testcase( eOp & WO_EQ ); - testcase( eOp & WO_IS ); - testcase( eOp & WO_ISNULL ); - rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); - }else{ - rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); - } - if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; - if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ - if( nOut ){ - pNew->nOut = sqlite3LogEst(nOut); - if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; - pNew->nOut -= nIn; - } - } - if( nOut==0 ) -#endif - { - pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); - if( eOp & WO_ISNULL ){ - /* TUNING: If there is no likelihood() value, assume that a - ** "col IS NULL" expression matches twice as many rows - ** as (col=?). */ - pNew->nOut += 10; - } - } - } - } - - /* Set rCostIdx to the cost of visiting selected rows in index. Add - ** it to pNew->rRun, which is currently set to the cost of the index - ** seek only. Then, if this is a non-covering index, add the cost of - ** visiting the rows in the main table. */ - rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; - pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); - if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); - } - ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); - - nOutUnadjusted = pNew->nOut; - pNew->rRun += nInMul + nIn; - pNew->nOut += nInMul + nIn; - whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); - rc = whereLoopInsert(pBuilder, pNew); - - if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ - pNew->nOut = saved_nOut; - }else{ - pNew->nOut = nOutUnadjusted; - } - - if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 - && pNew->u.btree.nEqnColumn - ){ - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); - } - pNew->nOut = saved_nOut; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - pBuilder->nRecValid = nRecValid; -#endif - } - pNew->prereq = saved_prereq; - pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nBtm = saved_nBtm; - pNew->u.btree.nTop = saved_nTop; - pNew->nSkip = saved_nSkip; - pNew->wsFlags = saved_wsFlags; - pNew->nOut = saved_nOut; - pNew->nLTerm = saved_nLTerm; - - /* Consider using a skip-scan if there are no WHERE clause constraints - ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. - ** - ** The magic number 18 is selected on the basis that scanning 17 rows - ** is almost always quicker than an index seek (even though if the index - ** contains fewer than 2^17 rows we assume otherwise in other parts of - ** the code). And, even if it is not, it should not be too much slower. - ** On the other hand, the extra seeks could end up being significantly - ** more expensive. */ - assert( 42==sqlite3LogEst(18) ); - if( saved_nEq==saved_nSkip - && saved_nEq+1nKeyCol - && pProbe->noSkipScan==0 - && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ - && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK - ){ - LogEst nIter; - pNew->u.btree.nEq++; - pNew->nSkip++; - pNew->aLTerm[pNew->nLTerm++] = 0; - pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - pNew->nOut -= nIter; - /* TUNING: Because uncertainties in the estimates for skip-scan queries, - ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ - nIter += 5; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); - pNew->nOut = saved_nOut; - pNew->u.btree.nEq = saved_nEq; - pNew->nSkip = saved_nSkip; - pNew->wsFlags = saved_wsFlags; - } - - WHERETRACE(0x800, ("END addBtreeIdx(%s), nEq=%d, rc=%d\n", - pProbe->zName, saved_nEq, rc)); - return rc; -} - -/* -** Return True if it is possible that pIndex might be useful in -** implementing the ORDER BY clause in pBuilder. -** -** Return False if pBuilder does not contain an ORDER BY clause or -** if there is no way for pIndex to be useful in implementing that -** ORDER BY clause. -*/ -static int indexMightHelpWithOrderBy( - WhereLoopBuilder *pBuilder, - Index *pIndex, - int iCursor -){ - ExprList *pOB; - ExprList *aColExpr; - int ii, jj; - - if( pIndex->bUnordered ) return 0; - if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; - for(ii=0; iinExpr; ii++){ - Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); - if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ - if( pExpr->iColumn<0 ) return 1; - for(jj=0; jjnKeyCol; jj++){ - if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; - } - }else if( (aColExpr = pIndex->aColExpr)!=0 ){ - for(jj=0; jjnKeyCol; jj++){ - if( pIndex->aiColumn[jj]!=XN_EXPR ) continue; - if( sqlite3ExprCompare(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){ - return 1; - } - } - } - } - return 0; -} - -/* -** Return a bitmask where 1s indicate that the corresponding column of -** the table is used by an index. Only the first 63 columns are considered. -*/ -static Bitmask columnsInIndex(Index *pIdx){ - Bitmask m = 0; - int j; - for(j=pIdx->nColumn-1; j>=0; j--){ - int x = pIdx->aiColumn[j]; - if( x>=0 ){ - testcase( x==BMS-1 ); - testcase( x==BMS-2 ); - if( xop==TK_AND ){ - if( !whereUsablePartialIndex(iTab,pWC,pWhere->pLeft) ) return 0; - pWhere = pWhere->pRight; - } - for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - Expr *pExpr = pTerm->pExpr; - if( sqlite3ExprImpliesExpr(pExpr, pWhere, iTab) - && (!ExprHasProperty(pExpr, EP_FromJoin) || pExpr->iRightJoinTable==iTab) - ){ - return 1; - } - } - return 0; -} - -/* -** Add all WhereLoop objects for a single table of the join where the table -** is identified by pBuilder->pNew->iTab. That table is guaranteed to be -** a b-tree table, not a virtual table. -** -** The costs (WhereLoop.rRun) of the b-tree loops added by this function -** are calculated as follows: -** -** For a full scan, assuming the table (or index) contains nRow rows: -** -** cost = nRow * 3.0 // full-table scan -** cost = nRow * K // scan of covering index -** cost = nRow * (K+3.0) // scan of non-covering index -** -** where K is a value between 1.1 and 3.0 set based on the relative -** estimated average size of the index and table records. -** -** For an index scan, where nVisit is the number of index rows visited -** by the scan, and nSeek is the number of seek operations required on -** the index b-tree: -** -** cost = nSeek * (log(nRow) + K * nVisit) // covering index -** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index -** -** Normally, nSeek is 1. nSeek values greater than 1 come about if the -** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when -** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. -** -** The estimated values (nRow, nVisit, nSeek) often contain a large amount -** of uncertainty. For this reason, scoring is designed to pick plans that -** "do the least harm" if the estimates are inaccurate. For example, a -** log(nRow) factor is omitted from a non-covering index scan in order to -** bias the scoring in favor of using an index, since the worst-case -** performance of using an index is far better than the worst-case performance -** of a full table scan. -*/ -static int whereLoopAddBtree( - WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mPrereq /* Extra prerequesites for using this table */ -){ - WhereInfo *pWInfo; /* WHERE analysis context */ - Index *pProbe; /* An index we are evaluating */ - Index sPk; /* A fake index object for the primary key */ - LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ - i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ - SrcList *pTabList; /* The FROM clause */ - struct SrcList_item *pSrc; /* The FROM clause btree term to add */ - WhereLoop *pNew; /* Template WhereLoop object */ - int rc = SQLITE_OK; /* Return code */ - int iSortIdx = 1; /* Index number */ - int b; /* A boolean value */ - LogEst rSize; /* number of rows in the table */ - LogEst rLogSize; /* Logarithm of the number of rows in the table */ - WhereClause *pWC; /* The parsed WHERE clause */ - Table *pTab; /* Table being queried */ - - pNew = pBuilder->pNew; - pWInfo = pBuilder->pWInfo; - pTabList = pWInfo->pTabList; - pSrc = pTabList->a + pNew->iTab; - pTab = pSrc->pTab; - pWC = pBuilder->pWC; - assert( !IsVirtual(pSrc->pTab) ); - - if( pSrc->pIBIndex ){ - /* An INDEXED BY clause specifies a particular index to use */ - pProbe = pSrc->pIBIndex; - }else if( !HasRowid(pTab) ){ - pProbe = pTab->pIndex; - }else{ - /* There is no INDEXED BY clause. Create a fake Index object in local - ** variable sPk to represent the rowid primary key index. Make this - ** fake index the first in a chain of Index objects with all of the real - ** indices to follow */ - Index *pFirst; /* First of real indices on the table */ - memset(&sPk, 0, sizeof(Index)); - sPk.nKeyCol = 1; - sPk.nColumn = 1; - sPk.aiColumn = &aiColumnPk; - sPk.aiRowLogEst = aiRowEstPk; - sPk.onError = OE_Replace; - sPk.pTable = pTab; - sPk.szIdxRow = pTab->szTabRow; - aiRowEstPk[0] = pTab->nRowLogEst; - aiRowEstPk[1] = 0; - pFirst = pSrc->pTab->pIndex; - if( pSrc->fg.notIndexed==0 ){ - /* The real indices of the table are only considered if the - ** NOT INDEXED qualifier is omitted from the FROM clause */ - sPk.pNext = pFirst; - } - pProbe = &sPk; - } - rSize = pTab->nRowLogEst; - rLogSize = estLog(rSize); - -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX - /* Automatic indexes */ - if( !pBuilder->pOrSet /* Not part of an OR optimization */ - && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 - && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 - && pSrc->pIBIndex==0 /* Has no INDEXED BY clause */ - && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ - && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ - && !pSrc->fg.isCorrelated /* Not a correlated subquery */ - && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ - ){ - /* Generate auto-index WhereLoops */ - WhereTerm *pTerm; - WhereTerm *pWCEnd = pWC->a + pWC->nTerm; - for(pTerm=pWC->a; rc==SQLITE_OK && pTermprereqRight & pNew->maskSelf ) continue; - if( termCanDriveIndex(pTerm, pSrc, 0) ){ - pNew->u.btree.nEq = 1; - pNew->nSkip = 0; - pNew->u.btree.pIndex = 0; - pNew->nLTerm = 1; - pNew->aLTerm[0] = pTerm; - /* TUNING: One-time cost for computing the automatic index is - ** estimated to be X*N*log2(N) where N is the number of rows in - ** the table being indexed and where X is 7 (LogEst=28) for normal - ** tables or 1.375 (LogEst=4) for views and subqueries. The value - ** of X is smaller for views and subqueries so that the query planner - ** will be more aggressive about generating automatic indexes for - ** those objects, since there is no opportunity to add schema - ** indexes on subqueries and views. */ - pNew->rSetup = rLogSize + rSize + 4; - if( pTab->pSelect==0 && (pTab->tabFlags & TF_Ephemeral)==0 ){ - pNew->rSetup += 24; - } - ApplyCostMultiplier(pNew->rSetup, pTab->costMult); - if( pNew->rSetup<0 ) pNew->rSetup = 0; - /* TUNING: Each index lookup yields 20 rows in the table. This - ** is more than the usual guess of 10 rows, since we have no way - ** of knowing how selective the index will ultimately be. It would - ** not be unreasonable to make this value much larger. */ - pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); - pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); - pNew->wsFlags = WHERE_AUTO_INDEX; - pNew->prereq = mPrereq | pTerm->prereqRight; - rc = whereLoopInsert(pBuilder, pNew); - } - } - } -#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ - - /* Loop over all indices - */ - for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ - if( pProbe->pPartIdxWhere!=0 - && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){ - testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ - continue; /* Partial index inappropriate for this query */ - } - rSize = pProbe->aiRowLogEst[0]; - pNew->u.btree.nEq = 0; - pNew->u.btree.nBtm = 0; - pNew->u.btree.nTop = 0; - pNew->nSkip = 0; - pNew->nLTerm = 0; - pNew->iSortIdx = 0; - pNew->rSetup = 0; - pNew->prereq = mPrereq; - pNew->nOut = rSize; - pNew->u.btree.pIndex = pProbe; - b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); - /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ - assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 ); - if( pProbe->tnum<=0 ){ - /* Integer primary key index */ - pNew->wsFlags = WHERE_IPK; - - /* Full table scan */ - pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: Cost of full table scan is (N*3.0). */ - pNew->rRun = rSize + 16; - ApplyCostMultiplier(pNew->rRun, pTab->costMult); - whereLoopOutputAdjust(pWC, pNew, rSize); - rc = whereLoopInsert(pBuilder, pNew); - pNew->nOut = rSize; - if( rc ) break; - }else{ - Bitmask m; - if( pProbe->isCovering ){ - pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; - m = 0; - }else{ - m = pSrc->colUsed & ~columnsInIndex(pProbe); - pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; - } - - /* Full scan via index */ - if( b - || !HasRowid(pTab) - || pProbe->pPartIdxWhere!=0 - || ( m==0 - && pProbe->bUnordered==0 - && (pProbe->szIdxRowszTabRow) - && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 - && sqlite3GlobalConfig.bUseCis - && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan) - ) - ){ - pNew->iSortIdx = b ? iSortIdx : 0; - - /* The cost of visiting the index rows is N*K, where K is - ** between 1.1 and 3.0, depending on the relative sizes of the - ** index and table rows. */ - pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; - if( m!=0 ){ - /* If this is a non-covering index scan, add in the cost of - ** doing table lookups. The cost will be 3x the number of - ** lookups. Take into account WHERE clause terms that can be - ** satisfied using just the index, and that do not require a - ** table lookup. */ - LogEst nLookup = rSize + 16; /* Base cost: N*3 */ - int ii; - int iCur = pSrc->iCursor; - WhereClause *pWC2 = &pWInfo->sWC; - for(ii=0; iinTerm; ii++){ - WhereTerm *pTerm = &pWC2->a[ii]; - if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){ - break; - } - /* pTerm can be evaluated using just the index. So reduce - ** the expected number of table lookups accordingly */ - if( pTerm->truthProb<=0 ){ - nLookup += pTerm->truthProb; - }else{ - nLookup--; - if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19; - } - } - - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup); - } - ApplyCostMultiplier(pNew->rRun, pTab->costMult); - whereLoopOutputAdjust(pWC, pNew, rSize); - rc = whereLoopInsert(pBuilder, pNew); - pNew->nOut = rSize; - if( rc ) break; - } - } - - rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - sqlite3Stat4ProbeFree(pBuilder->pRec); - pBuilder->nRecValid = 0; - pBuilder->pRec = 0; -#endif - - /* If there was an INDEXED BY clause, then only that one index is - ** considered. */ - if( pSrc->pIBIndex ) break; - } - return rc; -} - -#ifndef SQLITE_OMIT_VIRTUALTABLE - -/* -** Argument pIdxInfo is already populated with all constraints that may -** be used by the virtual table identified by pBuilder->pNew->iTab. This -** function marks a subset of those constraints usable, invokes the -** xBestIndex method and adds the returned plan to pBuilder. -** -** A constraint is marked usable if: -** -** * Argument mUsable indicates that its prerequisites are available, and -** -** * It is not one of the operators specified in the mExclude mask passed -** as the fourth argument (which in practice is either WO_IN or 0). -** -** Argument mPrereq is a mask of tables that must be scanned before the -** virtual table in question. These are added to the plans prerequisites -** before it is added to pBuilder. -** -** Output parameter *pbIn is set to true if the plan added to pBuilder -** uses one or more WO_IN terms, or false otherwise. -*/ -static int whereLoopAddVirtualOne( - WhereLoopBuilder *pBuilder, - Bitmask mPrereq, /* Mask of tables that must be used. */ - Bitmask mUsable, /* Mask of usable tables */ - u16 mExclude, /* Exclude terms using these operators */ - sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */ - u16 mNoOmit, /* Do not omit these constraints */ - int *pbIn /* OUT: True if plan uses an IN(...) op */ -){ - WhereClause *pWC = pBuilder->pWC; - struct sqlite3_index_constraint *pIdxCons; - struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage; - int i; - int mxTerm; - int rc = SQLITE_OK; - WhereLoop *pNew = pBuilder->pNew; - Parse *pParse = pBuilder->pWInfo->pParse; - struct SrcList_item *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab]; - int nConstraint = pIdxInfo->nConstraint; - - assert( (mUsable & mPrereq)==mPrereq ); - *pbIn = 0; - pNew->prereq = mPrereq; - - /* Set the usable flag on the subset of constraints identified by - ** arguments mUsable and mExclude. */ - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - for(i=0; ia[pIdxCons->iTermOffset]; - pIdxCons->usable = 0; - if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight - && (pTerm->eOperator & mExclude)==0 - ){ - pIdxCons->usable = 1; - } - } - - /* Initialize the output fields of the sqlite3_index_info structure */ - memset(pUsage, 0, sizeof(pUsage[0])*nConstraint); - assert( pIdxInfo->needToFreeIdxStr==0 ); - pIdxInfo->idxStr = 0; - pIdxInfo->idxNum = 0; - pIdxInfo->orderByConsumed = 0; - pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; - pIdxInfo->estimatedRows = 25; - pIdxInfo->idxFlags = 0; - pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; - - /* Invoke the virtual table xBestIndex() method */ - rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); - if( rc ) return rc; - - mxTerm = -1; - assert( pNew->nLSlot>=nConstraint ); - for(i=0; iaLTerm[i] = 0; - pNew->u.vtab.omitMask = 0; - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - for(i=0; i=0 ){ - WhereTerm *pTerm; - int j = pIdxCons->iTermOffset; - if( iTerm>=nConstraint - || j<0 - || j>=pWC->nTerm - || pNew->aLTerm[iTerm]!=0 - || pIdxCons->usable==0 - ){ - rc = SQLITE_ERROR; - sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); - return rc; - } - testcase( iTerm==nConstraint-1 ); - testcase( j==0 ); - testcase( j==pWC->nTerm-1 ); - pTerm = &pWC->a[j]; - pNew->prereq |= pTerm->prereqRight; - assert( iTermnLSlot ); - pNew->aLTerm[iTerm] = pTerm; - if( iTerm>mxTerm ) mxTerm = iTerm; - testcase( iTerm==15 ); - testcase( iTerm==16 ); - if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ - /* A virtual table that is constrained by an IN clause may not - ** consume the ORDER BY clause because (1) the order of IN terms - ** is not necessarily related to the order of output terms and - ** (2) Multiple outputs from a single IN value will not merge - ** together. */ - pIdxInfo->orderByConsumed = 0; - pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE; - *pbIn = 1; assert( (mExclude & WO_IN)==0 ); - } - } - } - pNew->u.vtab.omitMask &= ~mNoOmit; - - pNew->nLTerm = mxTerm+1; - assert( pNew->nLTerm<=pNew->nLSlot ); - pNew->u.vtab.idxNum = pIdxInfo->idxNum; - pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; - pIdxInfo->needToFreeIdxStr = 0; - pNew->u.vtab.idxStr = pIdxInfo->idxStr; - pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? - pIdxInfo->nOrderBy : 0); - pNew->rSetup = 0; - pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); - pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); - - /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated - ** that the scan will visit at most one row. Clear it otherwise. */ - if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){ - pNew->wsFlags |= WHERE_ONEROW; - }else{ - pNew->wsFlags &= ~WHERE_ONEROW; - } - rc = whereLoopInsert(pBuilder, pNew); - if( pNew->u.vtab.needFree ){ - sqlite3_free(pNew->u.vtab.idxStr); - pNew->u.vtab.needFree = 0; - } - WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", - *pbIn, (sqlite3_uint64)mPrereq, - (sqlite3_uint64)(pNew->prereq & ~mPrereq))); - - return rc; -} - - -/* -** Add all WhereLoop objects for a table of the join identified by -** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. -** -** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and -** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause -** entries that occur before the virtual table in the FROM clause and are -** separated from it by at least one LEFT or CROSS JOIN. Similarly, the -** mUnusable mask contains all FROM clause entries that occur after the -** virtual table and are separated from it by at least one LEFT or -** CROSS JOIN. -** -** For example, if the query were: -** -** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6; -** -** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6). -** -** All the tables in mPrereq must be scanned before the current virtual -** table. So any terms for which all prerequisites are satisfied by -** mPrereq may be specified as "usable" in all calls to xBestIndex. -** Conversely, all tables in mUnusable must be scanned after the current -** virtual table, so any terms for which the prerequisites overlap with -** mUnusable should always be configured as "not-usable" for xBestIndex. -*/ -static int whereLoopAddVirtual( - WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mPrereq, /* Tables that must be scanned before this one */ - Bitmask mUnusable /* Tables that must be scanned after this one */ -){ - int rc = SQLITE_OK; /* Return code */ - WhereInfo *pWInfo; /* WHERE analysis context */ - Parse *pParse; /* The parsing context */ - WhereClause *pWC; /* The WHERE clause */ - struct SrcList_item *pSrc; /* The FROM clause term to search */ - sqlite3_index_info *p; /* Object to pass to xBestIndex() */ - int nConstraint; /* Number of constraints in p */ - int bIn; /* True if plan uses IN(...) operator */ - WhereLoop *pNew; - Bitmask mBest; /* Tables used by best possible plan */ - u16 mNoOmit; - - assert( (mPrereq & mUnusable)==0 ); - pWInfo = pBuilder->pWInfo; - pParse = pWInfo->pParse; - pWC = pBuilder->pWC; - pNew = pBuilder->pNew; - pSrc = &pWInfo->pTabList->a[pNew->iTab]; - assert( IsVirtual(pSrc->pTab) ); - p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy, - &mNoOmit); - if( p==0 ) return SQLITE_NOMEM_BKPT; - pNew->rSetup = 0; - pNew->wsFlags = WHERE_VIRTUALTABLE; - pNew->nLTerm = 0; - pNew->u.vtab.needFree = 0; - nConstraint = p->nConstraint; - if( whereLoopResize(pParse->db, pNew, nConstraint) ){ - sqlite3DbFree(pParse->db, p); - return SQLITE_NOMEM_BKPT; - } - - /* First call xBestIndex() with all constraints usable. */ - WHERETRACE(0x40, (" VirtualOne: all usable\n")); - rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn); - - /* If the call to xBestIndex() with all terms enabled produced a plan - ** that does not require any source tables (IOW: a plan with mBest==0), - ** then there is no point in making any further calls to xBestIndex() - ** since they will all return the same result (if the xBestIndex() - ** implementation is sane). */ - if( rc==SQLITE_OK && (mBest = (pNew->prereq & ~mPrereq))!=0 ){ - int seenZero = 0; /* True if a plan with no prereqs seen */ - int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */ - Bitmask mPrev = 0; - Bitmask mBestNoIn = 0; - - /* If the plan produced by the earlier call uses an IN(...) term, call - ** xBestIndex again, this time with IN(...) terms disabled. */ - if( bIn ){ - WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n")); - rc = whereLoopAddVirtualOne( - pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn); - assert( bIn==0 ); - mBestNoIn = pNew->prereq & ~mPrereq; - if( mBestNoIn==0 ){ - seenZero = 1; - seenZeroNoIN = 1; - } - } - - /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq) - ** in the set of terms that apply to the current virtual table. */ - while( rc==SQLITE_OK ){ - int i; - Bitmask mNext = ALLBITS; - assert( mNext>0 ); - for(i=0; ia[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq - ); - if( mThis>mPrev && mThisprereq==mPrereq ){ - seenZero = 1; - if( bIn==0 ) seenZeroNoIN = 1; - } - } - - /* If the calls to xBestIndex() in the above loop did not find a plan - ** that requires no source tables at all (i.e. one guaranteed to be - ** usable), make a call here with all source tables disabled */ - if( rc==SQLITE_OK && seenZero==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled\n")); - rc = whereLoopAddVirtualOne( - pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn); - if( bIn==0 ) seenZeroNoIN = 1; - } - - /* If the calls to xBestIndex() have so far failed to find a plan - ** that requires no source tables at all and does not use an IN(...) - ** operator, make a final call to obtain one here. */ - if( rc==SQLITE_OK && seenZeroNoIN==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n")); - rc = whereLoopAddVirtualOne( - pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn); - } - } - - if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr); - sqlite3DbFree(pParse->db, p); - return rc; -} -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - -/* -** Add WhereLoop entries to handle OR terms. This works for either -** btrees or virtual tables. -*/ -static int whereLoopAddOr( - WhereLoopBuilder *pBuilder, - Bitmask mPrereq, - Bitmask mUnusable -){ - WhereInfo *pWInfo = pBuilder->pWInfo; - WhereClause *pWC; - WhereLoop *pNew; - WhereTerm *pTerm, *pWCEnd; - int rc = SQLITE_OK; - int iCur; - WhereClause tempWC; - WhereLoopBuilder sSubBuild; - WhereOrSet sSum, sCur; - struct SrcList_item *pItem; - - pWC = pBuilder->pWC; - pWCEnd = pWC->a + pWC->nTerm; - pNew = pBuilder->pNew; - memset(&sSum, 0, sizeof(sSum)); - pItem = pWInfo->pTabList->a + pNew->iTab; - iCur = pItem->iCursor; - - for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 - && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 - ){ - WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; - WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; - WhereTerm *pOrTerm; - int once = 1; - int i, j; - - sSubBuild = *pBuilder; - sSubBuild.pOrderBy = 0; - sSubBuild.pOrSet = &sCur; - - WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); - for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ - sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; - }else if( pOrTerm->leftCursor==iCur ){ - tempWC.pWInfo = pWC->pWInfo; - tempWC.pOuter = pWC; - tempWC.op = TK_AND; - tempWC.nTerm = 1; - tempWC.a = pOrTerm; - sSubBuild.pWC = &tempWC; - }else{ - continue; - } - sCur.n = 0; -#ifdef WHERETRACE_ENABLED - WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", - (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); - if( sqlite3WhereTrace & 0x400 ){ - sqlite3WhereClausePrint(sSubBuild.pWC); - } -#endif -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pItem->pTab) ){ - rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable); - }else -#endif - { - rc = whereLoopAddBtree(&sSubBuild, mPrereq); - } - if( rc==SQLITE_OK ){ - rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); - } - assert( rc==SQLITE_OK || sCur.n==0 ); - if( sCur.n==0 ){ - sSum.n = 0; - break; - }else if( once ){ - whereOrMove(&sSum, &sCur); - once = 0; - }else{ - WhereOrSet sPrev; - whereOrMove(&sPrev, &sSum); - sSum.n = 0; - for(i=0; inLTerm = 1; - pNew->aLTerm[0] = pTerm; - pNew->wsFlags = WHERE_MULTI_OR; - pNew->rSetup = 0; - pNew->iSortIdx = 0; - memset(&pNew->u, 0, sizeof(pNew->u)); - for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun + 1; - pNew->nOut = sSum.a[i].nOut; - pNew->prereq = sSum.a[i].prereq; - rc = whereLoopInsert(pBuilder, pNew); - } - WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); - } - } - return rc; -} - -/* -** Add all WhereLoop objects for all tables -*/ -static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ - WhereInfo *pWInfo = pBuilder->pWInfo; - Bitmask mPrereq = 0; - Bitmask mPrior = 0; - int iTab; - SrcList *pTabList = pWInfo->pTabList; - struct SrcList_item *pItem; - struct SrcList_item *pEnd = &pTabList->a[pWInfo->nLevel]; - sqlite3 *db = pWInfo->pParse->db; - int rc = SQLITE_OK; - WhereLoop *pNew; - u8 priorJointype = 0; - - /* Loop over the tables in the join, from left to right */ - pNew = pBuilder->pNew; - whereLoopInit(pNew); - for(iTab=0, pItem=pTabList->a; pItemiTab = iTab; - pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); - if( ((pItem->fg.jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){ - /* This condition is true when pItem is the FROM clause term on the - ** right-hand-side of a LEFT or CROSS JOIN. */ - mPrereq = mPrior; - } - priorJointype = pItem->fg.jointype; -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pItem->pTab) ){ - struct SrcList_item *p; - for(p=&pItem[1]; pfg.jointype & (JT_LEFT|JT_CROSS)) ){ - mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor); - } - } - rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable); - }else -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - { - rc = whereLoopAddBtree(pBuilder, mPrereq); - } - if( rc==SQLITE_OK ){ - rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable); - } - mPrior |= pNew->maskSelf; - if( rc || db->mallocFailed ) break; - } - - whereLoopClear(db, pNew); - return rc; -} - -/* -** Examine a WherePath (with the addition of the extra WhereLoop of the 5th -** parameters) to see if it outputs rows in the requested ORDER BY -** (or GROUP BY) without requiring a separate sort operation. Return N: -** -** N>0: N terms of the ORDER BY clause are satisfied -** N==0: No terms of the ORDER BY clause are satisfied -** N<0: Unknown yet how many terms of ORDER BY might be satisfied. -** -** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as -** strict. With GROUP BY and DISTINCT the only requirement is that -** equivalent rows appear immediately adjacent to one another. GROUP BY -** and DISTINCT do not require rows to appear in any particular order as long -** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT -** the pOrderBy terms can be matched in any order. With ORDER BY, the -** pOrderBy terms must be matched in strict left-to-right order. -*/ -static i8 wherePathSatisfiesOrderBy( - WhereInfo *pWInfo, /* The WHERE clause */ - ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ - WherePath *pPath, /* The WherePath to check */ - u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */ - u16 nLoop, /* Number of entries in pPath->aLoop[] */ - WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ - Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ -){ - u8 revSet; /* True if rev is known */ - u8 rev; /* Composite sort order */ - u8 revIdx; /* Index sort order */ - u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ - u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */ - u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ - u16 eqOpMask; /* Allowed equality operators */ - u16 nKeyCol; /* Number of key columns in pIndex */ - u16 nColumn; /* Total number of ordered columns in the index */ - u16 nOrderBy; /* Number terms in the ORDER BY clause */ - int iLoop; /* Index of WhereLoop in pPath being processed */ - int i, j; /* Loop counters */ - int iCur; /* Cursor number for current WhereLoop */ - int iColumn; /* A column number within table iCur */ - WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - Expr *pOBExpr; /* An expression from the ORDER BY clause */ - CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */ - Index *pIndex; /* The index associated with pLoop */ - sqlite3 *db = pWInfo->pParse->db; /* Database connection */ - Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ - Bitmask obDone; /* Mask of all ORDER BY terms */ - Bitmask orderDistinctMask; /* Mask of all well-ordered loops */ - Bitmask ready; /* Mask of inner loops */ - - /* - ** We say the WhereLoop is "one-row" if it generates no more than one - ** row of output. A WhereLoop is one-row if all of the following are true: - ** (a) All index columns match with WHERE_COLUMN_EQ. - ** (b) The index is unique - ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row. - ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags. - ** - ** We say the WhereLoop is "order-distinct" if the set of columns from - ** that WhereLoop that are in the ORDER BY clause are different for every - ** row of the WhereLoop. Every one-row WhereLoop is automatically - ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause - ** is not order-distinct. To be order-distinct is not quite the same as being - ** UNIQUE since a UNIQUE column or index can have multiple rows that - ** are NULL and NULL values are equivalent for the purpose of order-distinct. - ** To be order-distinct, the columns must be UNIQUE and NOT NULL. - ** - ** The rowid for a table is always UNIQUE and NOT NULL so whenever the - ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is - ** automatically order-distinct. - */ - - assert( pOrderBy!=0 ); - if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; - - nOrderBy = pOrderBy->nExpr; - testcase( nOrderBy==BMS-1 ); - if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */ - isOrderDistinct = 1; - obDone = MASKBIT(nOrderBy)-1; - orderDistinctMask = 0; - ready = 0; - eqOpMask = WO_EQ | WO_IS | WO_ISNULL; - if( wctrlFlags & WHERE_ORDERBY_LIMIT ) eqOpMask |= WO_IN; - for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; - if( iLoopaLoop[iLoop]; - if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue; - }else{ - pLoop = pLast; - } - if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - if( pLoop->u.vtab.isOrdered ) obSat = obDone; - break; - } - iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; - - /* Mark off any ORDER BY term X that is a column in the table of - ** the current loop for which there is term in the WHERE - ** clause of the form X IS NULL or X=? that reference only outer - ** loops. - */ - for(i=0; ia[i].pExpr); - if( pOBExpr->op!=TK_COLUMN ) continue; - if( pOBExpr->iTable!=iCur ) continue; - pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, - ~ready, eqOpMask, 0); - if( pTerm==0 ) continue; - if( pTerm->eOperator==WO_IN ){ - /* IN terms are only valid for sorting in the ORDER BY LIMIT - ** optimization, and then only if they are actually used - ** by the query plan */ - assert( wctrlFlags & WHERE_ORDERBY_LIMIT ); - for(j=0; jnLTerm && pTerm!=pLoop->aLTerm[j]; j++){} - if( j>=pLoop->nLTerm ) continue; - } - if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){ - const char *z1, *z2; - pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); - if( !pColl ) pColl = db->pDfltColl; - z1 = pColl->zName; - pColl = sqlite3ExprCollSeq(pWInfo->pParse, pTerm->pExpr); - if( !pColl ) pColl = db->pDfltColl; - z2 = pColl->zName; - if( sqlite3StrICmp(z1, z2)!=0 ) continue; - testcase( pTerm->pExpr->op==TK_IS ); - } - obSat |= MASKBIT(i); - } - - if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ - if( pLoop->wsFlags & WHERE_IPK ){ - pIndex = 0; - nKeyCol = 0; - nColumn = 1; - }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ - return 0; - }else{ - nKeyCol = pIndex->nKeyCol; - nColumn = pIndex->nColumn; - assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); - assert( pIndex->aiColumn[nColumn-1]==XN_ROWID - || !HasRowid(pIndex->pTable)); - isOrderDistinct = IsUniqueIndex(pIndex); - } - - /* Loop through all columns of the index and deal with the ones - ** that are not constrained by == or IN. - */ - rev = revSet = 0; - distinctColumns = 0; - for(j=0; j=pLoop->u.btree.nEq - || (pLoop->aLTerm[j]==0)==(jnSkip) - ); - if( ju.btree.nEq && j>=pLoop->nSkip ){ - u16 eOp = pLoop->aLTerm[j]->eOperator; - - /* Skip over == and IS and ISNULL terms. (Also skip IN terms when - ** doing WHERE_ORDERBY_LIMIT processing). - ** - ** If the current term is a column of an ((?,?) IN (SELECT...)) - ** expression for which the SELECT returns more than one column, - ** check that it is the only column used by this loop. Otherwise, - ** if it is one of two or more, none of the columns can be - ** considered to match an ORDER BY term. */ - if( (eOp & eqOpMask)!=0 ){ - if( eOp & WO_ISNULL ){ - testcase( isOrderDistinct ); - isOrderDistinct = 0; - } - continue; - }else if( ALWAYS(eOp & WO_IN) ){ - /* ALWAYS() justification: eOp is an equality operator due to the - ** ju.btree.nEq constraint above. Any equality other - ** than WO_IN is captured by the previous "if". So this one - ** always has to be WO_IN. */ - Expr *pX = pLoop->aLTerm[j]->pExpr; - for(i=j+1; iu.btree.nEq; i++){ - if( pLoop->aLTerm[i]->pExpr==pX ){ - assert( (pLoop->aLTerm[i]->eOperator & WO_IN) ); - bOnce = 0; - break; - } - } - } - } - - /* Get the column number in the table (iColumn) and sort order - ** (revIdx) for the j-th column of the index. - */ - if( pIndex ){ - iColumn = pIndex->aiColumn[j]; - revIdx = pIndex->aSortOrder[j]; - if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; - }else{ - iColumn = XN_ROWID; - revIdx = 0; - } - - /* An unconstrained column that might be NULL means that this - ** WhereLoop is not well-ordered - */ - if( isOrderDistinct - && iColumn>=0 - && j>=pLoop->u.btree.nEq - && pIndex->pTable->aCol[iColumn].notNull==0 - ){ - isOrderDistinct = 0; - } - - /* Find the ORDER BY term that corresponds to the j-th column - ** of the index and mark that ORDER BY term off - */ - isMatch = 0; - for(i=0; bOnce && ia[i].pExpr); - testcase( wctrlFlags & WHERE_GROUPBY ); - testcase( wctrlFlags & WHERE_DISTINCTBY ); - if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; - if( iColumn>=(-1) ){ - if( pOBExpr->op!=TK_COLUMN ) continue; - if( pOBExpr->iTable!=iCur ) continue; - if( pOBExpr->iColumn!=iColumn ) continue; - }else{ - if( sqlite3ExprCompare(pOBExpr,pIndex->aColExpr->a[j].pExpr,iCur) ){ - continue; - } - } - if( iColumn>=0 ){ - pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); - if( !pColl ) pColl = db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; - } - isMatch = 1; - break; - } - if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){ - /* Make sure the sort order is compatible in an ORDER BY clause. - ** Sort order is irrelevant for a GROUP BY clause. */ - if( revSet ){ - if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0; - }else{ - rev = revIdx ^ pOrderBy->a[i].sortOrder; - if( rev ) *pRevMask |= MASKBIT(iLoop); - revSet = 1; - } - } - if( isMatch ){ - if( iColumn==XN_ROWID ){ - testcase( distinctColumns==0 ); - distinctColumns = 1; - } - obSat |= MASKBIT(i); - }else{ - /* No match found */ - if( j==0 || jmaskSelf; - for(i=0; ia[i].pExpr; - mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); - if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; - if( (mTerm&~orderDistinctMask)==0 ){ - obSat |= MASKBIT(i); - } - } - } - } /* End the loop over all WhereLoops from outer-most down to inner-most */ - if( obSat==obDone ) return (i8)nOrderBy; - if( !isOrderDistinct ){ - for(i=nOrderBy-1; i>0; i--){ - Bitmask m = MASKBIT(i) - 1; - if( (obSat&m)==m ) return i; - } - return 0; - } - return -1; -} - - -/* -** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(), -** the planner assumes that the specified pOrderBy list is actually a GROUP -** BY clause - and so any order that groups rows as required satisfies the -** request. -** -** Normally, in this case it is not possible for the caller to determine -** whether or not the rows are really being delivered in sorted order, or -** just in some other order that provides the required grouping. However, -** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then -** this function may be called on the returned WhereInfo object. It returns -** true if the rows really will be sorted in the specified order, or false -** otherwise. -** -** For example, assuming: -** -** CREATE INDEX i1 ON t1(x, Y); -** -** then -** -** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1 -** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0 -*/ -SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo *pWInfo){ - assert( pWInfo->wctrlFlags & WHERE_GROUPBY ); - assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); - return pWInfo->sorted; -} - -#ifdef WHERETRACE_ENABLED -/* For debugging use only: */ -static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ - static char zName[65]; - int i; - for(i=0; iaLoop[i]->cId; } - if( pLast ) zName[i++] = pLast->cId; - zName[i] = 0; - return zName; -} -#endif - -/* -** Return the cost of sorting nRow rows, assuming that the keys have -** nOrderby columns and that the first nSorted columns are already in -** order. -*/ -static LogEst whereSortingCost( - WhereInfo *pWInfo, - LogEst nRow, - int nOrderBy, - int nSorted -){ - /* TUNING: Estimated cost of a full external sort, where N is - ** the number of rows to sort is: - ** - ** cost = (3.0 * N * log(N)). - ** - ** Or, if the order-by clause has X terms but only the last Y - ** terms are out of order, then block-sorting will reduce the - ** sorting cost to: - ** - ** cost = (3.0 * N * log(N)) * (Y/X) - ** - ** The (Y/X) term is implemented using stack variable rScale - ** below. */ - LogEst rScale, rSortCost; - assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); - rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; - rSortCost = nRow + rScale + 16; - - /* Multiple by log(M) where M is the number of output rows. - ** Use the LIMIT for M if it is smaller */ - if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimitiLimit; - } - rSortCost += estLog(nRow); - return rSortCost; -} - -/* -** Given the list of WhereLoop objects at pWInfo->pLoops, this routine -** attempts to find the lowest cost path that visits each WhereLoop -** once. This path is then loaded into the pWInfo->a[].pWLoop fields. -** -** Assume that the total number of output rows that will need to be sorted -** will be nRowEst (in the 10*log2 representation). Or, ignore sorting -** costs if nRowEst==0. -** -** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation -** error occurs. -*/ -static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ - int mxChoice; /* Maximum number of simultaneous paths tracked */ - int nLoop; /* Number of terms in the join */ - Parse *pParse; /* Parsing context */ - sqlite3 *db; /* The database connection */ - int iLoop; /* Loop counter over the terms of the join */ - int ii, jj; /* Loop counters */ - int mxI = 0; /* Index of next entry to replace */ - int nOrderBy; /* Number of ORDER BY clause terms */ - LogEst mxCost = 0; /* Maximum cost of a set of paths */ - LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */ - int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ - WherePath *aFrom; /* All nFrom paths at the previous level */ - WherePath *aTo; /* The nTo best paths at the current level */ - WherePath *pFrom; /* An element of aFrom[] that we are working on */ - WherePath *pTo; /* An element of aTo[] that we are working on */ - WhereLoop *pWLoop; /* One of the WhereLoop objects */ - WhereLoop **pX; /* Used to divy up the pSpace memory */ - LogEst *aSortCost = 0; /* Sorting and partial sorting costs */ - char *pSpace; /* Temporary memory used by this routine */ - int nSpace; /* Bytes of space allocated at pSpace */ - - pParse = pWInfo->pParse; - db = pParse->db; - nLoop = pWInfo->nLevel; - /* TUNING: For simple queries, only the best path is tracked. - ** For 2-way joins, the 5 best paths are followed. - ** For joins of 3 or more tables, track the 10 best paths */ - mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); - assert( nLoop<=pWInfo->pTabList->nSrc ); - WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); - - /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this - ** case the purpose of this call is to estimate the number of rows returned - ** by the overall query. Once this estimate has been obtained, the caller - ** will invoke this function a second time, passing the estimate as the - ** nRowEst parameter. */ - if( pWInfo->pOrderBy==0 || nRowEst==0 ){ - nOrderBy = 0; - }else{ - nOrderBy = pWInfo->pOrderBy->nExpr; - } - - /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ - nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; - nSpace += sizeof(LogEst) * nOrderBy; - pSpace = sqlite3DbMallocRawNN(db, nSpace); - if( pSpace==0 ) return SQLITE_NOMEM_BKPT; - aTo = (WherePath*)pSpace; - aFrom = aTo+mxChoice; - memset(aFrom, 0, sizeof(aFrom[0])); - pX = (WhereLoop**)(aFrom+mxChoice); - for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){ - pFrom->aLoop = pX; - } - if( nOrderBy ){ - /* If there is an ORDER BY clause and it is not being ignored, set up - ** space for the aSortCost[] array. Each element of the aSortCost array - ** is either zero - meaning it has not yet been initialized - or the - ** cost of sorting nRowEst rows of data where the first X terms of - ** the ORDER BY clause are already in order, where X is the array - ** index. */ - aSortCost = (LogEst*)pX; - memset(aSortCost, 0, sizeof(LogEst) * nOrderBy); - } - assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] ); - assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX ); - - /* Seed the search with a single WherePath containing zero WhereLoops. - ** - ** TUNING: Do not let the number of iterations go above 28. If the cost - ** of computing an automatic index is not paid back within the first 28 - ** rows, then do not use the automatic index. */ - aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) ); - nFrom = 1; - assert( aFrom[0].isOrdered==0 ); - if( nOrderBy ){ - /* If nLoop is zero, then there are no FROM terms in the query. Since - ** in this case the query may return a maximum of one row, the results - ** are already in the requested order. Set isOrdered to nOrderBy to - ** indicate this. Or, if nLoop is greater than zero, set isOrdered to - ** -1, indicating that the result set may or may not be ordered, - ** depending on the loops added to the current plan. */ - aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy; - } - - /* Compute successively longer WherePaths using the previous generation - ** of WherePaths as the basis for the next. Keep track of the mxChoice - ** best paths at each generation */ - for(iLoop=0; iLooppLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ - LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ - LogEst rCost; /* Cost of path (pFrom+pWLoop) */ - LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ - i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ - Bitmask maskNew; /* Mask of src visited by (..) */ - Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ - - if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; - if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; - if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<10 ){ - /* Do not use an automatic index if the this loop is expected - ** to run less than 2 times. */ - assert( 10==sqlite3LogEst(2) ); - continue; - } - /* At this point, pWLoop is a candidate to be the next loop. - ** Compute its cost */ - rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); - rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); - nOut = pFrom->nRow + pWLoop->nOut; - maskNew = pFrom->maskLoop | pWLoop->maskSelf; - if( isOrdered<0 ){ - isOrdered = wherePathSatisfiesOrderBy(pWInfo, - pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, - iLoop, pWLoop, &revMask); - }else{ - revMask = pFrom->revLoop; - } - if( isOrdered>=0 && isOrderedisOrdered^isOrdered)&0x80)==0" is equivalent - ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range - ** of legal values for isOrdered, -1..64. - */ - for(jj=0, pTo=aTo; jjmaskLoop==maskNew - && ((pTo->isOrdered^isOrdered)&0x80)==0 - ){ - testcase( jj==nTo-1 ); - break; - } - } - if( jj>=nTo ){ - /* None of the existing best-so-far paths match the candidate. */ - if( nTo>=mxChoice - && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted)) - ){ - /* The current candidate is no better than any of the mxChoice - ** paths currently in the best-so-far buffer. So discard - ** this candidate as not viable. */ -#ifdef WHERETRACE_ENABLED /* 0x4 */ - if( sqlite3WhereTrace&0x4 ){ - sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n", - wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrdered>=0 ? isOrdered+'0' : '?'); - } -#endif - continue; - } - /* If we reach this points it means that the new candidate path - ** needs to be added to the set of best-so-far paths. */ - if( nTo=0 ? isOrdered+'0' : '?'); - } -#endif - }else{ - /* Control reaches here if best-so-far path pTo=aTo[jj] covers the - ** same set of loops and has the sam isOrdered setting as the - ** candidate path. Check to see if the candidate should replace - ** pTo or if the candidate should be skipped */ - if( pTo->rCostrCost==rCost && pTo->nRow<=nOut) ){ -#ifdef WHERETRACE_ENABLED /* 0x4 */ - if( sqlite3WhereTrace&0x4 ){ - sqlite3DebugPrintf( - "Skip %s cost=%-3d,%3d order=%c", - wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrdered>=0 ? isOrdered+'0' : '?'); - sqlite3DebugPrintf(" vs %s cost=%-3d,%d order=%c\n", - wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); - } -#endif - /* Discard the candidate path from further consideration */ - testcase( pTo->rCost==rCost ); - continue; - } - testcase( pTo->rCost==rCost+1 ); - /* Control reaches here if the candidate path is better than the - ** pTo path. Replace pTo with the candidate. */ -#ifdef WHERETRACE_ENABLED /* 0x4 */ - if( sqlite3WhereTrace&0x4 ){ - sqlite3DebugPrintf( - "Update %s cost=%-3d,%3d order=%c", - wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrdered>=0 ? isOrdered+'0' : '?'); - sqlite3DebugPrintf(" was %s cost=%-3d,%3d order=%c\n", - wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); - } -#endif - } - /* pWLoop is a winner. Add it to the set of best so far */ - pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; - pTo->revLoop = revMask; - pTo->nRow = nOut; - pTo->rCost = rCost; - pTo->rUnsorted = rUnsorted; - pTo->isOrdered = isOrdered; - memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); - pTo->aLoop[iLoop] = pWLoop; - if( nTo>=mxChoice ){ - mxI = 0; - mxCost = aTo[0].rCost; - mxUnsorted = aTo[0].nRow; - for(jj=1, pTo=&aTo[1]; jjrCost>mxCost - || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) - ){ - mxCost = pTo->rCost; - mxUnsorted = pTo->rUnsorted; - mxI = jj; - } - } - } - } - } - -#ifdef WHERETRACE_ENABLED /* >=2 */ - if( sqlite3WhereTrace & 0x02 ){ - sqlite3DebugPrintf("---- after round %d ----\n", iLoop); - for(ii=0, pTo=aTo; iirCost, pTo->nRow, - pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); - if( pTo->isOrdered>0 ){ - sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); - }else{ - sqlite3DebugPrintf("\n"); - } - } - } -#endif - - /* Swap the roles of aFrom and aTo for the next generation */ - pFrom = aTo; - aTo = aFrom; - aFrom = pFrom; - nFrom = nTo; - } - - if( nFrom==0 ){ - sqlite3ErrorMsg(pParse, "no query solution"); - sqlite3DbFree(db, pSpace); - return SQLITE_ERROR; - } - - /* Find the lowest cost path. pFrom will be left pointing to that path */ - pFrom = aFrom; - for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; - } - assert( pWInfo->nLevel==nLoop ); - /* Load the lowest cost path into pWInfo */ - for(iLoop=0; iLoopa + iLoop; - pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop]; - pLevel->iFrom = pWLoop->iTab; - pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; - } - if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 - && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 - && pWInfo->eDistinct==WHERE_DISTINCT_NOOP - && nRowEst - ){ - Bitmask notUsed; - int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pDistinctSet, pFrom, - WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); - if( rc==pWInfo->pDistinctSet->nExpr ){ - pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; - } - } - if( pWInfo->pOrderBy ){ - if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ - if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ - pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; - } - }else{ - pWInfo->nOBSat = pFrom->isOrdered; - pWInfo->revMask = pFrom->revLoop; - if( pWInfo->nOBSat<=0 ){ - pWInfo->nOBSat = 0; - if( nLoop>0 ){ - u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags; - if( (wsFlags & WHERE_ONEROW)==0 - && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN) - ){ - Bitmask m = 0; - int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, - WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m); - testcase( wsFlags & WHERE_IPK ); - testcase( wsFlags & WHERE_COLUMN_IN ); - if( rc==pWInfo->pOrderBy->nExpr ){ - pWInfo->bOrderedInnerLoop = 1; - pWInfo->revMask = m; - } - } - } - } - } - if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) - && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0 - ){ - Bitmask revMask = 0; - int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, - pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask - ); - assert( pWInfo->sorted==0 ); - if( nOrder==pWInfo->pOrderBy->nExpr ){ - pWInfo->sorted = 1; - pWInfo->revMask = revMask; - } - } - } - - - pWInfo->nRowOut = pFrom->nRow; - - /* Free temporary memory and return success */ - sqlite3DbFree(db, pSpace); - return SQLITE_OK; -} - -/* -** Most queries use only a single table (they are not joins) and have -** simple == constraints against indexed fields. This routine attempts -** to plan those simple cases using much less ceremony than the -** general-purpose query planner, and thereby yield faster sqlite3_prepare() -** times for the common case. -** -** Return non-zero on success, if this query can be handled by this -** no-frills query planner. Return zero if this query needs the -** general-purpose query planner. -*/ -static int whereShortCut(WhereLoopBuilder *pBuilder){ - WhereInfo *pWInfo; - struct SrcList_item *pItem; - WhereClause *pWC; - WhereTerm *pTerm; - WhereLoop *pLoop; - int iCur; - int j; - Table *pTab; - Index *pIdx; - - pWInfo = pBuilder->pWInfo; - if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0; - assert( pWInfo->pTabList->nSrc>=1 ); - pItem = pWInfo->pTabList->a; - pTab = pItem->pTab; - if( IsVirtual(pTab) ) return 0; - if( pItem->fg.isIndexedBy ) return 0; - iCur = pItem->iCursor; - pWC = &pWInfo->sWC; - pLoop = pBuilder->pNew; - pLoop->wsFlags = 0; - pLoop->nSkip = 0; - pTerm = sqlite3WhereFindTerm(pWC, iCur, -1, 0, WO_EQ|WO_IS, 0); - if( pTerm ){ - testcase( pTerm->eOperator & WO_IS ); - pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; - pLoop->aLTerm[0] = pTerm; - pLoop->nLTerm = 1; - pLoop->u.btree.nEq = 1; - /* TUNING: Cost of a rowid lookup is 10 */ - pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */ - }else{ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - int opMask; - assert( pLoop->aLTermSpace==pLoop->aLTerm ); - if( !IsUniqueIndex(pIdx) - || pIdx->pPartIdxWhere!=0 - || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) - ) continue; - opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ; - for(j=0; jnKeyCol; j++){ - pTerm = sqlite3WhereFindTerm(pWC, iCur, j, 0, opMask, pIdx); - if( pTerm==0 ) break; - testcase( pTerm->eOperator & WO_IS ); - pLoop->aLTerm[j] = pTerm; - } - if( j!=pIdx->nKeyCol ) continue; - pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; - if( pIdx->isCovering || (pItem->colUsed & ~columnsInIndex(pIdx))==0 ){ - pLoop->wsFlags |= WHERE_IDX_ONLY; - } - pLoop->nLTerm = j; - pLoop->u.btree.nEq = j; - pLoop->u.btree.pIndex = pIdx; - /* TUNING: Cost of a unique index lookup is 15 */ - pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */ - break; - } - } - if( pLoop->wsFlags ){ - pLoop->nOut = (LogEst)1; - pWInfo->a[0].pWLoop = pLoop; - pLoop->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); - pWInfo->a[0].iTabCur = iCur; - pWInfo->nRowOut = 1; - if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; - if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ - pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; - } -#ifdef SQLITE_DEBUG - pLoop->cId = '0'; -#endif - return 1; - } - return 0; -} - -/* -** Generate the beginning of the loop used for WHERE clause processing. -** The return value is a pointer to an opaque structure that contains -** information needed to terminate the loop. Later, the calling routine -** should invoke sqlite3WhereEnd() with the return value of this function -** in order to complete the WHERE clause processing. -** -** If an error occurs, this routine returns NULL. -** -** The basic idea is to do a nested loop, one loop for each table in -** the FROM clause of a select. (INSERT and UPDATE statements are the -** same as a SELECT with only a single table in the FROM clause.) For -** example, if the SQL is this: -** -** SELECT * FROM t1, t2, t3 WHERE ...; -** -** Then the code generated is conceptually like the following: -** -** foreach row1 in t1 do \ Code generated -** foreach row2 in t2 do |-- by sqlite3WhereBegin() -** foreach row3 in t3 do / -** ... -** end \ Code generated -** end |-- by sqlite3WhereEnd() -** end / -** -** Note that the loops might not be nested in the order in which they -** appear in the FROM clause if a different order is better able to make -** use of indices. Note also that when the IN operator appears in -** the WHERE clause, it might result in additional nested loops for -** scanning through all values on the right-hand side of the IN. -** -** There are Btree cursors associated with each table. t1 uses cursor -** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor. -** And so forth. This routine generates code to open those VDBE cursors -** and sqlite3WhereEnd() generates the code to close them. -** -** The code that sqlite3WhereBegin() generates leaves the cursors named -** in pTabList pointing at their appropriate entries. The [...] code -** can use OP_Column and OP_Rowid opcodes on these cursors to extract -** data from the various tables of the loop. -** -** If the WHERE clause is empty, the foreach loops must each scan their -** entire tables. Thus a three-way join is an O(N^3) operation. But if -** the tables have indices and there are terms in the WHERE clause that -** refer to those indices, a complete table scan can be avoided and the -** code will run much faster. Most of the work of this routine is checking -** to see if there are indices that can be used to speed up the loop. -** -** Terms of the WHERE clause are also used to limit which rows actually -** make it to the "..." in the middle of the loop. After each "foreach", -** terms of the WHERE clause that use only terms in that loop and outer -** loops are evaluated and if false a jump is made around all subsequent -** inner loops (or around the "..." if the test occurs within the inner- -** most loop) -** -** OUTER JOINS -** -** An outer join of tables t1 and t2 is conceptally coded as follows: -** -** foreach row1 in t1 do -** flag = 0 -** foreach row2 in t2 do -** start: -** ... -** flag = 1 -** end -** if flag==0 then -** move the row2 cursor to a null row -** goto start -** fi -** end -** -** ORDER BY CLAUSE PROCESSING -** -** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause -** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement -** if there is one. If there is no ORDER BY clause or if this routine -** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. -** -** The iIdxCur parameter is the cursor number of an index. If -** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index -** to use for OR clause processing. The WHERE clause should use this -** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is -** the first cursor in an array of cursors for all indices. iIdxCur should -** be used to compute the appropriate cursor depending on which index is -** used. -*/ -SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( - Parse *pParse, /* The parser context */ - SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ - Expr *pWhere, /* The WHERE clause */ - ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ - ExprList *pDistinctSet, /* Try not to output two rows that duplicate these */ - u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ - int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number - ** If WHERE_USE_LIMIT, then the limit amount */ -){ - int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ - int nTabList; /* Number of elements in pTabList */ - WhereInfo *pWInfo; /* Will become the return value of this function */ - Vdbe *v = pParse->pVdbe; /* The virtual database engine */ - Bitmask notReady; /* Cursors that are not yet positioned */ - WhereLoopBuilder sWLB; /* The WhereLoop builder */ - WhereMaskSet *pMaskSet; /* The expression mask set */ - WhereLevel *pLevel; /* A single level in pWInfo->a[] */ - WhereLoop *pLoop; /* Pointer to a single WhereLoop object */ - int ii; /* Loop counter */ - sqlite3 *db; /* Database connection */ - int rc; /* Return code */ - u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */ - - assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || ( - (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 - && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 - )); - - /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */ - assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 - || (wctrlFlags & WHERE_USE_LIMIT)==0 ); - - /* Variable initialization */ - db = pParse->db; - memset(&sWLB, 0, sizeof(sWLB)); - - /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ - testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); - if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; - sWLB.pOrderBy = pOrderBy; - - /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via - ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ - if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ - wctrlFlags &= ~WHERE_WANT_DISTINCT; - } - - /* The number of tables in the FROM clause is limited by the number of - ** bits in a Bitmask - */ - testcase( pTabList->nSrc==BMS ); - if( pTabList->nSrc>BMS ){ - sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS); - return 0; - } - - /* This function normally generates a nested loop for all tables in - ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should - ** only generate code for the first table in pTabList and assume that - ** any cursors associated with subsequent tables are uninitialized. - */ - nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc; - - /* Allocate and initialize the WhereInfo structure that will become the - ** return value. A single allocation is used to store the WhereInfo - ** struct, the contents of WhereInfo.a[], the WhereClause structure - ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte - ** field (type Bitmask) it must be aligned on an 8-byte boundary on - ** some architectures. Hence the ROUND8() below. - */ - nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); - pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); - if( db->mallocFailed ){ - sqlite3DbFree(db, pWInfo); - pWInfo = 0; - goto whereBeginError; - } - pWInfo->pParse = pParse; - pWInfo->pTabList = pTabList; - pWInfo->pOrderBy = pOrderBy; - pWInfo->pDistinctSet = pDistinctSet; - pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; - pWInfo->nLevel = nTabList; - pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); - pWInfo->wctrlFlags = wctrlFlags; - pWInfo->iLimit = iAuxArg; - pWInfo->savedNQueryLoop = pParse->nQueryLoop; - memset(&pWInfo->nOBSat, 0, - offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat)); - memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel)); - assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */ - pMaskSet = &pWInfo->sMaskSet; - sWLB.pWInfo = pWInfo; - sWLB.pWC = &pWInfo->sWC; - sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo); - assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) ); - whereLoopInit(sWLB.pNew); -#ifdef SQLITE_DEBUG - sWLB.pNew->cId = '*'; -#endif - - /* Split the WHERE clause into separate subexpressions where each - ** subexpression is separated by an AND operator. - */ - initMaskSet(pMaskSet); - sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo); - sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND); - - /* Special case: a WHERE clause that is constant. Evaluate the - ** expression and either jump over all of the code or fall thru. - */ - for(ii=0; iinTerm; ii++){ - if( nTabList==0 || sqlite3ExprIsConstantNotJoin(sWLB.pWC->a[ii].pExpr) ){ - sqlite3ExprIfFalse(pParse, sWLB.pWC->a[ii].pExpr, pWInfo->iBreak, - SQLITE_JUMPIFNULL); - sWLB.pWC->a[ii].wtFlags |= TERM_CODED; - } - } - - /* Special case: No FROM clause - */ - if( nTabList==0 ){ - if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; - if( wctrlFlags & WHERE_WANT_DISTINCT ){ - pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; - } - } - - /* Assign a bit from the bitmask to every term in the FROM clause. - ** - ** The N-th term of the FROM clause is assigned a bitmask of 1<nSrc tables in - ** pTabList, not just the first nTabList tables. nTabList is normally - ** equal to pTabList->nSrc but might be shortened to 1 if the - ** WHERE_OR_SUBCLAUSE flag is set. - */ - for(ii=0; iinSrc; ii++){ - createMask(pMaskSet, pTabList->a[ii].iCursor); - sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); - } -#ifdef SQLITE_DEBUG - for(ii=0; iinSrc; ii++){ - Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); - assert( m==MASKBIT(ii) ); - } -#endif - - /* Analyze all of the subexpressions. */ - sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); - if( db->mallocFailed ) goto whereBeginError; - - if( wctrlFlags & WHERE_WANT_DISTINCT ){ - if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pDistinctSet) ){ - /* The DISTINCT marking is pointless. Ignore it. */ - pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; - }else if( pOrderBy==0 ){ - /* Try to ORDER BY the result set to make distinct processing easier */ - pWInfo->wctrlFlags |= WHERE_DISTINCTBY; - pWInfo->pOrderBy = pDistinctSet; - } - } - - /* Construct the WhereLoop objects */ -#if defined(WHERETRACE_ENABLED) - if( sqlite3WhereTrace & 0xffff ){ - sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags); - if( wctrlFlags & WHERE_USE_LIMIT ){ - sqlite3DebugPrintf(", limit: %d", iAuxArg); - } - sqlite3DebugPrintf(")\n"); - } - if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ - sqlite3WhereClausePrint(sWLB.pWC); - } -#endif - - if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ - rc = whereLoopAddAll(&sWLB); - if( rc ) goto whereBeginError; - -#ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ - WhereLoop *p; - int i; - static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" - "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; - for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ - p->cId = zLabel[i%sizeof(zLabel)]; - whereLoopPrint(p, sWLB.pWC); - } - } -#endif - - wherePathSolver(pWInfo, 0); - if( db->mallocFailed ) goto whereBeginError; - if( pWInfo->pOrderBy ){ - wherePathSolver(pWInfo, pWInfo->nRowOut+1); - if( db->mallocFailed ) goto whereBeginError; - } - } - if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ - pWInfo->revMask = ALLBITS; - } - if( pParse->nErr || NEVER(db->mallocFailed) ){ - goto whereBeginError; - } -#ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace ){ - sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); - if( pWInfo->nOBSat>0 ){ - sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); - } - switch( pWInfo->eDistinct ){ - case WHERE_DISTINCT_UNIQUE: { - sqlite3DebugPrintf(" DISTINCT=unique"); - break; - } - case WHERE_DISTINCT_ORDERED: { - sqlite3DebugPrintf(" DISTINCT=ordered"); - break; - } - case WHERE_DISTINCT_UNORDERED: { - sqlite3DebugPrintf(" DISTINCT=unordered"); - break; - } - } - sqlite3DebugPrintf("\n"); - for(ii=0; iinLevel; ii++){ - whereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC); - } - } -#endif - /* Attempt to omit tables from the join that do not effect the result */ - if( pWInfo->nLevel>=2 - && pDistinctSet!=0 - && OptimizationEnabled(db, SQLITE_OmitNoopJoin) - ){ - Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pDistinctSet); - if( sWLB.pOrderBy ){ - tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy); - } - while( pWInfo->nLevel>=2 ){ - WhereTerm *pTerm, *pEnd; - pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; - if( (pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0 ) break; - if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 - && (pLoop->wsFlags & WHERE_ONEROW)==0 - ){ - break; - } - if( (tabUsed & pLoop->maskSelf)!=0 ) break; - pEnd = sWLB.pWC->a + sWLB.pWC->nTerm; - for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 - && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) - ){ - break; - } - } - if( pTerm drop loop %c not used\n", pLoop->cId)); - pWInfo->nLevel--; - nTabList--; - } - } - WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); - pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; - - /* If the caller is an UPDATE or DELETE statement that is requesting - ** to use a one-pass algorithm, determine if this is appropriate. - */ - assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); - if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ - int wsFlags = pWInfo->a[0].pWLoop->wsFlags; - int bOnerow = (wsFlags & WHERE_ONEROW)!=0; - if( bOnerow - || ((wctrlFlags & WHERE_ONEPASS_MULTIROW)!=0 - && 0==(wsFlags & WHERE_VIRTUALTABLE)) - ){ - pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; - if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ - if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ - bFordelete = OPFLAG_FORDELETE; - } - pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY); - } - } - } - - /* Open all tables in the pTabList and any indices selected for - ** searching those tables. - */ - for(ii=0, pLevel=pWInfo->a; iia[pLevel->iFrom]; - pTab = pTabItem->pTab; - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - pLoop = pLevel->pWLoop; - if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ - /* Do nothing */ - }else -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); - int iCur = pTabItem->iCursor; - sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); - }else if( IsVirtual(pTab) ){ - /* noop */ - }else -#endif - if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){ - int op = OP_OpenRead; - if( pWInfo->eOnePass!=ONEPASS_OFF ){ - op = OP_OpenWrite; - pWInfo->aiCurOnePass[0] = pTabItem->iCursor; - }; - sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); - assert( pTabItem->iCursor==pLevel->iTabCur ); - testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); - testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); - if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nColcolUsed; - int n = 0; - for(; b; b=b>>1, n++){} - sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32); - assert( n<=pTab->nCol ); - } -#ifdef SQLITE_ENABLE_CURSOR_HINTS - if( pLoop->u.btree.pIndex!=0 ){ - sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); - }else -#endif - { - sqlite3VdbeChangeP5(v, bFordelete); - } -#ifdef SQLITE_ENABLE_COLUMN_USED_MASK - sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, - (const u8*)&pTabItem->colUsed, P4_INT64); -#endif - }else{ - sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - } - if( pLoop->wsFlags & WHERE_INDEXED ){ - Index *pIx = pLoop->u.btree.pIndex; - int iIndexCur; - int op = OP_OpenRead; - /* iAuxArg is always set if to a positive value if ONEPASS is possible */ - assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); - if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) - && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 - ){ - /* This is one term of an OR-optimization using the PRIMARY KEY of a - ** WITHOUT ROWID table. No need for a separate index */ - iIndexCur = pLevel->iTabCur; - op = 0; - }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ - Index *pJ = pTabItem->pTab->pIndex; - iIndexCur = iAuxArg; - assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); - while( ALWAYS(pJ) && pJ!=pIx ){ - iIndexCur++; - pJ = pJ->pNext; - } - op = OP_OpenWrite; - pWInfo->aiCurOnePass[1] = iIndexCur; - }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){ - iIndexCur = iAuxArg; - op = OP_ReopenIdx; - }else{ - iIndexCur = pParse->nTab++; - } - pLevel->iIdxCur = iIndexCur; - assert( pIx->pSchema==pTab->pSchema ); - assert( iIndexCur>=0 ); - if( op ){ - sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); - sqlite3VdbeSetP4KeyInfo(pParse, pIx); - if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 - && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 - && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 - ){ - sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */ - } - VdbeComment((v, "%s", pIx->zName)); -#ifdef SQLITE_ENABLE_COLUMN_USED_MASK - { - u64 colUsed = 0; - int ii, jj; - for(ii=0; iinColumn; ii++){ - jj = pIx->aiColumn[ii]; - if( jj<0 ) continue; - if( jj>63 ) jj = 63; - if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue; - colUsed |= ((u64)1)<<(ii<63 ? ii : 63); - } - sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0, - (u8*)&colUsed, P4_INT64); - } -#endif /* SQLITE_ENABLE_COLUMN_USED_MASK */ - } - } - if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); - } - pWInfo->iTop = sqlite3VdbeCurrentAddr(v); - if( db->mallocFailed ) goto whereBeginError; - - /* Generate the code to do the search. Each iteration of the for - ** loop below generates code for a single nested loop of the VM - ** program. - */ - notReady = ~(Bitmask)0; - for(ii=0; iia[ii]; - wsFlags = pLevel->pWLoop->wsFlags; -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX - if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ - constructAutomaticIndex(pParse, &pWInfo->sWC, - &pTabList->a[pLevel->iFrom], notReady, pLevel); - if( db->mallocFailed ) goto whereBeginError; - } -#endif - addrExplain = sqlite3WhereExplainOneScan( - pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags - ); - pLevel->addrBody = sqlite3VdbeCurrentAddr(v); - notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady); - pWInfo->iContinue = pLevel->addrCont; - if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){ - sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain); - } - } - - /* Done. */ - VdbeModuleComment((v, "Begin WHERE-core")); - return pWInfo; - - /* Jump here if malloc fails */ -whereBeginError: - if( pWInfo ){ - pParse->nQueryLoop = pWInfo->savedNQueryLoop; - whereInfoFree(db, pWInfo); - } - return 0; -} - -/* -** Generate the end of the WHERE loop. See comments on -** sqlite3WhereBegin() for additional information. -*/ -SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ - Parse *pParse = pWInfo->pParse; - Vdbe *v = pParse->pVdbe; - int i; - WhereLevel *pLevel; - WhereLoop *pLoop; - SrcList *pTabList = pWInfo->pTabList; - sqlite3 *db = pParse->db; - - /* Generate loop termination code. - */ - VdbeModuleComment((v, "End WHERE-core")); - sqlite3ExprCacheClear(pParse); - for(i=pWInfo->nLevel-1; i>=0; i--){ - int addr; - pLevel = &pWInfo->a[i]; - pLoop = pLevel->pWLoop; - sqlite3VdbeResolveLabel(v, pLevel->addrCont); - if( pLevel->op!=OP_Noop ){ - sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); - sqlite3VdbeChangeP5(v, pLevel->p5); - VdbeCoverage(v); - VdbeCoverageIf(v, pLevel->op==OP_Next); - VdbeCoverageIf(v, pLevel->op==OP_Prev); - VdbeCoverageIf(v, pLevel->op==OP_VNext); - } - if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ - struct InLoop *pIn; - int j; - sqlite3VdbeResolveLabel(v, pLevel->addrNxt); - for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ - sqlite3VdbeJumpHere(v, pIn->addrInTop+1); - if( pIn->eEndLoopOp!=OP_Noop ){ - sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); - VdbeCoverage(v); - VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen); - VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); - } - sqlite3VdbeJumpHere(v, pIn->addrInTop-1); - } - } - sqlite3VdbeResolveLabel(v, pLevel->addrBrk); - if( pLevel->addrSkip ){ - sqlite3VdbeGoto(v, pLevel->addrSkip); - VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); - sqlite3VdbeJumpHere(v, pLevel->addrSkip); - sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); - } -#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS - if( pLevel->addrLikeRep ){ - sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1), - pLevel->addrLikeRep); - VdbeCoverage(v); - } -#endif - if( pLevel->iLeftJoin ){ - int ws = pLoop->wsFlags; - addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); - assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); - if( (ws & WHERE_IDX_ONLY)==0 ){ - sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); - } - if( (ws & WHERE_INDEXED) - || ((ws & WHERE_MULTI_OR) && pLevel->u.pCovidx) - ){ - sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); - } - if( pLevel->op==OP_Return ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); - }else{ - sqlite3VdbeGoto(v, pLevel->addrFirst); - } - sqlite3VdbeJumpHere(v, addr); - } - VdbeModuleComment((v, "End WHERE-loop%d: %s", i, - pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); - } - - /* The "break" point is here, just past the end of the outer loop. - ** Set it. - */ - sqlite3VdbeResolveLabel(v, pWInfo->iBreak); - - assert( pWInfo->nLevel<=pTabList->nSrc ); - for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ - int k, last; - VdbeOp *pOp; - Index *pIdx = 0; - struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom]; - Table *pTab = pTabItem->pTab; - assert( pTab!=0 ); - pLoop = pLevel->pWLoop; - - /* For a co-routine, change all OP_Column references to the table of - ** the co-routine into OP_Copy of result contained in a register. - ** OP_Rowid becomes OP_Null. - */ - if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){ - translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur, - pTabItem->regResult, 0); - continue; - } - - /* Close all of the cursors that were opened by sqlite3WhereBegin. - ** Except, do not close cursors that will be reused by the OR optimization - ** (WHERE_OR_SUBCLAUSE). And do not close the OP_OpenWrite cursors - ** created for the ONEPASS optimization. - */ - if( (pTab->tabFlags & TF_Ephemeral)==0 - && pTab->pSelect==0 - && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 - ){ - int ws = pLoop->wsFlags; - if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){ - sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); - } - if( (ws & WHERE_INDEXED)!=0 - && (ws & (WHERE_IPK|WHERE_AUTO_INDEX))==0 - && pLevel->iIdxCur!=pWInfo->aiCurOnePass[1] - ){ - sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); - } - } - - /* If this scan uses an index, make VDBE code substitutions to read data - ** from the index instead of from the table where possible. In some cases - ** this optimization prevents the table from ever being read, which can - ** yield a significant performance boost. - ** - ** Calls to the code generator in between sqlite3WhereBegin and - ** sqlite3WhereEnd will have created code that references the table - ** directly. This loop scans all that code looking for opcodes - ** that reference the table and converts them into opcodes that - ** reference the index. - */ - if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){ - pIdx = pLoop->u.btree.pIndex; - }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ - pIdx = pLevel->u.pCovidx; - } - if( pIdx - && (pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable)) - && !db->mallocFailed - ){ - last = sqlite3VdbeCurrentAddr(v); - k = pLevel->addrBody; - pOp = sqlite3VdbeGetOp(v, k); - for(; kp1!=pLevel->iTabCur ) continue; - if( pOp->opcode==OP_Column ){ - int x = pOp->p2; - assert( pIdx->pTable==pTab ); - if( !HasRowid(pTab) ){ - Index *pPk = sqlite3PrimaryKeyIndex(pTab); - x = pPk->aiColumn[x]; - assert( x>=0 ); - } - x = sqlite3ColumnOfIndex(pIdx, x); - if( x>=0 ){ - pOp->p2 = x; - pOp->p1 = pLevel->iIdxCur; - } - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 ); - }else if( pOp->opcode==OP_Rowid ){ - pOp->p1 = pLevel->iIdxCur; - pOp->opcode = OP_IdxRowid; - } - } - } - } - - /* Final cleanup - */ - pParse->nQueryLoop = pWInfo->savedNQueryLoop; - whereInfoFree(db, pWInfo); - return; -} - -/************** End of where.c ***********************************************/ -/************** Begin file parse.c *******************************************/ -/* -** 2000-05-29 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Driver template for the LEMON parser generator. -** -** The "lemon" program processes an LALR(1) input grammar file, then uses -** this template to construct a parser. The "lemon" program inserts text -** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the -** interstitial "-" characters) contained in this template is changed into -** the value of the %name directive from the grammar. Otherwise, the content -** of this template is copied straight through into the generate parser -** source file. -** -** The following is the concatenation of all %include directives from the -** input grammar file: -*/ -/* #include */ -/************ Begin %include sections from the grammar ************************/ - -/* #include "sqliteInt.h" */ - -/* -** Disable all error recovery processing in the parser push-down -** automaton. -*/ -#define YYNOERRORRECOVERY 1 - -/* -** Make yytestcase() the same as testcase() -*/ -#define yytestcase(X) testcase(X) - -/* -** Indicate that sqlite3ParserFree() will never be called with a null -** pointer. -*/ -#define YYPARSEFREENEVERNULL 1 - -/* -** Alternative datatype for the argument to the malloc() routine passed -** into sqlite3ParserAlloc(). The default is size_t. -*/ -#define YYMALLOCARGTYPE u64 - -/* -** An instance of this structure holds information about the -** LIMIT clause of a SELECT statement. -*/ -struct LimitVal { - Expr *pLimit; /* The LIMIT expression. NULL if there is no limit */ - Expr *pOffset; /* The OFFSET expression. NULL if there is none */ -}; - -/* -** An instance of the following structure describes the event of a -** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, -** TK_DELETE, or TK_INSTEAD. If the event is of the form -** -** UPDATE ON (a,b,c) -** -** Then the "b" IdList records the list "a,b,c". -*/ -struct TrigEvent { int a; IdList * b; }; - -/* -** Disable lookaside memory allocation for objects that might be -** shared across database connections. -*/ -static void disableLookaside(Parse *pParse){ - pParse->disableLookaside++; - pParse->db->lookaside.bDisable++; -} - - - /* - ** For a compound SELECT statement, make sure p->pPrior->pNext==p for - ** all elements in the list. And make sure list length does not exceed - ** SQLITE_LIMIT_COMPOUND_SELECT. - */ - static void parserDoubleLinkSelect(Parse *pParse, Select *p){ - if( p->pPrior ){ - Select *pNext = 0, *pLoop; - int mxSelect, cnt = 0; - for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ - pLoop->pNext = pNext; - pLoop->selFlags |= SF_Compound; - } - if( (p->selFlags & SF_MultiValue)==0 && - (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && - cnt>mxSelect - ){ - sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); - } - } - } - - /* This is a utility routine used to set the ExprSpan.zStart and - ** ExprSpan.zEnd values of pOut so that the span covers the complete - ** range of text beginning with pStart and going to the end of pEnd. - */ - static void spanSet(ExprSpan *pOut, Token *pStart, Token *pEnd){ - pOut->zStart = pStart->z; - pOut->zEnd = &pEnd->z[pEnd->n]; - } - - /* Construct a new Expr object from a single identifier. Use the - ** new Expr to populate pOut. Set the span of pOut to be the identifier - ** that created the expression. - */ - static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token t){ - Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1); - if( p ){ - memset(p, 0, sizeof(Expr)); - p->op = (u8)op; - p->flags = EP_Leaf; - p->iAgg = -1; - p->u.zToken = (char*)&p[1]; - memcpy(p->u.zToken, t.z, t.n); - p->u.zToken[t.n] = 0; - if( sqlite3Isquote(p->u.zToken[0]) ){ - if( p->u.zToken[0]=='"' ) p->flags |= EP_DblQuoted; - sqlite3Dequote(p->u.zToken); - } -#if SQLITE_MAX_EXPR_DEPTH>0 - p->nHeight = 1; -#endif - } - pOut->pExpr = p; - pOut->zStart = t.z; - pOut->zEnd = &t.z[t.n]; - } - - /* This routine constructs a binary expression node out of two ExprSpan - ** objects and uses the result to populate a new ExprSpan object. - */ - static void spanBinaryExpr( - Parse *pParse, /* The parsing context. Errors accumulate here */ - int op, /* The binary operation */ - ExprSpan *pLeft, /* The left operand, and output */ - ExprSpan *pRight /* The right operand */ - ){ - pLeft->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); - pLeft->zEnd = pRight->zEnd; - } - - /* If doNot is true, then add a TK_NOT Expr-node wrapper around the - ** outside of *ppExpr. - */ - static void exprNot(Parse *pParse, int doNot, ExprSpan *pSpan){ - if( doNot ){ - pSpan->pExpr = sqlite3PExpr(pParse, TK_NOT, pSpan->pExpr, 0, 0); - } - } - - /* Construct an expression node for a unary postfix operator - */ - static void spanUnaryPostfix( - Parse *pParse, /* Parsing context to record errors */ - int op, /* The operator */ - ExprSpan *pOperand, /* The operand, and output */ - Token *pPostOp /* The operand token for setting the span */ - ){ - pOperand->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); - pOperand->zEnd = &pPostOp->z[pPostOp->n]; - } - - /* A routine to convert a binary TK_IS or TK_ISNOT expression into a - ** unary TK_ISNULL or TK_NOTNULL expression. */ - static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ - sqlite3 *db = pParse->db; - if( pA && pY && pY->op==TK_NULL ){ - pA->op = (u8)op; - sqlite3ExprDelete(db, pA->pRight); - pA->pRight = 0; - } - } - - /* Construct an expression node for a unary prefix operator - */ - static void spanUnaryPrefix( - ExprSpan *pOut, /* Write the new expression node here */ - Parse *pParse, /* Parsing context to record errors */ - int op, /* The operator */ - ExprSpan *pOperand, /* The operand */ - Token *pPreOp /* The operand token for setting the span */ - ){ - pOut->zStart = pPreOp->z; - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); - pOut->zEnd = pOperand->zEnd; - } - - /* Add a single new term to an ExprList that is used to store a - ** list of identifiers. Report an error if the ID list contains - ** a COLLATE clause or an ASC or DESC keyword, except ignore the - ** error while parsing a legacy schema. - */ - static ExprList *parserAddExprIdListTerm( - Parse *pParse, - ExprList *pPrior, - Token *pIdToken, - int hasCollate, - int sortOrder - ){ - ExprList *p = sqlite3ExprListAppend(pParse, pPrior, 0); - if( (hasCollate || sortOrder!=SQLITE_SO_UNDEFINED) - && pParse->db->init.busy==0 - ){ - sqlite3ErrorMsg(pParse, "syntax error after column name \"%.*s\"", - pIdToken->n, pIdToken->z); - } - sqlite3ExprListSetName(pParse, p, pIdToken, 1); - return p; - } -/**************** End of %include directives **********************************/ -/* These constants specify the various numeric values for terminal symbols -** in a format understandable to "makeheaders". This section is blank unless -** "lemon" is run with the "-m" command-line option. -***************** Begin makeheaders token definitions *************************/ -/**************** End makeheaders token definitions ***************************/ - -/* The next sections is a series of control #defines. -** various aspects of the generated parser. -** YYCODETYPE is the data type used to store the integer codes -** that represent terminal and non-terminal symbols. -** "unsigned char" is used if there are fewer than -** 256 symbols. Larger types otherwise. -** YYNOCODE is a number of type YYCODETYPE that is not used for -** any terminal or nonterminal symbol. -** YYFALLBACK If defined, this indicates that one or more tokens -** (also known as: "terminal symbols") have fall-back -** values which should be used if the original symbol -** would not parse. This permits keywords to sometimes -** be used as identifiers, for example. -** YYACTIONTYPE is the data type used for "action codes" - numbers -** that indicate what to do in response to the next -** token. -** sqlite3ParserTOKENTYPE is the data type used for minor type for terminal -** symbols. Background: A "minor type" is a semantic -** value associated with a terminal or non-terminal -** symbols. For example, for an "ID" terminal symbol, -** the minor type might be the name of the identifier. -** Each non-terminal can have a different minor type. -** Terminal symbols all have the same minor type, though. -** This macros defines the minor type for terminal -** symbols. -** YYMINORTYPE is the data type used for all minor types. -** This is typically a union of many types, one of -** which is sqlite3ParserTOKENTYPE. The entry in the union -** for terminal symbols is called "yy0". -** YYSTACKDEPTH is the maximum depth of the parser's stack. If -** zero the stack is dynamically sized using realloc() -** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument -** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument -** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser -** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser -** YYERRORSYMBOL is the code number of the error symbol. If not -** defined, then do no error processing. -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar -** YY_MAX_SHIFT Maximum value for shift actions -** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions -** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions -** YY_MIN_REDUCE Maximum value for reduce actions -** YY_ERROR_ACTION The yy_action[] code for syntax error -** YY_ACCEPT_ACTION The yy_action[] code for accept -** YY_NO_ACTION The yy_action[] code for no-op -*/ -#ifndef INTERFACE -# define INTERFACE 1 -#endif -/************* Begin control #defines *****************************************/ -#define YYCODETYPE unsigned char -#define YYNOCODE 252 -#define YYACTIONTYPE unsigned short int -#define YYWILDCARD 96 -#define sqlite3ParserTOKENTYPE Token -typedef union { - int yyinit; - sqlite3ParserTOKENTYPE yy0; - Expr* yy72; - TriggerStep* yy145; - ExprList* yy148; - SrcList* yy185; - ExprSpan yy190; - int yy194; - Select* yy243; - IdList* yy254; - With* yy285; - struct TrigEvent yy332; - struct LimitVal yy354; - struct {int value; int mask;} yy497; -} YYMINORTYPE; -#ifndef YYSTACKDEPTH -#define YYSTACKDEPTH 100 -#endif -#define sqlite3ParserARG_SDECL Parse *pParse; -#define sqlite3ParserARG_PDECL ,Parse *pParse -#define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse -#define sqlite3ParserARG_STORE yypParser->pParse = pParse -#define YYFALLBACK 1 -#define YYNSTATE 456 -#define YYNRULE 332 -#define YY_MAX_SHIFT 455 -#define YY_MIN_SHIFTREDUCE 668 -#define YY_MAX_SHIFTREDUCE 999 -#define YY_MIN_REDUCE 1000 -#define YY_MAX_REDUCE 1331 -#define YY_ERROR_ACTION 1332 -#define YY_ACCEPT_ACTION 1333 -#define YY_NO_ACTION 1334 -/************* End control #defines *******************************************/ - -/* Define the yytestcase() macro to be a no-op if is not already defined -** otherwise. -** -** Applications can choose to define yytestcase() in the %include section -** to a macro that can assist in verifying code coverage. For production -** code the yytestcase() macro should be turned off. But it is useful -** for testing. -*/ -#ifndef yytestcase -# define yytestcase(X) -#endif - - -/* Next are the tables used to determine what action to take based on the -** current state and lookahead token. These tables are used to implement -** functions that take a state number and lookahead value and return an -** action integer. -** -** Suppose the action integer is N. Then the action is determined as -** follows -** -** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead -** token onto the stack and goto state N. -** -** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then -** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. -** -** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE -** and YY_MAX_REDUCE -** -** N == YY_ERROR_ACTION A syntax error has occurred. -** -** N == YY_ACCEPT_ACTION The parser accepts its input. -** -** N == YY_NO_ACTION No such action. Denotes unused -** slots in the yy_action[] table. -** -** The action table is constructed as a single large table named yy_action[]. -** Given state S and lookahead X, the action is computed as either: -** -** (A) N = yy_action[ yy_shift_ofst[S] + X ] -** (B) N = yy_default[S] -** -** The (A) formula is preferred. The B formula is used instead if: -** (1) The yy_shift_ofst[S]+X value is out of range, or -** (2) yy_lookahead[yy_shift_ofst[S]+X] is not equal to X, or -** (3) yy_shift_ofst[S] equal YY_SHIFT_USE_DFLT. -** (Implementation note: YY_SHIFT_USE_DFLT is chosen so that -** YY_SHIFT_USE_DFLT+X will be out of range for all possible lookaheads X. -** Hence only tests (1) and (2) need to be evaluated.) -** -** The formulas above are for computing the action when the lookahead is -** a terminal symbol. If the lookahead is a non-terminal (as occurs after -** a reduce action) then the yy_reduce_ofst[] array is used in place of -** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of -** YY_SHIFT_USE_DFLT. -** -** The following are the tables generated in this section: -** -** yy_action[] A single table containing all actions. -** yy_lookahead[] A table containing the lookahead for each entry in -** yy_action. Used to detect hash collisions. -** yy_shift_ofst[] For each state, the offset into yy_action for -** shifting terminals. -** yy_reduce_ofst[] For each state, the offset into yy_action for -** shifting non-terminals after a reduce. -** yy_default[] Default action for each state. -** -*********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1567) -static const YYACTIONTYPE yy_action[] = { - /* 0 */ 325, 832, 351, 825, 5, 203, 203, 819, 99, 100, - /* 10 */ 90, 842, 842, 854, 857, 846, 846, 97, 97, 98, - /* 20 */ 98, 98, 98, 301, 96, 96, 96, 96, 95, 95, - /* 30 */ 94, 94, 94, 93, 351, 325, 977, 977, 824, 824, - /* 40 */ 826, 947, 354, 99, 100, 90, 842, 842, 854, 857, - /* 50 */ 846, 846, 97, 97, 98, 98, 98, 98, 338, 96, - /* 60 */ 96, 96, 96, 95, 95, 94, 94, 94, 93, 351, - /* 70 */ 95, 95, 94, 94, 94, 93, 351, 791, 977, 977, - /* 80 */ 325, 94, 94, 94, 93, 351, 792, 75, 99, 100, - /* 90 */ 90, 842, 842, 854, 857, 846, 846, 97, 97, 98, - /* 100 */ 98, 98, 98, 450, 96, 96, 96, 96, 95, 95, - /* 110 */ 94, 94, 94, 93, 351, 1333, 155, 155, 2, 325, - /* 120 */ 275, 146, 132, 52, 52, 93, 351, 99, 100, 90, - /* 130 */ 842, 842, 854, 857, 846, 846, 97, 97, 98, 98, - /* 140 */ 98, 98, 101, 96, 96, 96, 96, 95, 95, 94, - /* 150 */ 94, 94, 93, 351, 958, 958, 325, 268, 428, 413, - /* 160 */ 411, 61, 752, 752, 99, 100, 90, 842, 842, 854, - /* 170 */ 857, 846, 846, 97, 97, 98, 98, 98, 98, 60, - /* 180 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, - /* 190 */ 351, 325, 270, 329, 273, 277, 959, 960, 250, 99, - /* 200 */ 100, 90, 842, 842, 854, 857, 846, 846, 97, 97, - /* 210 */ 98, 98, 98, 98, 301, 96, 96, 96, 96, 95, - /* 220 */ 95, 94, 94, 94, 93, 351, 325, 938, 1326, 698, - /* 230 */ 706, 1326, 242, 412, 99, 100, 90, 842, 842, 854, - /* 240 */ 857, 846, 846, 97, 97, 98, 98, 98, 98, 347, - /* 250 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, - /* 260 */ 351, 325, 938, 1327, 384, 699, 1327, 381, 379, 99, - /* 270 */ 100, 90, 842, 842, 854, 857, 846, 846, 97, 97, - /* 280 */ 98, 98, 98, 98, 701, 96, 96, 96, 96, 95, - /* 290 */ 95, 94, 94, 94, 93, 351, 325, 92, 89, 178, - /* 300 */ 833, 936, 373, 700, 99, 100, 90, 842, 842, 854, - /* 310 */ 857, 846, 846, 97, 97, 98, 98, 98, 98, 375, - /* 320 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, - /* 330 */ 351, 325, 1276, 947, 354, 818, 936, 739, 739, 99, - /* 340 */ 100, 90, 842, 842, 854, 857, 846, 846, 97, 97, - /* 350 */ 98, 98, 98, 98, 230, 96, 96, 96, 96, 95, - /* 360 */ 95, 94, 94, 94, 93, 351, 325, 969, 227, 92, - /* 370 */ 89, 178, 373, 300, 99, 100, 90, 842, 842, 854, - /* 380 */ 857, 846, 846, 97, 97, 98, 98, 98, 98, 921, - /* 390 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, - /* 400 */ 351, 325, 449, 447, 447, 447, 147, 737, 737, 99, - /* 410 */ 100, 90, 842, 842, 854, 857, 846, 846, 97, 97, - /* 420 */ 98, 98, 98, 98, 296, 96, 96, 96, 96, 95, - /* 430 */ 95, 94, 94, 94, 93, 351, 325, 419, 231, 958, - /* 440 */ 958, 158, 25, 422, 99, 100, 90, 842, 842, 854, - /* 450 */ 857, 846, 846, 97, 97, 98, 98, 98, 98, 450, - /* 460 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, - /* 470 */ 351, 443, 224, 224, 420, 958, 958, 962, 325, 52, - /* 480 */ 52, 959, 960, 176, 415, 78, 99, 100, 90, 842, - /* 490 */ 842, 854, 857, 846, 846, 97, 97, 98, 98, 98, - /* 500 */ 98, 379, 96, 96, 96, 96, 95, 95, 94, 94, - /* 510 */ 94, 93, 351, 325, 428, 418, 298, 959, 960, 962, - /* 520 */ 81, 99, 88, 90, 842, 842, 854, 857, 846, 846, - /* 530 */ 97, 97, 98, 98, 98, 98, 717, 96, 96, 96, - /* 540 */ 96, 95, 95, 94, 94, 94, 93, 351, 325, 843, - /* 550 */ 843, 855, 858, 996, 318, 343, 379, 100, 90, 842, - /* 560 */ 842, 854, 857, 846, 846, 97, 97, 98, 98, 98, - /* 570 */ 98, 450, 96, 96, 96, 96, 95, 95, 94, 94, - /* 580 */ 94, 93, 351, 325, 350, 350, 350, 260, 377, 340, - /* 590 */ 929, 52, 52, 90, 842, 842, 854, 857, 846, 846, - /* 600 */ 97, 97, 98, 98, 98, 98, 361, 96, 96, 96, - /* 610 */ 96, 95, 95, 94, 94, 94, 93, 351, 86, 445, - /* 620 */ 847, 3, 1203, 361, 360, 378, 344, 813, 958, 958, - /* 630 */ 1300, 86, 445, 729, 3, 212, 169, 287, 405, 282, - /* 640 */ 404, 199, 232, 450, 300, 760, 83, 84, 280, 245, - /* 650 */ 262, 365, 251, 85, 352, 352, 92, 89, 178, 83, - /* 660 */ 84, 242, 412, 52, 52, 448, 85, 352, 352, 246, - /* 670 */ 959, 960, 194, 455, 670, 402, 399, 398, 448, 243, - /* 680 */ 221, 114, 434, 776, 361, 450, 397, 268, 747, 224, - /* 690 */ 224, 132, 132, 198, 832, 434, 452, 451, 428, 427, - /* 700 */ 819, 415, 734, 713, 132, 52, 52, 832, 268, 452, - /* 710 */ 451, 734, 194, 819, 363, 402, 399, 398, 450, 1271, - /* 720 */ 1271, 23, 958, 958, 86, 445, 397, 3, 228, 429, - /* 730 */ 895, 824, 824, 826, 827, 19, 203, 720, 52, 52, - /* 740 */ 428, 408, 439, 249, 824, 824, 826, 827, 19, 229, - /* 750 */ 403, 153, 83, 84, 761, 177, 241, 450, 721, 85, - /* 760 */ 352, 352, 120, 157, 959, 960, 58, 977, 409, 355, - /* 770 */ 330, 448, 268, 428, 430, 320, 790, 32, 32, 86, - /* 780 */ 445, 776, 3, 341, 98, 98, 98, 98, 434, 96, - /* 790 */ 96, 96, 96, 95, 95, 94, 94, 94, 93, 351, - /* 800 */ 832, 120, 452, 451, 813, 887, 819, 83, 84, 977, - /* 810 */ 813, 132, 410, 920, 85, 352, 352, 132, 407, 789, - /* 820 */ 958, 958, 92, 89, 178, 917, 448, 262, 370, 261, - /* 830 */ 82, 914, 80, 262, 370, 261, 776, 824, 824, 826, - /* 840 */ 827, 19, 934, 434, 96, 96, 96, 96, 95, 95, - /* 850 */ 94, 94, 94, 93, 351, 832, 74, 452, 451, 958, - /* 860 */ 958, 819, 959, 960, 120, 92, 89, 178, 945, 2, - /* 870 */ 918, 965, 268, 1, 976, 76, 445, 762, 3, 708, - /* 880 */ 901, 901, 387, 958, 958, 757, 919, 371, 740, 778, - /* 890 */ 756, 257, 824, 824, 826, 827, 19, 417, 741, 450, - /* 900 */ 24, 959, 960, 83, 84, 369, 958, 958, 177, 226, - /* 910 */ 85, 352, 352, 885, 315, 314, 313, 215, 311, 10, - /* 920 */ 10, 683, 448, 349, 348, 959, 960, 909, 777, 157, - /* 930 */ 120, 958, 958, 337, 776, 416, 711, 310, 450, 434, - /* 940 */ 450, 321, 450, 791, 103, 200, 175, 450, 959, 960, - /* 950 */ 908, 832, 792, 452, 451, 9, 9, 819, 10, 10, - /* 960 */ 52, 52, 51, 51, 180, 716, 248, 10, 10, 171, - /* 970 */ 170, 167, 339, 959, 960, 247, 984, 702, 702, 450, - /* 980 */ 715, 233, 686, 982, 889, 983, 182, 914, 824, 824, - /* 990 */ 826, 827, 19, 183, 256, 423, 132, 181, 394, 10, - /* 1000 */ 10, 889, 891, 749, 958, 958, 917, 268, 985, 198, - /* 1010 */ 985, 349, 348, 425, 415, 299, 817, 832, 326, 825, - /* 1020 */ 120, 332, 133, 819, 268, 98, 98, 98, 98, 91, - /* 1030 */ 96, 96, 96, 96, 95, 95, 94, 94, 94, 93, - /* 1040 */ 351, 157, 810, 371, 382, 359, 959, 960, 358, 268, - /* 1050 */ 450, 918, 368, 324, 824, 824, 826, 450, 709, 450, - /* 1060 */ 264, 380, 889, 450, 877, 746, 253, 919, 255, 433, - /* 1070 */ 36, 36, 234, 450, 234, 120, 269, 37, 37, 12, - /* 1080 */ 12, 334, 272, 27, 27, 450, 330, 118, 450, 162, - /* 1090 */ 742, 280, 450, 38, 38, 450, 985, 356, 985, 450, - /* 1100 */ 709, 1210, 450, 132, 450, 39, 39, 450, 40, 40, - /* 1110 */ 450, 362, 41, 41, 450, 42, 42, 450, 254, 28, - /* 1120 */ 28, 450, 29, 29, 31, 31, 450, 43, 43, 450, - /* 1130 */ 44, 44, 450, 714, 45, 45, 450, 11, 11, 767, - /* 1140 */ 450, 46, 46, 450, 268, 450, 105, 105, 450, 47, - /* 1150 */ 47, 450, 48, 48, 450, 237, 33, 33, 450, 172, - /* 1160 */ 49, 49, 450, 50, 50, 34, 34, 274, 122, 122, - /* 1170 */ 450, 123, 123, 450, 124, 124, 450, 898, 56, 56, - /* 1180 */ 450, 897, 35, 35, 450, 267, 450, 817, 450, 817, - /* 1190 */ 106, 106, 450, 53, 53, 385, 107, 107, 450, 817, - /* 1200 */ 108, 108, 817, 450, 104, 104, 121, 121, 119, 119, - /* 1210 */ 450, 117, 112, 112, 450, 276, 450, 225, 111, 111, - /* 1220 */ 450, 730, 450, 109, 109, 450, 673, 674, 675, 912, - /* 1230 */ 110, 110, 317, 998, 55, 55, 57, 57, 692, 331, - /* 1240 */ 54, 54, 26, 26, 696, 30, 30, 317, 937, 197, - /* 1250 */ 196, 195, 335, 281, 336, 446, 331, 745, 689, 436, - /* 1260 */ 440, 444, 120, 72, 386, 223, 175, 345, 757, 933, - /* 1270 */ 20, 286, 319, 756, 815, 372, 374, 202, 202, 202, - /* 1280 */ 263, 395, 285, 74, 208, 21, 696, 719, 718, 884, - /* 1290 */ 120, 120, 120, 120, 120, 754, 278, 828, 77, 74, - /* 1300 */ 726, 727, 785, 783, 880, 202, 999, 208, 894, 893, - /* 1310 */ 894, 893, 694, 816, 763, 116, 774, 1290, 431, 432, - /* 1320 */ 302, 999, 390, 303, 823, 697, 691, 680, 159, 289, - /* 1330 */ 679, 884, 681, 952, 291, 218, 293, 7, 316, 828, - /* 1340 */ 173, 805, 259, 364, 252, 911, 376, 713, 295, 435, - /* 1350 */ 308, 168, 955, 993, 135, 400, 990, 284, 882, 881, - /* 1360 */ 205, 928, 926, 59, 333, 62, 144, 156, 130, 72, - /* 1370 */ 802, 366, 367, 393, 137, 185, 189, 160, 139, 383, - /* 1380 */ 67, 896, 140, 141, 142, 148, 389, 812, 775, 266, - /* 1390 */ 219, 190, 154, 391, 913, 876, 271, 406, 191, 322, - /* 1400 */ 682, 733, 192, 342, 732, 724, 731, 711, 723, 421, - /* 1410 */ 705, 71, 323, 6, 204, 771, 288, 79, 297, 346, - /* 1420 */ 772, 704, 290, 283, 703, 770, 292, 294, 967, 239, - /* 1430 */ 769, 102, 862, 438, 426, 240, 424, 442, 73, 213, - /* 1440 */ 688, 238, 22, 453, 953, 214, 217, 216, 454, 677, - /* 1450 */ 676, 671, 753, 125, 115, 235, 126, 669, 353, 166, - /* 1460 */ 127, 244, 179, 357, 306, 304, 305, 307, 113, 892, - /* 1470 */ 327, 890, 811, 328, 134, 128, 136, 138, 743, 258, - /* 1480 */ 907, 184, 143, 129, 910, 186, 63, 64, 145, 187, - /* 1490 */ 906, 65, 8, 66, 13, 188, 202, 899, 265, 149, - /* 1500 */ 987, 388, 150, 685, 161, 392, 285, 193, 279, 396, - /* 1510 */ 151, 401, 68, 14, 15, 722, 69, 236, 831, 131, - /* 1520 */ 830, 860, 70, 751, 16, 414, 755, 4, 174, 220, - /* 1530 */ 222, 784, 201, 152, 779, 77, 74, 17, 18, 875, - /* 1540 */ 861, 859, 916, 864, 915, 207, 206, 942, 163, 437, - /* 1550 */ 948, 943, 164, 209, 1002, 441, 863, 165, 210, 829, - /* 1560 */ 695, 87, 312, 211, 1292, 1291, 309, -}; -static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 19, 95, 53, 97, 22, 24, 24, 101, 27, 28, - /* 10 */ 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, - /* 20 */ 39, 40, 41, 152, 43, 44, 45, 46, 47, 48, - /* 30 */ 49, 50, 51, 52, 53, 19, 55, 55, 132, 133, - /* 40 */ 134, 1, 2, 27, 28, 29, 30, 31, 32, 33, - /* 50 */ 34, 35, 36, 37, 38, 39, 40, 41, 187, 43, - /* 60 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 70 */ 47, 48, 49, 50, 51, 52, 53, 61, 97, 97, - /* 80 */ 19, 49, 50, 51, 52, 53, 70, 26, 27, 28, - /* 90 */ 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, - /* 100 */ 39, 40, 41, 152, 43, 44, 45, 46, 47, 48, - /* 110 */ 49, 50, 51, 52, 53, 144, 145, 146, 147, 19, - /* 120 */ 16, 22, 92, 172, 173, 52, 53, 27, 28, 29, - /* 130 */ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, - /* 140 */ 40, 41, 81, 43, 44, 45, 46, 47, 48, 49, - /* 150 */ 50, 51, 52, 53, 55, 56, 19, 152, 207, 208, - /* 160 */ 115, 24, 117, 118, 27, 28, 29, 30, 31, 32, - /* 170 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 79, - /* 180 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 190 */ 53, 19, 88, 157, 90, 23, 97, 98, 193, 27, - /* 200 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - /* 210 */ 38, 39, 40, 41, 152, 43, 44, 45, 46, 47, - /* 220 */ 48, 49, 50, 51, 52, 53, 19, 22, 23, 172, - /* 230 */ 23, 26, 119, 120, 27, 28, 29, 30, 31, 32, - /* 240 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 187, - /* 250 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 260 */ 53, 19, 22, 23, 228, 23, 26, 231, 152, 27, - /* 270 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - /* 280 */ 38, 39, 40, 41, 172, 43, 44, 45, 46, 47, - /* 290 */ 48, 49, 50, 51, 52, 53, 19, 221, 222, 223, - /* 300 */ 23, 96, 152, 172, 27, 28, 29, 30, 31, 32, - /* 310 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 152, - /* 320 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 330 */ 53, 19, 0, 1, 2, 23, 96, 190, 191, 27, - /* 340 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - /* 350 */ 38, 39, 40, 41, 238, 43, 44, 45, 46, 47, - /* 360 */ 48, 49, 50, 51, 52, 53, 19, 185, 218, 221, - /* 370 */ 222, 223, 152, 152, 27, 28, 29, 30, 31, 32, - /* 380 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 241, - /* 390 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 400 */ 53, 19, 152, 168, 169, 170, 22, 190, 191, 27, - /* 410 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - /* 420 */ 38, 39, 40, 41, 152, 43, 44, 45, 46, 47, - /* 430 */ 48, 49, 50, 51, 52, 53, 19, 19, 218, 55, - /* 440 */ 56, 24, 22, 152, 27, 28, 29, 30, 31, 32, - /* 450 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 152, - /* 460 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 470 */ 53, 250, 194, 195, 56, 55, 56, 55, 19, 172, - /* 480 */ 173, 97, 98, 152, 206, 138, 27, 28, 29, 30, - /* 490 */ 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - /* 500 */ 41, 152, 43, 44, 45, 46, 47, 48, 49, 50, - /* 510 */ 51, 52, 53, 19, 207, 208, 152, 97, 98, 97, - /* 520 */ 138, 27, 28, 29, 30, 31, 32, 33, 34, 35, - /* 530 */ 36, 37, 38, 39, 40, 41, 181, 43, 44, 45, - /* 540 */ 46, 47, 48, 49, 50, 51, 52, 53, 19, 30, - /* 550 */ 31, 32, 33, 247, 248, 19, 152, 28, 29, 30, - /* 560 */ 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - /* 570 */ 41, 152, 43, 44, 45, 46, 47, 48, 49, 50, - /* 580 */ 51, 52, 53, 19, 168, 169, 170, 238, 19, 53, - /* 590 */ 152, 172, 173, 29, 30, 31, 32, 33, 34, 35, - /* 600 */ 36, 37, 38, 39, 40, 41, 152, 43, 44, 45, - /* 610 */ 46, 47, 48, 49, 50, 51, 52, 53, 19, 20, - /* 620 */ 101, 22, 23, 169, 170, 56, 207, 85, 55, 56, - /* 630 */ 23, 19, 20, 26, 22, 99, 100, 101, 102, 103, - /* 640 */ 104, 105, 238, 152, 152, 210, 47, 48, 112, 152, - /* 650 */ 108, 109, 110, 54, 55, 56, 221, 222, 223, 47, - /* 660 */ 48, 119, 120, 172, 173, 66, 54, 55, 56, 152, - /* 670 */ 97, 98, 99, 148, 149, 102, 103, 104, 66, 154, - /* 680 */ 23, 156, 83, 26, 230, 152, 113, 152, 163, 194, - /* 690 */ 195, 92, 92, 30, 95, 83, 97, 98, 207, 208, - /* 700 */ 101, 206, 179, 180, 92, 172, 173, 95, 152, 97, - /* 710 */ 98, 188, 99, 101, 219, 102, 103, 104, 152, 119, - /* 720 */ 120, 196, 55, 56, 19, 20, 113, 22, 193, 163, - /* 730 */ 11, 132, 133, 134, 135, 136, 24, 65, 172, 173, - /* 740 */ 207, 208, 250, 152, 132, 133, 134, 135, 136, 193, - /* 750 */ 78, 84, 47, 48, 49, 98, 199, 152, 86, 54, - /* 760 */ 55, 56, 196, 152, 97, 98, 209, 55, 163, 244, - /* 770 */ 107, 66, 152, 207, 208, 164, 175, 172, 173, 19, - /* 780 */ 20, 124, 22, 111, 38, 39, 40, 41, 83, 43, - /* 790 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 800 */ 95, 196, 97, 98, 85, 152, 101, 47, 48, 97, - /* 810 */ 85, 92, 207, 193, 54, 55, 56, 92, 49, 175, - /* 820 */ 55, 56, 221, 222, 223, 12, 66, 108, 109, 110, - /* 830 */ 137, 163, 139, 108, 109, 110, 26, 132, 133, 134, - /* 840 */ 135, 136, 152, 83, 43, 44, 45, 46, 47, 48, - /* 850 */ 49, 50, 51, 52, 53, 95, 26, 97, 98, 55, - /* 860 */ 56, 101, 97, 98, 196, 221, 222, 223, 146, 147, - /* 870 */ 57, 171, 152, 22, 26, 19, 20, 49, 22, 179, - /* 880 */ 108, 109, 110, 55, 56, 116, 73, 219, 75, 124, - /* 890 */ 121, 152, 132, 133, 134, 135, 136, 163, 85, 152, - /* 900 */ 232, 97, 98, 47, 48, 237, 55, 56, 98, 5, - /* 910 */ 54, 55, 56, 193, 10, 11, 12, 13, 14, 172, - /* 920 */ 173, 17, 66, 47, 48, 97, 98, 152, 124, 152, - /* 930 */ 196, 55, 56, 186, 124, 152, 106, 160, 152, 83, - /* 940 */ 152, 164, 152, 61, 22, 211, 212, 152, 97, 98, - /* 950 */ 152, 95, 70, 97, 98, 172, 173, 101, 172, 173, - /* 960 */ 172, 173, 172, 173, 60, 181, 62, 172, 173, 47, - /* 970 */ 48, 123, 186, 97, 98, 71, 100, 55, 56, 152, - /* 980 */ 181, 186, 21, 107, 152, 109, 82, 163, 132, 133, - /* 990 */ 134, 135, 136, 89, 16, 207, 92, 93, 19, 172, - /* 1000 */ 173, 169, 170, 195, 55, 56, 12, 152, 132, 30, - /* 1010 */ 134, 47, 48, 186, 206, 225, 152, 95, 114, 97, - /* 1020 */ 196, 245, 246, 101, 152, 38, 39, 40, 41, 42, - /* 1030 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 1040 */ 53, 152, 163, 219, 152, 141, 97, 98, 193, 152, - /* 1050 */ 152, 57, 91, 164, 132, 133, 134, 152, 55, 152, - /* 1060 */ 152, 237, 230, 152, 103, 193, 88, 73, 90, 75, - /* 1070 */ 172, 173, 183, 152, 185, 196, 152, 172, 173, 172, - /* 1080 */ 173, 217, 152, 172, 173, 152, 107, 22, 152, 24, - /* 1090 */ 193, 112, 152, 172, 173, 152, 132, 242, 134, 152, - /* 1100 */ 97, 140, 152, 92, 152, 172, 173, 152, 172, 173, - /* 1110 */ 152, 100, 172, 173, 152, 172, 173, 152, 140, 172, - /* 1120 */ 173, 152, 172, 173, 172, 173, 152, 172, 173, 152, - /* 1130 */ 172, 173, 152, 152, 172, 173, 152, 172, 173, 213, - /* 1140 */ 152, 172, 173, 152, 152, 152, 172, 173, 152, 172, - /* 1150 */ 173, 152, 172, 173, 152, 210, 172, 173, 152, 26, - /* 1160 */ 172, 173, 152, 172, 173, 172, 173, 152, 172, 173, - /* 1170 */ 152, 172, 173, 152, 172, 173, 152, 59, 172, 173, - /* 1180 */ 152, 63, 172, 173, 152, 193, 152, 152, 152, 152, - /* 1190 */ 172, 173, 152, 172, 173, 77, 172, 173, 152, 152, - /* 1200 */ 172, 173, 152, 152, 172, 173, 172, 173, 172, 173, - /* 1210 */ 152, 22, 172, 173, 152, 152, 152, 22, 172, 173, - /* 1220 */ 152, 152, 152, 172, 173, 152, 7, 8, 9, 163, - /* 1230 */ 172, 173, 22, 23, 172, 173, 172, 173, 166, 167, - /* 1240 */ 172, 173, 172, 173, 55, 172, 173, 22, 23, 108, - /* 1250 */ 109, 110, 217, 152, 217, 166, 167, 163, 163, 163, - /* 1260 */ 163, 163, 196, 130, 217, 211, 212, 217, 116, 23, - /* 1270 */ 22, 101, 26, 121, 23, 23, 23, 26, 26, 26, - /* 1280 */ 23, 23, 112, 26, 26, 37, 97, 100, 101, 55, - /* 1290 */ 196, 196, 196, 196, 196, 23, 23, 55, 26, 26, - /* 1300 */ 7, 8, 23, 152, 23, 26, 96, 26, 132, 132, - /* 1310 */ 134, 134, 23, 152, 152, 26, 152, 122, 152, 191, - /* 1320 */ 152, 96, 234, 152, 152, 152, 152, 152, 197, 210, - /* 1330 */ 152, 97, 152, 152, 210, 233, 210, 198, 150, 97, - /* 1340 */ 184, 201, 239, 214, 214, 201, 239, 180, 214, 227, - /* 1350 */ 200, 198, 155, 67, 243, 176, 69, 175, 175, 175, - /* 1360 */ 122, 159, 159, 240, 159, 240, 22, 220, 27, 130, - /* 1370 */ 201, 18, 159, 18, 189, 158, 158, 220, 192, 159, - /* 1380 */ 137, 236, 192, 192, 192, 189, 74, 189, 159, 235, - /* 1390 */ 159, 158, 22, 177, 201, 201, 159, 107, 158, 177, - /* 1400 */ 159, 174, 158, 76, 174, 182, 174, 106, 182, 125, - /* 1410 */ 174, 107, 177, 22, 159, 216, 215, 137, 159, 53, - /* 1420 */ 216, 176, 215, 174, 174, 216, 215, 215, 174, 229, - /* 1430 */ 216, 129, 224, 177, 126, 229, 127, 177, 128, 25, - /* 1440 */ 162, 226, 26, 161, 13, 153, 6, 153, 151, 151, - /* 1450 */ 151, 151, 205, 165, 178, 178, 165, 4, 3, 22, - /* 1460 */ 165, 142, 15, 94, 202, 204, 203, 201, 16, 23, - /* 1470 */ 249, 23, 120, 249, 246, 111, 131, 123, 20, 16, - /* 1480 */ 1, 125, 123, 111, 56, 64, 37, 37, 131, 122, - /* 1490 */ 1, 37, 5, 37, 22, 107, 26, 80, 140, 80, - /* 1500 */ 87, 72, 107, 20, 24, 19, 112, 105, 23, 79, - /* 1510 */ 22, 79, 22, 22, 22, 58, 22, 79, 23, 68, - /* 1520 */ 23, 23, 26, 116, 22, 26, 23, 22, 122, 23, - /* 1530 */ 23, 56, 64, 22, 124, 26, 26, 64, 64, 23, - /* 1540 */ 23, 23, 23, 11, 23, 22, 26, 23, 22, 24, - /* 1550 */ 1, 23, 22, 26, 251, 24, 23, 22, 122, 23, - /* 1560 */ 23, 22, 15, 122, 122, 122, 23, -}; -#define YY_SHIFT_USE_DFLT (1567) -#define YY_SHIFT_COUNT (455) -#define YY_SHIFT_MIN (-94) -#define YY_SHIFT_MAX (1549) -static const short yy_shift_ofst[] = { - /* 0 */ 40, 599, 904, 612, 760, 760, 760, 760, 725, -19, - /* 10 */ 16, 16, 100, 760, 760, 760, 760, 760, 760, 760, - /* 20 */ 876, 876, 573, 542, 719, 600, 61, 137, 172, 207, - /* 30 */ 242, 277, 312, 347, 382, 417, 459, 459, 459, 459, - /* 40 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, - /* 50 */ 459, 459, 459, 494, 459, 529, 564, 564, 705, 760, - /* 60 */ 760, 760, 760, 760, 760, 760, 760, 760, 760, 760, - /* 70 */ 760, 760, 760, 760, 760, 760, 760, 760, 760, 760, - /* 80 */ 760, 760, 760, 760, 760, 760, 760, 760, 760, 760, - /* 90 */ 856, 760, 760, 760, 760, 760, 760, 760, 760, 760, - /* 100 */ 760, 760, 760, 760, 987, 746, 746, 746, 746, 746, - /* 110 */ 801, 23, 32, 949, 961, 979, 964, 964, 949, 73, - /* 120 */ 113, -51, 1567, 1567, 1567, 536, 536, 536, 99, 99, - /* 130 */ 813, 813, 667, 205, 240, 949, 949, 949, 949, 949, - /* 140 */ 949, 949, 949, 949, 949, 949, 949, 949, 949, 949, - /* 150 */ 949, 949, 949, 949, 949, 332, 1011, 422, 422, 113, - /* 160 */ 30, 30, 30, 30, 30, 30, 1567, 1567, 1567, 922, - /* 170 */ -94, -94, 384, 613, 828, 420, 765, 804, 851, 949, - /* 180 */ 949, 949, 949, 949, 949, 949, 949, 949, 949, 949, - /* 190 */ 949, 949, 949, 949, 949, 672, 672, 672, 949, 949, - /* 200 */ 657, 949, 949, 949, -18, 949, 949, 994, 949, 949, - /* 210 */ 949, 949, 949, 949, 949, 949, 949, 949, 772, 1118, - /* 220 */ 712, 712, 712, 810, 45, 769, 1219, 1133, 418, 418, - /* 230 */ 569, 1133, 569, 830, 607, 663, 882, 418, 693, 882, - /* 240 */ 882, 848, 1152, 1065, 1286, 1238, 1238, 1287, 1287, 1238, - /* 250 */ 1344, 1341, 1239, 1353, 1353, 1353, 1353, 1238, 1355, 1239, - /* 260 */ 1344, 1341, 1341, 1239, 1238, 1355, 1243, 1312, 1238, 1238, - /* 270 */ 1355, 1370, 1238, 1355, 1238, 1355, 1370, 1290, 1290, 1290, - /* 280 */ 1327, 1370, 1290, 1301, 1290, 1327, 1290, 1290, 1284, 1304, - /* 290 */ 1284, 1304, 1284, 1304, 1284, 1304, 1238, 1391, 1238, 1280, - /* 300 */ 1370, 1366, 1366, 1370, 1302, 1308, 1310, 1309, 1239, 1414, - /* 310 */ 1416, 1431, 1431, 1440, 1440, 1440, 1440, 1567, 1567, 1567, - /* 320 */ 1567, 1567, 1567, 1567, 1567, 519, 978, 1210, 1225, 104, - /* 330 */ 1141, 1189, 1246, 1248, 1251, 1252, 1253, 1257, 1258, 1273, - /* 340 */ 1003, 1187, 1293, 1170, 1272, 1279, 1234, 1281, 1176, 1177, - /* 350 */ 1289, 1242, 1195, 1453, 1455, 1437, 1319, 1447, 1369, 1452, - /* 360 */ 1446, 1448, 1352, 1345, 1364, 1354, 1458, 1356, 1463, 1479, - /* 370 */ 1359, 1357, 1449, 1450, 1454, 1456, 1372, 1428, 1421, 1367, - /* 380 */ 1489, 1487, 1472, 1388, 1358, 1417, 1470, 1419, 1413, 1429, - /* 390 */ 1395, 1480, 1483, 1486, 1394, 1402, 1488, 1430, 1490, 1491, - /* 400 */ 1485, 1492, 1432, 1457, 1494, 1438, 1451, 1495, 1497, 1498, - /* 410 */ 1496, 1407, 1502, 1503, 1505, 1499, 1406, 1506, 1507, 1475, - /* 420 */ 1468, 1511, 1410, 1509, 1473, 1510, 1474, 1516, 1509, 1517, - /* 430 */ 1518, 1519, 1520, 1521, 1523, 1532, 1524, 1526, 1525, 1527, - /* 440 */ 1528, 1530, 1531, 1527, 1533, 1535, 1536, 1537, 1539, 1436, - /* 450 */ 1441, 1442, 1443, 1543, 1547, 1549, -}; -#define YY_REDUCE_USE_DFLT (-130) -#define YY_REDUCE_COUNT (324) -#define YY_REDUCE_MIN (-129) -#define YY_REDUCE_MAX (1300) -static const short yy_reduce_ofst[] = { - /* 0 */ -29, 566, 525, 605, -49, 307, 491, 533, 668, 435, - /* 10 */ 601, 644, 148, 747, 786, 795, 419, 788, 827, 790, - /* 20 */ 454, 832, 889, 495, 824, 734, 76, 76, 76, 76, - /* 30 */ 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, - /* 40 */ 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, - /* 50 */ 76, 76, 76, 76, 76, 76, 76, 76, 783, 898, - /* 60 */ 905, 907, 911, 921, 933, 936, 940, 943, 947, 950, - /* 70 */ 952, 955, 958, 962, 965, 969, 974, 977, 980, 984, - /* 80 */ 988, 991, 993, 996, 999, 1002, 1006, 1010, 1018, 1021, - /* 90 */ 1024, 1028, 1032, 1034, 1036, 1040, 1046, 1051, 1058, 1062, - /* 100 */ 1064, 1068, 1070, 1073, 76, 76, 76, 76, 76, 76, - /* 110 */ 76, 76, 76, 855, 36, 523, 235, 416, 777, 76, - /* 120 */ 278, 76, 76, 76, 76, 700, 700, 700, 150, 220, - /* 130 */ 147, 217, 221, 306, 306, 611, 5, 535, 556, 620, - /* 140 */ 720, 872, 897, 116, 864, 349, 1035, 1037, 404, 1047, - /* 150 */ 992, -129, 1050, 492, 62, 722, 879, 1072, 1089, 808, - /* 160 */ 1066, 1094, 1095, 1096, 1097, 1098, 776, 1054, 557, 57, - /* 170 */ 112, 131, 167, 182, 250, 272, 291, 331, 364, 438, - /* 180 */ 497, 517, 591, 653, 690, 739, 775, 798, 892, 908, - /* 190 */ 924, 930, 1015, 1063, 1069, 355, 784, 799, 981, 1101, - /* 200 */ 926, 1151, 1161, 1162, 945, 1164, 1166, 1128, 1168, 1171, - /* 210 */ 1172, 250, 1173, 1174, 1175, 1178, 1180, 1181, 1088, 1102, - /* 220 */ 1119, 1124, 1126, 926, 1131, 1139, 1188, 1140, 1129, 1130, - /* 230 */ 1103, 1144, 1107, 1179, 1156, 1167, 1182, 1134, 1122, 1183, - /* 240 */ 1184, 1150, 1153, 1197, 1111, 1202, 1203, 1123, 1125, 1205, - /* 250 */ 1147, 1185, 1169, 1186, 1190, 1191, 1192, 1213, 1217, 1193, - /* 260 */ 1157, 1196, 1198, 1194, 1220, 1218, 1145, 1154, 1229, 1231, - /* 270 */ 1233, 1216, 1237, 1240, 1241, 1244, 1222, 1227, 1230, 1232, - /* 280 */ 1223, 1235, 1236, 1245, 1249, 1226, 1250, 1254, 1199, 1201, - /* 290 */ 1204, 1207, 1209, 1211, 1214, 1212, 1255, 1208, 1259, 1215, - /* 300 */ 1256, 1200, 1206, 1260, 1247, 1261, 1263, 1262, 1266, 1278, - /* 310 */ 1282, 1292, 1294, 1297, 1298, 1299, 1300, 1221, 1224, 1228, - /* 320 */ 1288, 1291, 1276, 1277, 1295, -}; -static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1281, 1271, 1271, 1271, 1203, 1203, 1203, 1203, 1271, 1096, - /* 10 */ 1125, 1125, 1255, 1332, 1332, 1332, 1332, 1332, 1332, 1202, - /* 20 */ 1332, 1332, 1332, 1332, 1271, 1100, 1131, 1332, 1332, 1332, - /* 30 */ 1332, 1204, 1205, 1332, 1332, 1332, 1254, 1256, 1141, 1140, - /* 40 */ 1139, 1138, 1237, 1112, 1136, 1129, 1133, 1204, 1198, 1199, - /* 50 */ 1197, 1201, 1205, 1332, 1132, 1167, 1182, 1166, 1332, 1332, - /* 60 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 70 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 80 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 90 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 100 */ 1332, 1332, 1332, 1332, 1176, 1181, 1188, 1180, 1177, 1169, - /* 110 */ 1168, 1170, 1171, 1332, 1019, 1067, 1332, 1332, 1332, 1172, - /* 120 */ 1332, 1173, 1185, 1184, 1183, 1262, 1289, 1288, 1332, 1332, - /* 130 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 140 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 150 */ 1332, 1332, 1332, 1332, 1332, 1281, 1271, 1025, 1025, 1332, - /* 160 */ 1271, 1271, 1271, 1271, 1271, 1271, 1267, 1100, 1091, 1332, - /* 170 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 180 */ 1259, 1257, 1332, 1218, 1332, 1332, 1332, 1332, 1332, 1332, - /* 190 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 200 */ 1332, 1332, 1332, 1332, 1096, 1332, 1332, 1332, 1332, 1332, - /* 210 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1283, 1332, 1232, - /* 220 */ 1096, 1096, 1096, 1098, 1080, 1090, 1004, 1135, 1114, 1114, - /* 230 */ 1321, 1135, 1321, 1042, 1303, 1039, 1125, 1114, 1200, 1125, - /* 240 */ 1125, 1097, 1090, 1332, 1324, 1105, 1105, 1323, 1323, 1105, - /* 250 */ 1146, 1070, 1135, 1076, 1076, 1076, 1076, 1105, 1016, 1135, - /* 260 */ 1146, 1070, 1070, 1135, 1105, 1016, 1236, 1318, 1105, 1105, - /* 270 */ 1016, 1211, 1105, 1016, 1105, 1016, 1211, 1068, 1068, 1068, - /* 280 */ 1057, 1211, 1068, 1042, 1068, 1057, 1068, 1068, 1118, 1113, - /* 290 */ 1118, 1113, 1118, 1113, 1118, 1113, 1105, 1206, 1105, 1332, - /* 300 */ 1211, 1215, 1215, 1211, 1130, 1119, 1128, 1126, 1135, 1022, - /* 310 */ 1060, 1286, 1286, 1282, 1282, 1282, 1282, 1329, 1329, 1267, - /* 320 */ 1298, 1298, 1044, 1044, 1298, 1332, 1332, 1332, 1332, 1332, - /* 330 */ 1332, 1293, 1332, 1220, 1332, 1332, 1332, 1332, 1332, 1332, - /* 340 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 350 */ 1332, 1332, 1152, 1332, 1000, 1264, 1332, 1332, 1263, 1332, - /* 360 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 370 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1320, - /* 380 */ 1332, 1332, 1332, 1332, 1332, 1332, 1235, 1234, 1332, 1332, - /* 390 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 400 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, - /* 410 */ 1332, 1082, 1332, 1332, 1332, 1307, 1332, 1332, 1332, 1332, - /* 420 */ 1332, 1332, 1332, 1127, 1332, 1120, 1332, 1332, 1311, 1332, - /* 430 */ 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1332, 1273, - /* 440 */ 1332, 1332, 1332, 1272, 1332, 1332, 1332, 1332, 1332, 1154, - /* 450 */ 1332, 1153, 1157, 1332, 1010, 1332, -}; -/********** End of lemon-generated parsing tables *****************************/ - -/* The next table maps tokens (terminal symbols) into fallback tokens. -** If a construct like the following: -** -** %fallback ID X Y Z. -** -** appears in the grammar, then ID becomes a fallback token for X, Y, -** and Z. Whenever one of the tokens X, Y, or Z is input to the parser -** but it does not parse, the type of the token is changed to ID and -** the parse is retried before an error is thrown. -** -** This feature can be used, for example, to cause some keywords in a language -** to revert to identifiers if they keyword does not apply in the context where -** it appears. -*/ -#ifdef YYFALLBACK -static const YYCODETYPE yyFallback[] = { - 0, /* $ => nothing */ - 0, /* SEMI => nothing */ - 55, /* EXPLAIN => ID */ - 55, /* QUERY => ID */ - 55, /* PLAN => ID */ - 55, /* BEGIN => ID */ - 0, /* TRANSACTION => nothing */ - 55, /* DEFERRED => ID */ - 55, /* IMMEDIATE => ID */ - 55, /* EXCLUSIVE => ID */ - 0, /* COMMIT => nothing */ - 55, /* END => ID */ - 55, /* ROLLBACK => ID */ - 55, /* SAVEPOINT => ID */ - 55, /* RELEASE => ID */ - 0, /* TO => nothing */ - 0, /* TABLE => nothing */ - 0, /* CREATE => nothing */ - 55, /* IF => ID */ - 0, /* NOT => nothing */ - 0, /* EXISTS => nothing */ - 55, /* TEMP => ID */ - 0, /* LP => nothing */ - 0, /* RP => nothing */ - 0, /* AS => nothing */ - 55, /* WITHOUT => ID */ - 0, /* COMMA => nothing */ - 0, /* OR => nothing */ - 0, /* AND => nothing */ - 0, /* IS => nothing */ - 55, /* MATCH => ID */ - 55, /* LIKE_KW => ID */ - 0, /* BETWEEN => nothing */ - 0, /* IN => nothing */ - 0, /* ISNULL => nothing */ - 0, /* NOTNULL => nothing */ - 0, /* NE => nothing */ - 0, /* EQ => nothing */ - 0, /* GT => nothing */ - 0, /* LE => nothing */ - 0, /* LT => nothing */ - 0, /* GE => nothing */ - 0, /* ESCAPE => nothing */ - 0, /* BITAND => nothing */ - 0, /* BITOR => nothing */ - 0, /* LSHIFT => nothing */ - 0, /* RSHIFT => nothing */ - 0, /* PLUS => nothing */ - 0, /* MINUS => nothing */ - 0, /* STAR => nothing */ - 0, /* SLASH => nothing */ - 0, /* REM => nothing */ - 0, /* CONCAT => nothing */ - 0, /* COLLATE => nothing */ - 0, /* BITNOT => nothing */ - 0, /* ID => nothing */ - 0, /* INDEXED => nothing */ - 55, /* ABORT => ID */ - 55, /* ACTION => ID */ - 55, /* AFTER => ID */ - 55, /* ANALYZE => ID */ - 55, /* ASC => ID */ - 55, /* ATTACH => ID */ - 55, /* BEFORE => ID */ - 55, /* BY => ID */ - 55, /* CASCADE => ID */ - 55, /* CAST => ID */ - 55, /* COLUMNKW => ID */ - 55, /* CONFLICT => ID */ - 55, /* DATABASE => ID */ - 55, /* DESC => ID */ - 55, /* DETACH => ID */ - 55, /* EACH => ID */ - 55, /* FAIL => ID */ - 55, /* FOR => ID */ - 55, /* IGNORE => ID */ - 55, /* INITIALLY => ID */ - 55, /* INSTEAD => ID */ - 55, /* NO => ID */ - 55, /* KEY => ID */ - 55, /* OF => ID */ - 55, /* OFFSET => ID */ - 55, /* PRAGMA => ID */ - 55, /* RAISE => ID */ - 55, /* RECURSIVE => ID */ - 55, /* REPLACE => ID */ - 55, /* RESTRICT => ID */ - 55, /* ROW => ID */ - 55, /* TRIGGER => ID */ - 55, /* VACUUM => ID */ - 55, /* VIEW => ID */ - 55, /* VIRTUAL => ID */ - 55, /* WITH => ID */ - 55, /* REINDEX => ID */ - 55, /* RENAME => ID */ - 55, /* CTIME_KW => ID */ -}; -#endif /* YYFALLBACK */ - -/* The following structure represents a single element of the -** parser's stack. Information stored includes: -** -** + The state number for the parser at this level of the stack. -** -** + The value of the token stored at this level of the stack. -** (In other words, the "major" token.) -** -** + The semantic value stored at this level of the stack. This is -** the information used by the action routines in the grammar. -** It is sometimes called the "minor" token. -** -** After the "shift" half of a SHIFTREDUCE action, the stateno field -** actually contains the reduce action for the second half of the -** SHIFTREDUCE. -*/ -struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ - YYCODETYPE major; /* The major token value. This is the code - ** number for the token at this stack level */ - YYMINORTYPE minor; /* The user-supplied minor token value. This - ** is the value of the token */ -}; -typedef struct yyStackEntry yyStackEntry; - -/* The state of the parser is completely contained in an instance of -** the following structure */ -struct yyParser { - yyStackEntry *yytos; /* Pointer to top element of the stack */ -#ifdef YYTRACKMAXSTACKDEPTH - int yyhwm; /* High-water mark of the stack */ -#endif -#ifndef YYNOERRORRECOVERY - int yyerrcnt; /* Shifts left before out of the error */ -#endif - sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ -#if YYSTACKDEPTH<=0 - int yystksz; /* Current side of the stack */ - yyStackEntry *yystack; /* The parser's stack */ - yyStackEntry yystk0; /* First stack entry */ -#else - yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ -#endif -}; -typedef struct yyParser yyParser; - -#ifndef NDEBUG -/* #include */ -static FILE *yyTraceFILE = 0; -static char *yyTracePrompt = 0; -#endif /* NDEBUG */ - -#ifndef NDEBUG -/* -** Turn parser tracing on by giving a stream to which to write the trace -** and a prompt to preface each trace message. Tracing is turned off -** by making either argument NULL -** -** Inputs: -**
      -**
    • A FILE* to which trace output should be written. -** If NULL, then tracing is turned off. -**
    • A prefix string written at the beginning of every -** line of trace output. If NULL, then tracing is -** turned off. -**
    -** -** Outputs: -** None. -*/ -SQLITE_PRIVATE void sqlite3ParserTrace(FILE *TraceFILE, char *zTracePrompt){ - yyTraceFILE = TraceFILE; - yyTracePrompt = zTracePrompt; - if( yyTraceFILE==0 ) yyTracePrompt = 0; - else if( yyTracePrompt==0 ) yyTraceFILE = 0; -} -#endif /* NDEBUG */ - -#ifndef NDEBUG -/* For tracing shifts, the names of all terminals and nonterminals -** are required. The following table supplies these names */ -static const char *const yyTokenName[] = { - "$", "SEMI", "EXPLAIN", "QUERY", - "PLAN", "BEGIN", "TRANSACTION", "DEFERRED", - "IMMEDIATE", "EXCLUSIVE", "COMMIT", "END", - "ROLLBACK", "SAVEPOINT", "RELEASE", "TO", - "TABLE", "CREATE", "IF", "NOT", - "EXISTS", "TEMP", "LP", "RP", - "AS", "WITHOUT", "COMMA", "OR", - "AND", "IS", "MATCH", "LIKE_KW", - "BETWEEN", "IN", "ISNULL", "NOTNULL", - "NE", "EQ", "GT", "LE", - "LT", "GE", "ESCAPE", "BITAND", - "BITOR", "LSHIFT", "RSHIFT", "PLUS", - "MINUS", "STAR", "SLASH", "REM", - "CONCAT", "COLLATE", "BITNOT", "ID", - "INDEXED", "ABORT", "ACTION", "AFTER", - "ANALYZE", "ASC", "ATTACH", "BEFORE", - "BY", "CASCADE", "CAST", "COLUMNKW", - "CONFLICT", "DATABASE", "DESC", "DETACH", - "EACH", "FAIL", "FOR", "IGNORE", - "INITIALLY", "INSTEAD", "NO", "KEY", - "OF", "OFFSET", "PRAGMA", "RAISE", - "RECURSIVE", "REPLACE", "RESTRICT", "ROW", - "TRIGGER", "VACUUM", "VIEW", "VIRTUAL", - "WITH", "REINDEX", "RENAME", "CTIME_KW", - "ANY", "STRING", "JOIN_KW", "CONSTRAINT", - "DEFAULT", "NULL", "PRIMARY", "UNIQUE", - "CHECK", "REFERENCES", "AUTOINCR", "ON", - "INSERT", "DELETE", "UPDATE", "SET", - "DEFERRABLE", "FOREIGN", "DROP", "UNION", - "ALL", "EXCEPT", "INTERSECT", "SELECT", - "VALUES", "DISTINCT", "DOT", "FROM", - "JOIN", "USING", "ORDER", "GROUP", - "HAVING", "LIMIT", "WHERE", "INTO", - "FLOAT", "BLOB", "INTEGER", "VARIABLE", - "CASE", "WHEN", "THEN", "ELSE", - "INDEX", "ALTER", "ADD", "error", - "input", "cmdlist", "ecmd", "explain", - "cmdx", "cmd", "transtype", "trans_opt", - "nm", "savepoint_opt", "create_table", "create_table_args", - "createkw", "temp", "ifnotexists", "dbnm", - "columnlist", "conslist_opt", "table_options", "select", - "columnname", "carglist", "typetoken", "typename", - "signed", "plus_num", "minus_num", "ccons", - "term", "expr", "onconf", "sortorder", - "autoinc", "eidlist_opt", "refargs", "defer_subclause", - "refarg", "refact", "init_deferred_pred_opt", "conslist", - "tconscomma", "tcons", "sortlist", "eidlist", - "defer_subclause_opt", "orconf", "resolvetype", "raisetype", - "ifexists", "fullname", "selectnowith", "oneselect", - "with", "multiselect_op", "distinct", "selcollist", - "from", "where_opt", "groupby_opt", "having_opt", - "orderby_opt", "limit_opt", "values", "nexprlist", - "exprlist", "sclp", "as", "seltablist", - "stl_prefix", "joinop", "indexed_opt", "on_opt", - "using_opt", "idlist", "setlist", "insert_cmd", - "idlist_opt", "likeop", "between_op", "in_op", - "paren_exprlist", "case_operand", "case_exprlist", "case_else", - "uniqueflag", "collate", "nmnum", "trigger_decl", - "trigger_cmd_list", "trigger_time", "trigger_event", "foreach_clause", - "when_clause", "trigger_cmd", "trnm", "tridxby", - "database_kw_opt", "key_opt", "add_column_fullname", "kwcolumn_opt", - "create_vtab", "vtabarglist", "vtabarg", "vtabargtoken", - "lp", "anylist", "wqlist", -}; -#endif /* NDEBUG */ - -#ifndef NDEBUG -/* For tracing reduce actions, the names of all rules are required. -*/ -static const char *const yyRuleName[] = { - /* 0 */ "explain ::= EXPLAIN", - /* 1 */ "explain ::= EXPLAIN QUERY PLAN", - /* 2 */ "cmdx ::= cmd", - /* 3 */ "cmd ::= BEGIN transtype trans_opt", - /* 4 */ "transtype ::=", - /* 5 */ "transtype ::= DEFERRED", - /* 6 */ "transtype ::= IMMEDIATE", - /* 7 */ "transtype ::= EXCLUSIVE", - /* 8 */ "cmd ::= COMMIT trans_opt", - /* 9 */ "cmd ::= END trans_opt", - /* 10 */ "cmd ::= ROLLBACK trans_opt", - /* 11 */ "cmd ::= SAVEPOINT nm", - /* 12 */ "cmd ::= RELEASE savepoint_opt nm", - /* 13 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", - /* 14 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", - /* 15 */ "createkw ::= CREATE", - /* 16 */ "ifnotexists ::=", - /* 17 */ "ifnotexists ::= IF NOT EXISTS", - /* 18 */ "temp ::= TEMP", - /* 19 */ "temp ::=", - /* 20 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", - /* 21 */ "create_table_args ::= AS select", - /* 22 */ "table_options ::=", - /* 23 */ "table_options ::= WITHOUT nm", - /* 24 */ "columnname ::= nm typetoken", - /* 25 */ "typetoken ::=", - /* 26 */ "typetoken ::= typename LP signed RP", - /* 27 */ "typetoken ::= typename LP signed COMMA signed RP", - /* 28 */ "typename ::= typename ID|STRING", - /* 29 */ "ccons ::= CONSTRAINT nm", - /* 30 */ "ccons ::= DEFAULT term", - /* 31 */ "ccons ::= DEFAULT LP expr RP", - /* 32 */ "ccons ::= DEFAULT PLUS term", - /* 33 */ "ccons ::= DEFAULT MINUS term", - /* 34 */ "ccons ::= DEFAULT ID|INDEXED", - /* 35 */ "ccons ::= NOT NULL onconf", - /* 36 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", - /* 37 */ "ccons ::= UNIQUE onconf", - /* 38 */ "ccons ::= CHECK LP expr RP", - /* 39 */ "ccons ::= REFERENCES nm eidlist_opt refargs", - /* 40 */ "ccons ::= defer_subclause", - /* 41 */ "ccons ::= COLLATE ID|STRING", - /* 42 */ "autoinc ::=", - /* 43 */ "autoinc ::= AUTOINCR", - /* 44 */ "refargs ::=", - /* 45 */ "refargs ::= refargs refarg", - /* 46 */ "refarg ::= MATCH nm", - /* 47 */ "refarg ::= ON INSERT refact", - /* 48 */ "refarg ::= ON DELETE refact", - /* 49 */ "refarg ::= ON UPDATE refact", - /* 50 */ "refact ::= SET NULL", - /* 51 */ "refact ::= SET DEFAULT", - /* 52 */ "refact ::= CASCADE", - /* 53 */ "refact ::= RESTRICT", - /* 54 */ "refact ::= NO ACTION", - /* 55 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", - /* 56 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", - /* 57 */ "init_deferred_pred_opt ::=", - /* 58 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", - /* 59 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", - /* 60 */ "conslist_opt ::=", - /* 61 */ "tconscomma ::= COMMA", - /* 62 */ "tcons ::= CONSTRAINT nm", - /* 63 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", - /* 64 */ "tcons ::= UNIQUE LP sortlist RP onconf", - /* 65 */ "tcons ::= CHECK LP expr RP onconf", - /* 66 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", - /* 67 */ "defer_subclause_opt ::=", - /* 68 */ "onconf ::=", - /* 69 */ "onconf ::= ON CONFLICT resolvetype", - /* 70 */ "orconf ::=", - /* 71 */ "orconf ::= OR resolvetype", - /* 72 */ "resolvetype ::= IGNORE", - /* 73 */ "resolvetype ::= REPLACE", - /* 74 */ "cmd ::= DROP TABLE ifexists fullname", - /* 75 */ "ifexists ::= IF EXISTS", - /* 76 */ "ifexists ::=", - /* 77 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", - /* 78 */ "cmd ::= DROP VIEW ifexists fullname", - /* 79 */ "cmd ::= select", - /* 80 */ "select ::= with selectnowith", - /* 81 */ "selectnowith ::= selectnowith multiselect_op oneselect", - /* 82 */ "multiselect_op ::= UNION", - /* 83 */ "multiselect_op ::= UNION ALL", - /* 84 */ "multiselect_op ::= EXCEPT|INTERSECT", - /* 85 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", - /* 86 */ "values ::= VALUES LP nexprlist RP", - /* 87 */ "values ::= values COMMA LP exprlist RP", - /* 88 */ "distinct ::= DISTINCT", - /* 89 */ "distinct ::= ALL", - /* 90 */ "distinct ::=", - /* 91 */ "sclp ::=", - /* 92 */ "selcollist ::= sclp expr as", - /* 93 */ "selcollist ::= sclp STAR", - /* 94 */ "selcollist ::= sclp nm DOT STAR", - /* 95 */ "as ::= AS nm", - /* 96 */ "as ::=", - /* 97 */ "from ::=", - /* 98 */ "from ::= FROM seltablist", - /* 99 */ "stl_prefix ::= seltablist joinop", - /* 100 */ "stl_prefix ::=", - /* 101 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", - /* 102 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", - /* 103 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", - /* 104 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", - /* 105 */ "dbnm ::=", - /* 106 */ "dbnm ::= DOT nm", - /* 107 */ "fullname ::= nm dbnm", - /* 108 */ "joinop ::= COMMA|JOIN", - /* 109 */ "joinop ::= JOIN_KW JOIN", - /* 110 */ "joinop ::= JOIN_KW nm JOIN", - /* 111 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 112 */ "on_opt ::= ON expr", - /* 113 */ "on_opt ::=", - /* 114 */ "indexed_opt ::=", - /* 115 */ "indexed_opt ::= INDEXED BY nm", - /* 116 */ "indexed_opt ::= NOT INDEXED", - /* 117 */ "using_opt ::= USING LP idlist RP", - /* 118 */ "using_opt ::=", - /* 119 */ "orderby_opt ::=", - /* 120 */ "orderby_opt ::= ORDER BY sortlist", - /* 121 */ "sortlist ::= sortlist COMMA expr sortorder", - /* 122 */ "sortlist ::= expr sortorder", - /* 123 */ "sortorder ::= ASC", - /* 124 */ "sortorder ::= DESC", - /* 125 */ "sortorder ::=", - /* 126 */ "groupby_opt ::=", - /* 127 */ "groupby_opt ::= GROUP BY nexprlist", - /* 128 */ "having_opt ::=", - /* 129 */ "having_opt ::= HAVING expr", - /* 130 */ "limit_opt ::=", - /* 131 */ "limit_opt ::= LIMIT expr", - /* 132 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 133 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 134 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", - /* 135 */ "where_opt ::=", - /* 136 */ "where_opt ::= WHERE expr", - /* 137 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", - /* 138 */ "setlist ::= setlist COMMA nm EQ expr", - /* 139 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", - /* 140 */ "setlist ::= nm EQ expr", - /* 141 */ "setlist ::= LP idlist RP EQ expr", - /* 142 */ "cmd ::= with insert_cmd INTO fullname idlist_opt select", - /* 143 */ "cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES", - /* 144 */ "insert_cmd ::= INSERT orconf", - /* 145 */ "insert_cmd ::= REPLACE", - /* 146 */ "idlist_opt ::=", - /* 147 */ "idlist_opt ::= LP idlist RP", - /* 148 */ "idlist ::= idlist COMMA nm", - /* 149 */ "idlist ::= nm", - /* 150 */ "expr ::= LP expr RP", - /* 151 */ "term ::= NULL", - /* 152 */ "expr ::= ID|INDEXED", - /* 153 */ "expr ::= JOIN_KW", - /* 154 */ "expr ::= nm DOT nm", - /* 155 */ "expr ::= nm DOT nm DOT nm", - /* 156 */ "term ::= FLOAT|BLOB", - /* 157 */ "term ::= STRING", - /* 158 */ "term ::= INTEGER", - /* 159 */ "expr ::= VARIABLE", - /* 160 */ "expr ::= expr COLLATE ID|STRING", - /* 161 */ "expr ::= CAST LP expr AS typetoken RP", - /* 162 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 163 */ "expr ::= ID|INDEXED LP STAR RP", - /* 164 */ "term ::= CTIME_KW", - /* 165 */ "expr ::= LP nexprlist COMMA expr RP", - /* 166 */ "expr ::= expr AND expr", - /* 167 */ "expr ::= expr OR expr", - /* 168 */ "expr ::= expr LT|GT|GE|LE expr", - /* 169 */ "expr ::= expr EQ|NE expr", - /* 170 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 171 */ "expr ::= expr PLUS|MINUS expr", - /* 172 */ "expr ::= expr STAR|SLASH|REM expr", - /* 173 */ "expr ::= expr CONCAT expr", - /* 174 */ "likeop ::= LIKE_KW|MATCH", - /* 175 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 176 */ "expr ::= expr likeop expr", - /* 177 */ "expr ::= expr likeop expr ESCAPE expr", - /* 178 */ "expr ::= expr ISNULL|NOTNULL", - /* 179 */ "expr ::= expr NOT NULL", - /* 180 */ "expr ::= expr IS expr", - /* 181 */ "expr ::= expr IS NOT expr", - /* 182 */ "expr ::= NOT expr", - /* 183 */ "expr ::= BITNOT expr", - /* 184 */ "expr ::= MINUS expr", - /* 185 */ "expr ::= PLUS expr", - /* 186 */ "between_op ::= BETWEEN", - /* 187 */ "between_op ::= NOT BETWEEN", - /* 188 */ "expr ::= expr between_op expr AND expr", - /* 189 */ "in_op ::= IN", - /* 190 */ "in_op ::= NOT IN", - /* 191 */ "expr ::= expr in_op LP exprlist RP", - /* 192 */ "expr ::= LP select RP", - /* 193 */ "expr ::= expr in_op LP select RP", - /* 194 */ "expr ::= expr in_op nm dbnm paren_exprlist", - /* 195 */ "expr ::= EXISTS LP select RP", - /* 196 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 197 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 198 */ "case_exprlist ::= WHEN expr THEN expr", - /* 199 */ "case_else ::= ELSE expr", - /* 200 */ "case_else ::=", - /* 201 */ "case_operand ::= expr", - /* 202 */ "case_operand ::=", - /* 203 */ "exprlist ::=", - /* 204 */ "nexprlist ::= nexprlist COMMA expr", - /* 205 */ "nexprlist ::= expr", - /* 206 */ "paren_exprlist ::=", - /* 207 */ "paren_exprlist ::= LP exprlist RP", - /* 208 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 209 */ "uniqueflag ::= UNIQUE", - /* 210 */ "uniqueflag ::=", - /* 211 */ "eidlist_opt ::=", - /* 212 */ "eidlist_opt ::= LP eidlist RP", - /* 213 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 214 */ "eidlist ::= nm collate sortorder", - /* 215 */ "collate ::=", - /* 216 */ "collate ::= COLLATE ID|STRING", - /* 217 */ "cmd ::= DROP INDEX ifexists fullname", - /* 218 */ "cmd ::= VACUUM", - /* 219 */ "cmd ::= VACUUM nm", - /* 220 */ "cmd ::= PRAGMA nm dbnm", - /* 221 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 222 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 223 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 224 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 225 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 226 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 227 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 228 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 229 */ "trigger_time ::= BEFORE", - /* 230 */ "trigger_time ::= AFTER", - /* 231 */ "trigger_time ::= INSTEAD OF", - /* 232 */ "trigger_time ::=", - /* 233 */ "trigger_event ::= DELETE|INSERT", - /* 234 */ "trigger_event ::= UPDATE", - /* 235 */ "trigger_event ::= UPDATE OF idlist", - /* 236 */ "when_clause ::=", - /* 237 */ "when_clause ::= WHEN expr", - /* 238 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 239 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 240 */ "trnm ::= nm DOT nm", - /* 241 */ "tridxby ::= INDEXED BY nm", - /* 242 */ "tridxby ::= NOT INDEXED", - /* 243 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", - /* 244 */ "trigger_cmd ::= insert_cmd INTO trnm idlist_opt select", - /* 245 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", - /* 246 */ "trigger_cmd ::= select", - /* 247 */ "expr ::= RAISE LP IGNORE RP", - /* 248 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 249 */ "raisetype ::= ROLLBACK", - /* 250 */ "raisetype ::= ABORT", - /* 251 */ "raisetype ::= FAIL", - /* 252 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 253 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 254 */ "cmd ::= DETACH database_kw_opt expr", - /* 255 */ "key_opt ::=", - /* 256 */ "key_opt ::= KEY expr", - /* 257 */ "cmd ::= REINDEX", - /* 258 */ "cmd ::= REINDEX nm dbnm", - /* 259 */ "cmd ::= ANALYZE", - /* 260 */ "cmd ::= ANALYZE nm dbnm", - /* 261 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 262 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 263 */ "add_column_fullname ::= fullname", - /* 264 */ "cmd ::= create_vtab", - /* 265 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 266 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 267 */ "vtabarg ::=", - /* 268 */ "vtabargtoken ::= ANY", - /* 269 */ "vtabargtoken ::= lp anylist RP", - /* 270 */ "lp ::= LP", - /* 271 */ "with ::=", - /* 272 */ "with ::= WITH wqlist", - /* 273 */ "with ::= WITH RECURSIVE wqlist", - /* 274 */ "wqlist ::= nm eidlist_opt AS LP select RP", - /* 275 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", - /* 276 */ "input ::= cmdlist", - /* 277 */ "cmdlist ::= cmdlist ecmd", - /* 278 */ "cmdlist ::= ecmd", - /* 279 */ "ecmd ::= SEMI", - /* 280 */ "ecmd ::= explain cmdx SEMI", - /* 281 */ "explain ::=", - /* 282 */ "trans_opt ::=", - /* 283 */ "trans_opt ::= TRANSACTION", - /* 284 */ "trans_opt ::= TRANSACTION nm", - /* 285 */ "savepoint_opt ::= SAVEPOINT", - /* 286 */ "savepoint_opt ::=", - /* 287 */ "cmd ::= create_table create_table_args", - /* 288 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 289 */ "columnlist ::= columnname carglist", - /* 290 */ "nm ::= ID|INDEXED", - /* 291 */ "nm ::= STRING", - /* 292 */ "nm ::= JOIN_KW", - /* 293 */ "typetoken ::= typename", - /* 294 */ "typename ::= ID|STRING", - /* 295 */ "signed ::= plus_num", - /* 296 */ "signed ::= minus_num", - /* 297 */ "carglist ::= carglist ccons", - /* 298 */ "carglist ::=", - /* 299 */ "ccons ::= NULL onconf", - /* 300 */ "conslist_opt ::= COMMA conslist", - /* 301 */ "conslist ::= conslist tconscomma tcons", - /* 302 */ "conslist ::= tcons", - /* 303 */ "tconscomma ::=", - /* 304 */ "defer_subclause_opt ::= defer_subclause", - /* 305 */ "resolvetype ::= raisetype", - /* 306 */ "selectnowith ::= oneselect", - /* 307 */ "oneselect ::= values", - /* 308 */ "sclp ::= selcollist COMMA", - /* 309 */ "as ::= ID|STRING", - /* 310 */ "expr ::= term", - /* 311 */ "exprlist ::= nexprlist", - /* 312 */ "nmnum ::= plus_num", - /* 313 */ "nmnum ::= nm", - /* 314 */ "nmnum ::= ON", - /* 315 */ "nmnum ::= DELETE", - /* 316 */ "nmnum ::= DEFAULT", - /* 317 */ "plus_num ::= INTEGER|FLOAT", - /* 318 */ "foreach_clause ::=", - /* 319 */ "foreach_clause ::= FOR EACH ROW", - /* 320 */ "trnm ::= nm", - /* 321 */ "tridxby ::=", - /* 322 */ "database_kw_opt ::= DATABASE", - /* 323 */ "database_kw_opt ::=", - /* 324 */ "kwcolumn_opt ::=", - /* 325 */ "kwcolumn_opt ::= COLUMNKW", - /* 326 */ "vtabarglist ::= vtabarg", - /* 327 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 328 */ "vtabarg ::= vtabarg vtabargtoken", - /* 329 */ "anylist ::=", - /* 330 */ "anylist ::= anylist LP anylist RP", - /* 331 */ "anylist ::= anylist ANY", -}; -#endif /* NDEBUG */ - - -#if YYSTACKDEPTH<=0 -/* -** Try to increase the size of the parser stack. Return the number -** of errors. Return 0 on success. -*/ -static int yyGrowStack(yyParser *p){ - int newSize; - int idx; - yyStackEntry *pNew; - - newSize = p->yystksz*2 + 100; - idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; - if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->yystk0; - }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); - } - if( pNew ){ - p->yystack = pNew; - p->yytos = &p->yystack[idx]; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", - yyTracePrompt, p->yystksz, newSize); - } -#endif - p->yystksz = newSize; - } - return pNew==0; -} -#endif - -/* Datatype of the argument to the memory allocated passed as the -** second argument to sqlite3ParserAlloc() below. This can be changed by -** putting an appropriate #define in the %include section of the input -** grammar. -*/ -#ifndef YYMALLOCARGTYPE -# define YYMALLOCARGTYPE size_t -#endif - -/* -** This function allocates a new parser. -** The only argument is a pointer to a function which works like -** malloc. -** -** Inputs: -** A pointer to the function used to allocate memory. -** -** Outputs: -** A pointer to a parser. This pointer is used in subsequent calls -** to sqlite3Parser and sqlite3ParserFree. -*/ -SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){ - yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( pParser ){ -#ifdef YYTRACKMAXSTACKDEPTH - pParser->yyhwm = 0; -#endif -#if YYSTACKDEPTH<=0 - pParser->yytos = NULL; - pParser->yystack = NULL; - pParser->yystksz = 0; - if( yyGrowStack(pParser) ){ - pParser->yystack = &pParser->yystk0; - pParser->yystksz = 1; - } -#endif -#ifndef YYNOERRORRECOVERY - pParser->yyerrcnt = -1; -#endif - pParser->yytos = pParser->yystack; - pParser->yystack[0].stateno = 0; - pParser->yystack[0].major = 0; - } - return pParser; -} - -/* The following function deletes the "minor type" or semantic value -** associated with a symbol. The symbol can be either a terminal -** or nonterminal. "yymajor" is the symbol code, and "yypminor" is -** a pointer to the value to be deleted. The code used to do the -** deletions is derived from the %destructor and/or %token_destructor -** directives of the input grammar. -*/ -static void yy_destructor( - yyParser *yypParser, /* The parser */ - YYCODETYPE yymajor, /* Type code for object to destroy */ - YYMINORTYPE *yypminor /* The object to be destroyed */ -){ - sqlite3ParserARG_FETCH; - switch( yymajor ){ - /* Here is inserted the actions which take place when a - ** terminal or non-terminal is destroyed. This can happen - ** when the symbol is popped from the stack during a - ** reduce or during error processing or when a parser is - ** being destroyed before it is finished parsing. - ** - ** Note: during a reduce, the only symbols destroyed are those - ** which appear on the RHS of the rule, but which are *not* used - ** inside the C code. - */ -/********* Begin destructor definitions ***************************************/ - case 163: /* select */ - case 194: /* selectnowith */ - case 195: /* oneselect */ - case 206: /* values */ -{ -sqlite3SelectDelete(pParse->db, (yypminor->yy243)); -} - break; - case 172: /* term */ - case 173: /* expr */ -{ -sqlite3ExprDelete(pParse->db, (yypminor->yy190).pExpr); -} - break; - case 177: /* eidlist_opt */ - case 186: /* sortlist */ - case 187: /* eidlist */ - case 199: /* selcollist */ - case 202: /* groupby_opt */ - case 204: /* orderby_opt */ - case 207: /* nexprlist */ - case 208: /* exprlist */ - case 209: /* sclp */ - case 218: /* setlist */ - case 224: /* paren_exprlist */ - case 226: /* case_exprlist */ -{ -sqlite3ExprListDelete(pParse->db, (yypminor->yy148)); -} - break; - case 193: /* fullname */ - case 200: /* from */ - case 211: /* seltablist */ - case 212: /* stl_prefix */ -{ -sqlite3SrcListDelete(pParse->db, (yypminor->yy185)); -} - break; - case 196: /* with */ - case 250: /* wqlist */ -{ -sqlite3WithDelete(pParse->db, (yypminor->yy285)); -} - break; - case 201: /* where_opt */ - case 203: /* having_opt */ - case 215: /* on_opt */ - case 225: /* case_operand */ - case 227: /* case_else */ - case 236: /* when_clause */ - case 241: /* key_opt */ -{ -sqlite3ExprDelete(pParse->db, (yypminor->yy72)); -} - break; - case 216: /* using_opt */ - case 217: /* idlist */ - case 220: /* idlist_opt */ -{ -sqlite3IdListDelete(pParse->db, (yypminor->yy254)); -} - break; - case 232: /* trigger_cmd_list */ - case 237: /* trigger_cmd */ -{ -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy145)); -} - break; - case 234: /* trigger_event */ -{ -sqlite3IdListDelete(pParse->db, (yypminor->yy332).b); -} - break; -/********* End destructor definitions *****************************************/ - default: break; /* If no destructor action specified: do nothing */ - } -} - -/* -** Pop the parser's stack once. -** -** If there is a destructor routine associated with the token which -** is popped from the stack, then call it. -*/ -static void yy_pop_parser_stack(yyParser *pParser){ - yyStackEntry *yytos; - assert( pParser->yytos!=0 ); - assert( pParser->yytos > pParser->yystack ); - yytos = pParser->yytos--; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sPopping %s\n", - yyTracePrompt, - yyTokenName[yytos->major]); - } -#endif - yy_destructor(pParser, yytos->major, &yytos->minor); -} - -/* -** Deallocate and destroy a parser. Destructors are called for -** all stack elements before shutting the parser down. -** -** If the YYPARSEFREENEVERNULL macro exists (for example because it -** is defined in a %include section of the input grammar) then it is -** assumed that the input pointer is never NULL. -*/ -SQLITE_PRIVATE void sqlite3ParserFree( - void *p, /* The parser to be deleted */ - void (*freeProc)(void*) /* Function used to reclaim memory */ -){ - yyParser *pParser = (yyParser*)p; -#ifndef YYPARSEFREENEVERNULL - if( pParser==0 ) return; -#endif - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); -#endif - (*freeProc)((void*)pParser); -} - -/* -** Return the peak depth of the stack for a parser. -*/ -#ifdef YYTRACKMAXSTACKDEPTH -SQLITE_PRIVATE int sqlite3ParserStackPeak(void *p){ - yyParser *pParser = (yyParser*)p; - return pParser->yyhwm; -} -#endif - -/* -** Find the appropriate action for a parser given the terminal -** look-ahead token iLookAhead. -*/ -static unsigned int yy_find_shift_action( - yyParser *pParser, /* The parser */ - YYCODETYPE iLookAhead /* The look-ahead token */ -){ - int i; - int stateno = pParser->yytos->stateno; - - if( stateno>=YY_MIN_REDUCE ) return stateno; - assert( stateno <= YY_SHIFT_COUNT ); - do{ - i = yy_shift_ofst[stateno]; - assert( iLookAhead!=YYNOCODE ); - i += iLookAhead; - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ -#ifdef YYFALLBACK - YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); - } -#endif - assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ - iLookAhead = iFallback; - continue; - } -#endif -#ifdef YYWILDCARD - { - int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 - ){ -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", - yyTracePrompt, yyTokenName[iLookAhead], - yyTokenName[YYWILDCARD]); - } -#endif /* NDEBUG */ - return yy_action[j]; - } - } -#endif /* YYWILDCARD */ - return yy_default[stateno]; - }else{ - return yy_action[i]; - } - }while(1); -} - -/* -** Find the appropriate action for a parser given the non-terminal -** look-ahead token iLookAhead. -*/ -static int yy_find_reduce_action( - int stateno, /* Current state number */ - YYCODETYPE iLookAhead /* The look-ahead token */ -){ - int i; -#ifdef YYERRORSYMBOL - if( stateno>YY_REDUCE_COUNT ){ - return yy_default[stateno]; - } -#else - assert( stateno<=YY_REDUCE_COUNT ); -#endif - i = yy_reduce_ofst[stateno]; - assert( i!=YY_REDUCE_USE_DFLT ); - assert( iLookAhead!=YYNOCODE ); - i += iLookAhead; -#ifdef YYERRORSYMBOL - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ - return yy_default[stateno]; - } -#else - assert( i>=0 && iyytos--; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); - } -#endif - while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will execute if the parser - ** stack every overflows */ -/******** Begin %stack_overflow code ******************************************/ - - sqlite3ErrorMsg(pParse, "parser stack overflow"); -/******** End %stack_overflow code ********************************************/ - sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument var */ -} - -/* -** Print tracing information for a SHIFT action -*/ -#ifndef NDEBUG -static void yyTraceShift(yyParser *yypParser, int yyNewState){ - if( yyTraceFILE ){ - if( yyNewStateyytos->major], - yyNewState); - }else{ - fprintf(yyTraceFILE,"%sShift '%s'\n", - yyTracePrompt,yyTokenName[yypParser->yytos->major]); - } - } -} -#else -# define yyTraceShift(X,Y) -#endif - -/* -** Perform a shift action. -*/ -static void yy_shift( - yyParser *yypParser, /* The parser to be shifted */ - int yyNewState, /* The new state to shift in */ - int yyMajor, /* The major token to shift in */ - sqlite3ParserTOKENTYPE yyMinor /* The minor token to shift in */ -){ - yyStackEntry *yytos; - yypParser->yytos++; -#ifdef YYTRACKMAXSTACKDEPTH - if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH] ){ - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ - if( yyGrowStack(yypParser) ){ - yyStackOverflow(yypParser); - return; - } - } -#endif - if( yyNewState > YY_MAX_SHIFT ){ - yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; - } - yytos = yypParser->yytos; - yytos->stateno = (YYACTIONTYPE)yyNewState; - yytos->major = (YYCODETYPE)yyMajor; - yytos->minor.yy0 = yyMinor; - yyTraceShift(yypParser, yyNewState); -} - -/* The following table contains information about every rule that -** is used during the reduce. -*/ -static const struct { - YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ - unsigned char nrhs; /* Number of right-hand side symbols in the rule */ -} yyRuleInfo[] = { - { 147, 1 }, - { 147, 3 }, - { 148, 1 }, - { 149, 3 }, - { 150, 0 }, - { 150, 1 }, - { 150, 1 }, - { 150, 1 }, - { 149, 2 }, - { 149, 2 }, - { 149, 2 }, - { 149, 2 }, - { 149, 3 }, - { 149, 5 }, - { 154, 6 }, - { 156, 1 }, - { 158, 0 }, - { 158, 3 }, - { 157, 1 }, - { 157, 0 }, - { 155, 5 }, - { 155, 2 }, - { 162, 0 }, - { 162, 2 }, - { 164, 2 }, - { 166, 0 }, - { 166, 4 }, - { 166, 6 }, - { 167, 2 }, - { 171, 2 }, - { 171, 2 }, - { 171, 4 }, - { 171, 3 }, - { 171, 3 }, - { 171, 2 }, - { 171, 3 }, - { 171, 5 }, - { 171, 2 }, - { 171, 4 }, - { 171, 4 }, - { 171, 1 }, - { 171, 2 }, - { 176, 0 }, - { 176, 1 }, - { 178, 0 }, - { 178, 2 }, - { 180, 2 }, - { 180, 3 }, - { 180, 3 }, - { 180, 3 }, - { 181, 2 }, - { 181, 2 }, - { 181, 1 }, - { 181, 1 }, - { 181, 2 }, - { 179, 3 }, - { 179, 2 }, - { 182, 0 }, - { 182, 2 }, - { 182, 2 }, - { 161, 0 }, - { 184, 1 }, - { 185, 2 }, - { 185, 7 }, - { 185, 5 }, - { 185, 5 }, - { 185, 10 }, - { 188, 0 }, - { 174, 0 }, - { 174, 3 }, - { 189, 0 }, - { 189, 2 }, - { 190, 1 }, - { 190, 1 }, - { 149, 4 }, - { 192, 2 }, - { 192, 0 }, - { 149, 9 }, - { 149, 4 }, - { 149, 1 }, - { 163, 2 }, - { 194, 3 }, - { 197, 1 }, - { 197, 2 }, - { 197, 1 }, - { 195, 9 }, - { 206, 4 }, - { 206, 5 }, - { 198, 1 }, - { 198, 1 }, - { 198, 0 }, - { 209, 0 }, - { 199, 3 }, - { 199, 2 }, - { 199, 4 }, - { 210, 2 }, - { 210, 0 }, - { 200, 0 }, - { 200, 2 }, - { 212, 2 }, - { 212, 0 }, - { 211, 7 }, - { 211, 9 }, - { 211, 7 }, - { 211, 7 }, - { 159, 0 }, - { 159, 2 }, - { 193, 2 }, - { 213, 1 }, - { 213, 2 }, - { 213, 3 }, - { 213, 4 }, - { 215, 2 }, - { 215, 0 }, - { 214, 0 }, - { 214, 3 }, - { 214, 2 }, - { 216, 4 }, - { 216, 0 }, - { 204, 0 }, - { 204, 3 }, - { 186, 4 }, - { 186, 2 }, - { 175, 1 }, - { 175, 1 }, - { 175, 0 }, - { 202, 0 }, - { 202, 3 }, - { 203, 0 }, - { 203, 2 }, - { 205, 0 }, - { 205, 2 }, - { 205, 4 }, - { 205, 4 }, - { 149, 6 }, - { 201, 0 }, - { 201, 2 }, - { 149, 8 }, - { 218, 5 }, - { 218, 7 }, - { 218, 3 }, - { 218, 5 }, - { 149, 6 }, - { 149, 7 }, - { 219, 2 }, - { 219, 1 }, - { 220, 0 }, - { 220, 3 }, - { 217, 3 }, - { 217, 1 }, - { 173, 3 }, - { 172, 1 }, - { 173, 1 }, - { 173, 1 }, - { 173, 3 }, - { 173, 5 }, - { 172, 1 }, - { 172, 1 }, - { 172, 1 }, - { 173, 1 }, - { 173, 3 }, - { 173, 6 }, - { 173, 5 }, - { 173, 4 }, - { 172, 1 }, - { 173, 5 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 221, 1 }, - { 221, 2 }, - { 173, 3 }, - { 173, 5 }, - { 173, 2 }, - { 173, 3 }, - { 173, 3 }, - { 173, 4 }, - { 173, 2 }, - { 173, 2 }, - { 173, 2 }, - { 173, 2 }, - { 222, 1 }, - { 222, 2 }, - { 173, 5 }, - { 223, 1 }, - { 223, 2 }, - { 173, 5 }, - { 173, 3 }, - { 173, 5 }, - { 173, 5 }, - { 173, 4 }, - { 173, 5 }, - { 226, 5 }, - { 226, 4 }, - { 227, 2 }, - { 227, 0 }, - { 225, 1 }, - { 225, 0 }, - { 208, 0 }, - { 207, 3 }, - { 207, 1 }, - { 224, 0 }, - { 224, 3 }, - { 149, 12 }, - { 228, 1 }, - { 228, 0 }, - { 177, 0 }, - { 177, 3 }, - { 187, 5 }, - { 187, 3 }, - { 229, 0 }, - { 229, 2 }, - { 149, 4 }, - { 149, 1 }, - { 149, 2 }, - { 149, 3 }, - { 149, 5 }, - { 149, 6 }, - { 149, 5 }, - { 149, 6 }, - { 169, 2 }, - { 170, 2 }, - { 149, 5 }, - { 231, 11 }, - { 233, 1 }, - { 233, 1 }, - { 233, 2 }, - { 233, 0 }, - { 234, 1 }, - { 234, 1 }, - { 234, 3 }, - { 236, 0 }, - { 236, 2 }, - { 232, 3 }, - { 232, 2 }, - { 238, 3 }, - { 239, 3 }, - { 239, 2 }, - { 237, 7 }, - { 237, 5 }, - { 237, 5 }, - { 237, 1 }, - { 173, 4 }, - { 173, 6 }, - { 191, 1 }, - { 191, 1 }, - { 191, 1 }, - { 149, 4 }, - { 149, 6 }, - { 149, 3 }, - { 241, 0 }, - { 241, 2 }, - { 149, 1 }, - { 149, 3 }, - { 149, 1 }, - { 149, 3 }, - { 149, 6 }, - { 149, 7 }, - { 242, 1 }, - { 149, 1 }, - { 149, 4 }, - { 244, 8 }, - { 246, 0 }, - { 247, 1 }, - { 247, 3 }, - { 248, 1 }, - { 196, 0 }, - { 196, 2 }, - { 196, 3 }, - { 250, 6 }, - { 250, 8 }, - { 144, 1 }, - { 145, 2 }, - { 145, 1 }, - { 146, 1 }, - { 146, 3 }, - { 147, 0 }, - { 151, 0 }, - { 151, 1 }, - { 151, 2 }, - { 153, 1 }, - { 153, 0 }, - { 149, 2 }, - { 160, 4 }, - { 160, 2 }, - { 152, 1 }, - { 152, 1 }, - { 152, 1 }, - { 166, 1 }, - { 167, 1 }, - { 168, 1 }, - { 168, 1 }, - { 165, 2 }, - { 165, 0 }, - { 171, 2 }, - { 161, 2 }, - { 183, 3 }, - { 183, 1 }, - { 184, 0 }, - { 188, 1 }, - { 190, 1 }, - { 194, 1 }, - { 195, 1 }, - { 209, 2 }, - { 210, 1 }, - { 173, 1 }, - { 208, 1 }, - { 230, 1 }, - { 230, 1 }, - { 230, 1 }, - { 230, 1 }, - { 230, 1 }, - { 169, 1 }, - { 235, 0 }, - { 235, 3 }, - { 238, 1 }, - { 239, 0 }, - { 240, 1 }, - { 240, 0 }, - { 243, 0 }, - { 243, 1 }, - { 245, 1 }, - { 245, 3 }, - { 246, 2 }, - { 249, 0 }, - { 249, 4 }, - { 249, 2 }, -}; - -static void yy_accept(yyParser*); /* Forward Declaration */ - -/* -** Perform a reduce action and the shift that must immediately -** follow the reduce. -*/ -static void yy_reduce( - yyParser *yypParser, /* The parser */ - unsigned int yyruleno /* Number of the rule by which to reduce */ -){ - int yygoto; /* The next state */ - int yyact; /* The next action */ - yyStackEntry *yymsp; /* The top of the parser's stack */ - int yysize; /* Amount to pop the stack */ - sqlite3ParserARG_FETCH; - yymsp = yypParser->yytos; -#ifndef NDEBUG - if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; - fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt, - yyRuleName[yyruleno], yymsp[-yysize].stateno); - } -#endif /* NDEBUG */ - - /* Check that the stack is large enough to grow by a single entry - ** if the RHS of the rule is empty. This ensures that there is room - ** enough on the stack to push the LHS value */ - if( yyRuleInfo[yyruleno].nrhs==0 ){ -#ifdef YYTRACKMAXSTACKDEPTH - if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH-1] ){ - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ - if( yyGrowStack(yypParser) ){ - yyStackOverflow(yypParser); - return; - } - yymsp = yypParser->yytos; - } -#endif - } - - switch( yyruleno ){ - /* Beginning here are the reduction cases. A typical example - ** follows: - ** case 0: - ** #line - ** { ... } // User supplied code - ** #line - ** break; - */ -/********** Begin reduce actions **********************************************/ - YYMINORTYPE yylhsminor; - case 0: /* explain ::= EXPLAIN */ -{ pParse->explain = 1; } - break; - case 1: /* explain ::= EXPLAIN QUERY PLAN */ -{ pParse->explain = 2; } - break; - case 2: /* cmdx ::= cmd */ -{ sqlite3FinishCoding(pParse); } - break; - case 3: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy194);} - break; - case 4: /* transtype ::= */ -{yymsp[1].minor.yy194 = TK_DEFERRED;} - break; - case 5: /* transtype ::= DEFERRED */ - case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); - case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); -{yymsp[0].minor.yy194 = yymsp[0].major; /*A-overwrites-X*/} - break; - case 8: /* cmd ::= COMMIT trans_opt */ - case 9: /* cmd ::= END trans_opt */ yytestcase(yyruleno==9); -{sqlite3CommitTransaction(pParse);} - break; - case 10: /* cmd ::= ROLLBACK trans_opt */ -{sqlite3RollbackTransaction(pParse);} - break; - case 11: /* cmd ::= SAVEPOINT nm */ -{ - sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); -} - break; - case 12: /* cmd ::= RELEASE savepoint_opt nm */ -{ - sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); -} - break; - case 13: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ -{ - sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); -} - break; - case 14: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ -{ - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy194,0,0,yymsp[-2].minor.yy194); -} - break; - case 15: /* createkw ::= CREATE */ -{disableLookaside(pParse);} - break; - case 16: /* ifnotexists ::= */ - case 19: /* temp ::= */ yytestcase(yyruleno==19); - case 22: /* table_options ::= */ yytestcase(yyruleno==22); - case 42: /* autoinc ::= */ yytestcase(yyruleno==42); - case 57: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==57); - case 67: /* defer_subclause_opt ::= */ yytestcase(yyruleno==67); - case 76: /* ifexists ::= */ yytestcase(yyruleno==76); - case 90: /* distinct ::= */ yytestcase(yyruleno==90); - case 215: /* collate ::= */ yytestcase(yyruleno==215); -{yymsp[1].minor.yy194 = 0;} - break; - case 17: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy194 = 1;} - break; - case 18: /* temp ::= TEMP */ - case 43: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==43); -{yymsp[0].minor.yy194 = 1;} - break; - case 20: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ -{ - sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy194,0); -} - break; - case 21: /* create_table_args ::= AS select */ -{ - sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy243); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy243); -} - break; - case 23: /* table_options ::= WITHOUT nm */ -{ - if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yymsp[-1].minor.yy194 = TF_WithoutRowid | TF_NoVisibleRowid; - }else{ - yymsp[-1].minor.yy194 = 0; - sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); - } -} - break; - case 24: /* columnname ::= nm typetoken */ -{sqlite3AddColumn(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} - break; - case 25: /* typetoken ::= */ - case 60: /* conslist_opt ::= */ yytestcase(yyruleno==60); - case 96: /* as ::= */ yytestcase(yyruleno==96); -{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} - break; - case 26: /* typetoken ::= typename LP signed RP */ -{ - yymsp[-3].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); -} - break; - case 27: /* typetoken ::= typename LP signed COMMA signed RP */ -{ - yymsp[-5].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); -} - break; - case 28: /* typename ::= typename ID|STRING */ -{yymsp[-1].minor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} - break; - case 29: /* ccons ::= CONSTRAINT nm */ - case 62: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==62); -{pParse->constraintName = yymsp[0].minor.yy0;} - break; - case 30: /* ccons ::= DEFAULT term */ - case 32: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==32); -{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy190);} - break; - case 31: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy190);} - break; - case 33: /* ccons ::= DEFAULT MINUS term */ -{ - ExprSpan v; - v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy190.pExpr, 0, 0); - v.zStart = yymsp[-1].minor.yy0.z; - v.zEnd = yymsp[0].minor.yy190.zEnd; - sqlite3AddDefaultValue(pParse,&v); -} - break; - case 34: /* ccons ::= DEFAULT ID|INDEXED */ -{ - ExprSpan v; - spanExpr(&v, pParse, TK_STRING, yymsp[0].minor.yy0); - sqlite3AddDefaultValue(pParse,&v); -} - break; - case 35: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy194);} - break; - case 36: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy194,yymsp[0].minor.yy194,yymsp[-2].minor.yy194);} - break; - case 37: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy194,0,0,0,0, - SQLITE_IDXTYPE_UNIQUE);} - break; - case 38: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy190.pExpr);} - break; - case 39: /* ccons ::= REFERENCES nm eidlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy148,yymsp[0].minor.yy194);} - break; - case 40: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy194);} - break; - case 41: /* ccons ::= COLLATE ID|STRING */ -{sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} - break; - case 44: /* refargs ::= */ -{ yymsp[1].minor.yy194 = OE_None*0x0101; /* EV: R-19803-45884 */} - break; - case 45: /* refargs ::= refargs refarg */ -{ yymsp[-1].minor.yy194 = (yymsp[-1].minor.yy194 & ~yymsp[0].minor.yy497.mask) | yymsp[0].minor.yy497.value; } - break; - case 46: /* refarg ::= MATCH nm */ -{ yymsp[-1].minor.yy497.value = 0; yymsp[-1].minor.yy497.mask = 0x000000; } - break; - case 47: /* refarg ::= ON INSERT refact */ -{ yymsp[-2].minor.yy497.value = 0; yymsp[-2].minor.yy497.mask = 0x000000; } - break; - case 48: /* refarg ::= ON DELETE refact */ -{ yymsp[-2].minor.yy497.value = yymsp[0].minor.yy194; yymsp[-2].minor.yy497.mask = 0x0000ff; } - break; - case 49: /* refarg ::= ON UPDATE refact */ -{ yymsp[-2].minor.yy497.value = yymsp[0].minor.yy194<<8; yymsp[-2].minor.yy497.mask = 0x00ff00; } - break; - case 50: /* refact ::= SET NULL */ -{ yymsp[-1].minor.yy194 = OE_SetNull; /* EV: R-33326-45252 */} - break; - case 51: /* refact ::= SET DEFAULT */ -{ yymsp[-1].minor.yy194 = OE_SetDflt; /* EV: R-33326-45252 */} - break; - case 52: /* refact ::= CASCADE */ -{ yymsp[0].minor.yy194 = OE_Cascade; /* EV: R-33326-45252 */} - break; - case 53: /* refact ::= RESTRICT */ -{ yymsp[0].minor.yy194 = OE_Restrict; /* EV: R-33326-45252 */} - break; - case 54: /* refact ::= NO ACTION */ -{ yymsp[-1].minor.yy194 = OE_None; /* EV: R-33326-45252 */} - break; - case 55: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ -{yymsp[-2].minor.yy194 = 0;} - break; - case 56: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - case 71: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==71); - case 144: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==144); -{yymsp[-1].minor.yy194 = yymsp[0].minor.yy194;} - break; - case 58: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ - case 75: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==75); - case 187: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==187); - case 190: /* in_op ::= NOT IN */ yytestcase(yyruleno==190); - case 216: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==216); -{yymsp[-1].minor.yy194 = 1;} - break; - case 59: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ -{yymsp[-1].minor.yy194 = 0;} - break; - case 61: /* tconscomma ::= COMMA */ -{pParse->constraintName.n = 0;} - break; - case 63: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy148,yymsp[0].minor.yy194,yymsp[-2].minor.yy194,0);} - break; - case 64: /* tcons ::= UNIQUE LP sortlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy148,yymsp[0].minor.yy194,0,0,0,0, - SQLITE_IDXTYPE_UNIQUE);} - break; - case 65: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy190.pExpr);} - break; - case 66: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ -{ - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy148, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy148, yymsp[-1].minor.yy194); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy194); -} - break; - case 68: /* onconf ::= */ - case 70: /* orconf ::= */ yytestcase(yyruleno==70); -{yymsp[1].minor.yy194 = OE_Default;} - break; - case 69: /* onconf ::= ON CONFLICT resolvetype */ -{yymsp[-2].minor.yy194 = yymsp[0].minor.yy194;} - break; - case 72: /* resolvetype ::= IGNORE */ -{yymsp[0].minor.yy194 = OE_Ignore;} - break; - case 73: /* resolvetype ::= REPLACE */ - case 145: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==145); -{yymsp[0].minor.yy194 = OE_Replace;} - break; - case 74: /* cmd ::= DROP TABLE ifexists fullname */ -{ - sqlite3DropTable(pParse, yymsp[0].minor.yy185, 0, yymsp[-1].minor.yy194); -} - break; - case 77: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ -{ - sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy148, yymsp[0].minor.yy243, yymsp[-7].minor.yy194, yymsp[-5].minor.yy194); -} - break; - case 78: /* cmd ::= DROP VIEW ifexists fullname */ -{ - sqlite3DropTable(pParse, yymsp[0].minor.yy185, 1, yymsp[-1].minor.yy194); -} - break; - case 79: /* cmd ::= select */ -{ - SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy243, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy243); -} - break; - case 80: /* select ::= with selectnowith */ -{ - Select *p = yymsp[0].minor.yy243; - if( p ){ - p->pWith = yymsp[-1].minor.yy285; - parserDoubleLinkSelect(pParse, p); - }else{ - sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy285); - } - yymsp[-1].minor.yy243 = p; /*A-overwrites-W*/ -} - break; - case 81: /* selectnowith ::= selectnowith multiselect_op oneselect */ -{ - Select *pRhs = yymsp[0].minor.yy243; - Select *pLhs = yymsp[-2].minor.yy243; - if( pRhs && pRhs->pPrior ){ - SrcList *pFrom; - Token x; - x.n = 0; - parserDoubleLinkSelect(pParse, pRhs); - pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); - pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0); - } - if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy194; - pRhs->pPrior = pLhs; - if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; - pRhs->selFlags &= ~SF_MultiValue; - if( yymsp[-1].minor.yy194!=TK_ALL ) pParse->hasCompound = 1; - }else{ - sqlite3SelectDelete(pParse->db, pLhs); - } - yymsp[-2].minor.yy243 = pRhs; -} - break; - case 82: /* multiselect_op ::= UNION */ - case 84: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==84); -{yymsp[0].minor.yy194 = yymsp[0].major; /*A-overwrites-OP*/} - break; - case 83: /* multiselect_op ::= UNION ALL */ -{yymsp[-1].minor.yy194 = TK_ALL;} - break; - case 85: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ -{ -#if SELECTTRACE_ENABLED - Token s = yymsp[-8].minor.yy0; /*A-overwrites-S*/ -#endif - yymsp[-8].minor.yy243 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy148,yymsp[-5].minor.yy185,yymsp[-4].minor.yy72,yymsp[-3].minor.yy148,yymsp[-2].minor.yy72,yymsp[-1].minor.yy148,yymsp[-7].minor.yy194,yymsp[0].minor.yy354.pLimit,yymsp[0].minor.yy354.pOffset); -#if SELECTTRACE_ENABLED - /* Populate the Select.zSelName[] string that is used to help with - ** query planner debugging, to differentiate between multiple Select - ** objects in a complex query. - ** - ** If the SELECT keyword is immediately followed by a C-style comment - ** then extract the first few alphanumeric characters from within that - ** comment to be the zSelName value. Otherwise, the label is #N where - ** is an integer that is incremented with each SELECT statement seen. - */ - if( yymsp[-8].minor.yy243!=0 ){ - const char *z = s.z+6; - int i; - sqlite3_snprintf(sizeof(yymsp[-8].minor.yy243->zSelName), yymsp[-8].minor.yy243->zSelName, "#%d", - ++pParse->nSelect); - while( z[0]==' ' ) z++; - if( z[0]=='/' && z[1]=='*' ){ - z += 2; - while( z[0]==' ' ) z++; - for(i=0; sqlite3Isalnum(z[i]); i++){} - sqlite3_snprintf(sizeof(yymsp[-8].minor.yy243->zSelName), yymsp[-8].minor.yy243->zSelName, "%.*s", i, z); - } - } -#endif /* SELECTRACE_ENABLED */ -} - break; - case 86: /* values ::= VALUES LP nexprlist RP */ -{ - yymsp[-3].minor.yy243 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy148,0,0,0,0,0,SF_Values,0,0); -} - break; - case 87: /* values ::= values COMMA LP exprlist RP */ -{ - Select *pRight, *pLeft = yymsp[-4].minor.yy243; - pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy148,0,0,0,0,0,SF_Values|SF_MultiValue,0,0); - if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; - if( pRight ){ - pRight->op = TK_ALL; - pRight->pPrior = pLeft; - yymsp[-4].minor.yy243 = pRight; - }else{ - yymsp[-4].minor.yy243 = pLeft; - } -} - break; - case 88: /* distinct ::= DISTINCT */ -{yymsp[0].minor.yy194 = SF_Distinct;} - break; - case 89: /* distinct ::= ALL */ -{yymsp[0].minor.yy194 = SF_All;} - break; - case 91: /* sclp ::= */ - case 119: /* orderby_opt ::= */ yytestcase(yyruleno==119); - case 126: /* groupby_opt ::= */ yytestcase(yyruleno==126); - case 203: /* exprlist ::= */ yytestcase(yyruleno==203); - case 206: /* paren_exprlist ::= */ yytestcase(yyruleno==206); - case 211: /* eidlist_opt ::= */ yytestcase(yyruleno==211); -{yymsp[1].minor.yy148 = 0;} - break; - case 92: /* selcollist ::= sclp expr as */ -{ - yymsp[-2].minor.yy148 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy148, yymsp[-1].minor.yy190.pExpr); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-2].minor.yy148, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yymsp[-2].minor.yy148,&yymsp[-1].minor.yy190); -} - break; - case 93: /* selcollist ::= sclp STAR */ -{ - Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); - yymsp[-1].minor.yy148 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy148, p); -} - break; - case 94: /* selcollist ::= sclp nm DOT STAR */ -{ - Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0, 0); - Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); - Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); - yymsp[-3].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy148, pDot); -} - break; - case 95: /* as ::= AS nm */ - case 106: /* dbnm ::= DOT nm */ yytestcase(yyruleno==106); - case 225: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==225); - case 226: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==226); -{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} - break; - case 97: /* from ::= */ -{yymsp[1].minor.yy185 = sqlite3DbMallocZero(pParse->db, sizeof(*yymsp[1].minor.yy185));} - break; - case 98: /* from ::= FROM seltablist */ -{ - yymsp[-1].minor.yy185 = yymsp[0].minor.yy185; - sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy185); -} - break; - case 99: /* stl_prefix ::= seltablist joinop */ -{ - if( ALWAYS(yymsp[-1].minor.yy185 && yymsp[-1].minor.yy185->nSrc>0) ) yymsp[-1].minor.yy185->a[yymsp[-1].minor.yy185->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy194; -} - break; - case 100: /* stl_prefix ::= */ -{yymsp[1].minor.yy185 = 0;} - break; - case 101: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ -{ - yymsp[-6].minor.yy185 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy185,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy72,yymsp[0].minor.yy254); - sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy185, &yymsp[-2].minor.yy0); -} - break; - case 102: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ -{ - yymsp[-8].minor.yy185 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy185,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy72,yymsp[0].minor.yy254); - sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy185, yymsp[-4].minor.yy148); -} - break; - case 103: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ -{ - yymsp[-6].minor.yy185 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy185,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy243,yymsp[-1].minor.yy72,yymsp[0].minor.yy254); - } - break; - case 104: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ -{ - if( yymsp[-6].minor.yy185==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy72==0 && yymsp[0].minor.yy254==0 ){ - yymsp[-6].minor.yy185 = yymsp[-4].minor.yy185; - }else if( yymsp[-4].minor.yy185->nSrc==1 ){ - yymsp[-6].minor.yy185 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy185,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy72,yymsp[0].minor.yy254); - if( yymsp[-6].minor.yy185 ){ - struct SrcList_item *pNew = &yymsp[-6].minor.yy185->a[yymsp[-6].minor.yy185->nSrc-1]; - struct SrcList_item *pOld = yymsp[-4].minor.yy185->a; - pNew->zName = pOld->zName; - pNew->zDatabase = pOld->zDatabase; - pNew->pSelect = pOld->pSelect; - pOld->zName = pOld->zDatabase = 0; - pOld->pSelect = 0; - } - sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy185); - }else{ - Select *pSubquery; - sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy185); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy185,0,0,0,0,SF_NestedFrom,0,0); - yymsp[-6].minor.yy185 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy185,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy72,yymsp[0].minor.yy254); - } - } - break; - case 105: /* dbnm ::= */ - case 114: /* indexed_opt ::= */ yytestcase(yyruleno==114); -{yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} - break; - case 107: /* fullname ::= nm dbnm */ -{yymsp[-1].minor.yy185 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} - break; - case 108: /* joinop ::= COMMA|JOIN */ -{ yymsp[0].minor.yy194 = JT_INNER; } - break; - case 109: /* joinop ::= JOIN_KW JOIN */ -{yymsp[-1].minor.yy194 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} - break; - case 110: /* joinop ::= JOIN_KW nm JOIN */ -{yymsp[-2].minor.yy194 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} - break; - case 111: /* joinop ::= JOIN_KW nm nm JOIN */ -{yymsp[-3].minor.yy194 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} - break; - case 112: /* on_opt ::= ON expr */ - case 129: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==129); - case 136: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==136); - case 199: /* case_else ::= ELSE expr */ yytestcase(yyruleno==199); -{yymsp[-1].minor.yy72 = yymsp[0].minor.yy190.pExpr;} - break; - case 113: /* on_opt ::= */ - case 128: /* having_opt ::= */ yytestcase(yyruleno==128); - case 135: /* where_opt ::= */ yytestcase(yyruleno==135); - case 200: /* case_else ::= */ yytestcase(yyruleno==200); - case 202: /* case_operand ::= */ yytestcase(yyruleno==202); -{yymsp[1].minor.yy72 = 0;} - break; - case 115: /* indexed_opt ::= INDEXED BY nm */ -{yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} - break; - case 116: /* indexed_opt ::= NOT INDEXED */ -{yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} - break; - case 117: /* using_opt ::= USING LP idlist RP */ -{yymsp[-3].minor.yy254 = yymsp[-1].minor.yy254;} - break; - case 118: /* using_opt ::= */ - case 146: /* idlist_opt ::= */ yytestcase(yyruleno==146); -{yymsp[1].minor.yy254 = 0;} - break; - case 120: /* orderby_opt ::= ORDER BY sortlist */ - case 127: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==127); -{yymsp[-2].minor.yy148 = yymsp[0].minor.yy148;} - break; - case 121: /* sortlist ::= sortlist COMMA expr sortorder */ -{ - yymsp[-3].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy148,yymsp[-1].minor.yy190.pExpr); - sqlite3ExprListSetSortOrder(yymsp[-3].minor.yy148,yymsp[0].minor.yy194); -} - break; - case 122: /* sortlist ::= expr sortorder */ -{ - yymsp[-1].minor.yy148 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy190.pExpr); /*A-overwrites-Y*/ - sqlite3ExprListSetSortOrder(yymsp[-1].minor.yy148,yymsp[0].minor.yy194); -} - break; - case 123: /* sortorder ::= ASC */ -{yymsp[0].minor.yy194 = SQLITE_SO_ASC;} - break; - case 124: /* sortorder ::= DESC */ -{yymsp[0].minor.yy194 = SQLITE_SO_DESC;} - break; - case 125: /* sortorder ::= */ -{yymsp[1].minor.yy194 = SQLITE_SO_UNDEFINED;} - break; - case 130: /* limit_opt ::= */ -{yymsp[1].minor.yy354.pLimit = 0; yymsp[1].minor.yy354.pOffset = 0;} - break; - case 131: /* limit_opt ::= LIMIT expr */ -{yymsp[-1].minor.yy354.pLimit = yymsp[0].minor.yy190.pExpr; yymsp[-1].minor.yy354.pOffset = 0;} - break; - case 132: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yymsp[-3].minor.yy354.pLimit = yymsp[-2].minor.yy190.pExpr; yymsp[-3].minor.yy354.pOffset = yymsp[0].minor.yy190.pExpr;} - break; - case 133: /* limit_opt ::= LIMIT expr COMMA expr */ -{yymsp[-3].minor.yy354.pOffset = yymsp[-2].minor.yy190.pExpr; yymsp[-3].minor.yy354.pLimit = yymsp[0].minor.yy190.pExpr;} - break; - case 134: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ -{ - sqlite3WithPush(pParse, yymsp[-5].minor.yy285, 1); - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy185, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy185,yymsp[0].minor.yy72); -} - break; - case 137: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ -{ - sqlite3WithPush(pParse, yymsp[-7].minor.yy285, 1); - sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy185, &yymsp[-3].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy148,"set list"); - sqlite3Update(pParse,yymsp[-4].minor.yy185,yymsp[-1].minor.yy148,yymsp[0].minor.yy72,yymsp[-5].minor.yy194); -} - break; - case 138: /* setlist ::= setlist COMMA nm EQ expr */ -{ - yymsp[-4].minor.yy148 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy148, yymsp[0].minor.yy190.pExpr); - sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy148, &yymsp[-2].minor.yy0, 1); -} - break; - case 139: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ -{ - yymsp[-6].minor.yy148 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy148, yymsp[-3].minor.yy254, yymsp[0].minor.yy190.pExpr); -} - break; - case 140: /* setlist ::= nm EQ expr */ -{ - yylhsminor.yy148 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy190.pExpr); - sqlite3ExprListSetName(pParse, yylhsminor.yy148, &yymsp[-2].minor.yy0, 1); -} - yymsp[-2].minor.yy148 = yylhsminor.yy148; - break; - case 141: /* setlist ::= LP idlist RP EQ expr */ -{ - yymsp[-4].minor.yy148 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy254, yymsp[0].minor.yy190.pExpr); -} - break; - case 142: /* cmd ::= with insert_cmd INTO fullname idlist_opt select */ -{ - sqlite3WithPush(pParse, yymsp[-5].minor.yy285, 1); - sqlite3Insert(pParse, yymsp[-2].minor.yy185, yymsp[0].minor.yy243, yymsp[-1].minor.yy254, yymsp[-4].minor.yy194); -} - break; - case 143: /* cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES */ -{ - sqlite3WithPush(pParse, yymsp[-6].minor.yy285, 1); - sqlite3Insert(pParse, yymsp[-3].minor.yy185, 0, yymsp[-2].minor.yy254, yymsp[-5].minor.yy194); -} - break; - case 147: /* idlist_opt ::= LP idlist RP */ -{yymsp[-2].minor.yy254 = yymsp[-1].minor.yy254;} - break; - case 148: /* idlist ::= idlist COMMA nm */ -{yymsp[-2].minor.yy254 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy254,&yymsp[0].minor.yy0);} - break; - case 149: /* idlist ::= nm */ -{yymsp[0].minor.yy254 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} - break; - case 150: /* expr ::= LP expr RP */ -{spanSet(&yymsp[-2].minor.yy190,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ yymsp[-2].minor.yy190.pExpr = yymsp[-1].minor.yy190.pExpr;} - break; - case 151: /* term ::= NULL */ - case 156: /* term ::= FLOAT|BLOB */ yytestcase(yyruleno==156); - case 157: /* term ::= STRING */ yytestcase(yyruleno==157); -{spanExpr(&yymsp[0].minor.yy190,pParse,yymsp[0].major,yymsp[0].minor.yy0);/*A-overwrites-X*/} - break; - case 152: /* expr ::= ID|INDEXED */ - case 153: /* expr ::= JOIN_KW */ yytestcase(yyruleno==153); -{spanExpr(&yymsp[0].minor.yy190,pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} - break; - case 154: /* expr ::= nm DOT nm */ -{ - Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); - Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); - spanSet(&yymsp[-2].minor.yy190,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ - yymsp[-2].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); -} - break; - case 155: /* expr ::= nm DOT nm DOT nm */ -{ - Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-4].minor.yy0, 1); - Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); - Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); - Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); - spanSet(&yymsp[-4].minor.yy190,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ - yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); -} - break; - case 158: /* term ::= INTEGER */ -{ - yylhsminor.yy190.pExpr = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); - yylhsminor.yy190.zStart = yymsp[0].minor.yy0.z; - yylhsminor.yy190.zEnd = yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n; - if( yylhsminor.yy190.pExpr ) yylhsminor.yy190.pExpr->flags |= EP_Leaf; -} - yymsp[0].minor.yy190 = yylhsminor.yy190; - break; - case 159: /* expr ::= VARIABLE */ -{ - if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ - u32 n = yymsp[0].minor.yy0.n; - spanExpr(&yymsp[0].minor.yy190, pParse, TK_VARIABLE, yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy190.pExpr, n); - }else{ - /* When doing a nested parse, one can include terms in an expression - ** that look like this: #1 #2 ... These terms refer to registers - ** in the virtual machine. #N is the N-th register. */ - Token t = yymsp[0].minor.yy0; /*A-overwrites-X*/ - assert( t.n>=2 ); - spanSet(&yymsp[0].minor.yy190, &t, &t); - if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); - yymsp[0].minor.yy190.pExpr = 0; - }else{ - yymsp[0].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, 0); - if( yymsp[0].minor.yy190.pExpr ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy190.pExpr->iTable); - } - } -} - break; - case 160: /* expr ::= expr COLLATE ID|STRING */ -{ - yymsp[-2].minor.yy190.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy190.pExpr, &yymsp[0].minor.yy0, 1); - yymsp[-2].minor.yy190.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; -} - break; - case 161: /* expr ::= CAST LP expr AS typetoken RP */ -{ - spanSet(&yymsp[-5].minor.yy190,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ - yymsp[-5].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy190.pExpr, 0, &yymsp[-1].minor.yy0); -} - break; - case 162: /* expr ::= ID|INDEXED LP distinct exprlist RP */ -{ - if( yymsp[-1].minor.yy148 && yymsp[-1].minor.yy148->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ - sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); - } - yylhsminor.yy190.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy148, &yymsp[-4].minor.yy0); - spanSet(&yylhsminor.yy190,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); - if( yymsp[-2].minor.yy194==SF_Distinct && yylhsminor.yy190.pExpr ){ - yylhsminor.yy190.pExpr->flags |= EP_Distinct; - } -} - yymsp[-4].minor.yy190 = yylhsminor.yy190; - break; - case 163: /* expr ::= ID|INDEXED LP STAR RP */ -{ - yylhsminor.yy190.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); - spanSet(&yylhsminor.yy190,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); -} - yymsp[-3].minor.yy190 = yylhsminor.yy190; - break; - case 164: /* term ::= CTIME_KW */ -{ - yylhsminor.yy190.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); - spanSet(&yylhsminor.yy190, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); -} - yymsp[0].minor.yy190 = yylhsminor.yy190; - break; - case 165: /* expr ::= LP nexprlist COMMA expr RP */ -{ - ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy148, yymsp[-1].minor.yy190.pExpr); - yylhsminor.yy190.pExpr = sqlite3PExpr(pParse, TK_VECTOR, 0, 0, 0); - if( yylhsminor.yy190.pExpr ){ - yylhsminor.yy190.pExpr->x.pList = pList; - spanSet(&yylhsminor.yy190, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0); - }else{ - sqlite3ExprListDelete(pParse->db, pList); - } -} - yymsp[-4].minor.yy190 = yylhsminor.yy190; - break; - case 166: /* expr ::= expr AND expr */ - case 167: /* expr ::= expr OR expr */ yytestcase(yyruleno==167); - case 168: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==168); - case 169: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==169); - case 170: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==170); - case 171: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==171); - case 172: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==172); - case 173: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==173); -{spanBinaryExpr(pParse,yymsp[-1].major,&yymsp[-2].minor.yy190,&yymsp[0].minor.yy190);} - break; - case 174: /* likeop ::= LIKE_KW|MATCH */ -{yymsp[0].minor.yy0=yymsp[0].minor.yy0;/*A-overwrites-X*/} - break; - case 175: /* likeop ::= NOT LIKE_KW|MATCH */ -{yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} - break; - case 176: /* expr ::= expr likeop expr */ -{ - ExprList *pList; - int bNot = yymsp[-1].minor.yy0.n & 0x80000000; - yymsp[-1].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy190.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy190.pExpr); - yymsp[-2].minor.yy190.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0); - exprNot(pParse, bNot, &yymsp[-2].minor.yy190); - yymsp[-2].minor.yy190.zEnd = yymsp[0].minor.yy190.zEnd; - if( yymsp[-2].minor.yy190.pExpr ) yymsp[-2].minor.yy190.pExpr->flags |= EP_InfixFunc; -} - break; - case 177: /* expr ::= expr likeop expr ESCAPE expr */ -{ - ExprList *pList; - int bNot = yymsp[-3].minor.yy0.n & 0x80000000; - yymsp[-3].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy190.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy190.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy190.pExpr); - yymsp[-4].minor.yy190.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0); - exprNot(pParse, bNot, &yymsp[-4].minor.yy190); - yymsp[-4].minor.yy190.zEnd = yymsp[0].minor.yy190.zEnd; - if( yymsp[-4].minor.yy190.pExpr ) yymsp[-4].minor.yy190.pExpr->flags |= EP_InfixFunc; -} - break; - case 178: /* expr ::= expr ISNULL|NOTNULL */ -{spanUnaryPostfix(pParse,yymsp[0].major,&yymsp[-1].minor.yy190,&yymsp[0].minor.yy0);} - break; - case 179: /* expr ::= expr NOT NULL */ -{spanUnaryPostfix(pParse,TK_NOTNULL,&yymsp[-2].minor.yy190,&yymsp[0].minor.yy0);} - break; - case 180: /* expr ::= expr IS expr */ -{ - spanBinaryExpr(pParse,TK_IS,&yymsp[-2].minor.yy190,&yymsp[0].minor.yy190); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy190.pExpr, yymsp[-2].minor.yy190.pExpr, TK_ISNULL); -} - break; - case 181: /* expr ::= expr IS NOT expr */ -{ - spanBinaryExpr(pParse,TK_ISNOT,&yymsp[-3].minor.yy190,&yymsp[0].minor.yy190); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy190.pExpr, yymsp[-3].minor.yy190.pExpr, TK_NOTNULL); -} - break; - case 182: /* expr ::= NOT expr */ - case 183: /* expr ::= BITNOT expr */ yytestcase(yyruleno==183); -{spanUnaryPrefix(&yymsp[-1].minor.yy190,pParse,yymsp[-1].major,&yymsp[0].minor.yy190,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} - break; - case 184: /* expr ::= MINUS expr */ -{spanUnaryPrefix(&yymsp[-1].minor.yy190,pParse,TK_UMINUS,&yymsp[0].minor.yy190,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} - break; - case 185: /* expr ::= PLUS expr */ -{spanUnaryPrefix(&yymsp[-1].minor.yy190,pParse,TK_UPLUS,&yymsp[0].minor.yy190,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} - break; - case 186: /* between_op ::= BETWEEN */ - case 189: /* in_op ::= IN */ yytestcase(yyruleno==189); -{yymsp[0].minor.yy194 = 0;} - break; - case 188: /* expr ::= expr between_op expr AND expr */ -{ - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy190.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy190.pExpr); - yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy190.pExpr, 0, 0); - if( yymsp[-4].minor.yy190.pExpr ){ - yymsp[-4].minor.yy190.pExpr->x.pList = pList; - }else{ - sqlite3ExprListDelete(pParse->db, pList); - } - exprNot(pParse, yymsp[-3].minor.yy194, &yymsp[-4].minor.yy190); - yymsp[-4].minor.yy190.zEnd = yymsp[0].minor.yy190.zEnd; -} - break; - case 191: /* expr ::= expr in_op LP exprlist RP */ -{ - if( yymsp[-1].minor.yy148==0 ){ - /* Expressions of the form - ** - ** expr1 IN () - ** expr1 NOT IN () - ** - ** simplify to constants 0 (false) and 1 (true), respectively, - ** regardless of the value of expr1. - */ - sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy190.pExpr); - yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy194]); - }else if( yymsp[-1].minor.yy148->nExpr==1 ){ - /* Expressions of the form: - ** - ** expr1 IN (?1) - ** expr1 NOT IN (?2) - ** - ** with exactly one value on the RHS can be simplified to something - ** like this: - ** - ** expr1 == ?1 - ** expr1 <> ?2 - ** - ** But, the RHS of the == or <> is marked with the EP_Generic flag - ** so that it may not contribute to the computation of comparison - ** affinity or the collating sequence to use for comparison. Otherwise, - ** the semantics would be subtly different from IN or NOT IN. - */ - Expr *pRHS = yymsp[-1].minor.yy148->a[0].pExpr; - yymsp[-1].minor.yy148->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy148); - /* pRHS cannot be NULL because a malloc error would have been detected - ** before now and control would have never reached this point */ - if( ALWAYS(pRHS) ){ - pRHS->flags &= ~EP_Collate; - pRHS->flags |= EP_Generic; - } - yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, yymsp[-3].minor.yy194 ? TK_NE : TK_EQ, yymsp[-4].minor.yy190.pExpr, pRHS, 0); - }else{ - yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy190.pExpr, 0, 0); - if( yymsp[-4].minor.yy190.pExpr ){ - yymsp[-4].minor.yy190.pExpr->x.pList = yymsp[-1].minor.yy148; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy190.pExpr); - }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy148); - } - exprNot(pParse, yymsp[-3].minor.yy194, &yymsp[-4].minor.yy190); - } - yymsp[-4].minor.yy190.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; - } - break; - case 192: /* expr ::= LP select RP */ -{ - spanSet(&yymsp[-2].minor.yy190,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ - yymsp[-2].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy190.pExpr, yymsp[-1].minor.yy243); - } - break; - case 193: /* expr ::= expr in_op LP select RP */ -{ - yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy190.pExpr, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy190.pExpr, yymsp[-1].minor.yy243); - exprNot(pParse, yymsp[-3].minor.yy194, &yymsp[-4].minor.yy190); - yymsp[-4].minor.yy190.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; - } - break; - case 194: /* expr ::= expr in_op nm dbnm paren_exprlist */ -{ - SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); - Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - if( yymsp[0].minor.yy148 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy148); - yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy190.pExpr, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy190.pExpr, pSelect); - exprNot(pParse, yymsp[-3].minor.yy194, &yymsp[-4].minor.yy190); - yymsp[-4].minor.yy190.zEnd = yymsp[-1].minor.yy0.z ? &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n] : &yymsp[-2].minor.yy0.z[yymsp[-2].minor.yy0.n]; - } - break; - case 195: /* expr ::= EXISTS LP select RP */ -{ - Expr *p; - spanSet(&yymsp[-3].minor.yy190,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ - p = yymsp[-3].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); - sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy243); - } - break; - case 196: /* expr ::= CASE case_operand case_exprlist case_else END */ -{ - spanSet(&yymsp[-4].minor.yy190,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-C*/ - yymsp[-4].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy72, 0, 0); - if( yymsp[-4].minor.yy190.pExpr ){ - yymsp[-4].minor.yy190.pExpr->x.pList = yymsp[-1].minor.yy72 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy148,yymsp[-1].minor.yy72) : yymsp[-2].minor.yy148; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy190.pExpr); - }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy148); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy72); - } -} - break; - case 197: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ -{ - yymsp[-4].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy148, yymsp[-2].minor.yy190.pExpr); - yymsp[-4].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy148, yymsp[0].minor.yy190.pExpr); -} - break; - case 198: /* case_exprlist ::= WHEN expr THEN expr */ -{ - yymsp[-3].minor.yy148 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy190.pExpr); - yymsp[-3].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy148, yymsp[0].minor.yy190.pExpr); -} - break; - case 201: /* case_operand ::= expr */ -{yymsp[0].minor.yy72 = yymsp[0].minor.yy190.pExpr; /*A-overwrites-X*/} - break; - case 204: /* nexprlist ::= nexprlist COMMA expr */ -{yymsp[-2].minor.yy148 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy148,yymsp[0].minor.yy190.pExpr);} - break; - case 205: /* nexprlist ::= expr */ -{yymsp[0].minor.yy148 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy190.pExpr); /*A-overwrites-Y*/} - break; - case 207: /* paren_exprlist ::= LP exprlist RP */ - case 212: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==212); -{yymsp[-2].minor.yy148 = yymsp[-1].minor.yy148;} - break; - case 208: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ -{ - sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy148, yymsp[-10].minor.yy194, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy72, SQLITE_SO_ASC, yymsp[-8].minor.yy194, SQLITE_IDXTYPE_APPDEF); -} - break; - case 209: /* uniqueflag ::= UNIQUE */ - case 250: /* raisetype ::= ABORT */ yytestcase(yyruleno==250); -{yymsp[0].minor.yy194 = OE_Abort;} - break; - case 210: /* uniqueflag ::= */ -{yymsp[1].minor.yy194 = OE_None;} - break; - case 213: /* eidlist ::= eidlist COMMA nm collate sortorder */ -{ - yymsp[-4].minor.yy148 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy148, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy194, yymsp[0].minor.yy194); -} - break; - case 214: /* eidlist ::= nm collate sortorder */ -{ - yymsp[-2].minor.yy148 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy194, yymsp[0].minor.yy194); /*A-overwrites-Y*/ -} - break; - case 217: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy185, yymsp[-1].minor.yy194);} - break; - case 218: /* cmd ::= VACUUM */ -{sqlite3Vacuum(pParse,0);} - break; - case 219: /* cmd ::= VACUUM nm */ -{sqlite3Vacuum(pParse,&yymsp[0].minor.yy0);} - break; - case 220: /* cmd ::= PRAGMA nm dbnm */ -{sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} - break; - case 221: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ -{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} - break; - case 222: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ -{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} - break; - case 223: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ -{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} - break; - case 224: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ -{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} - break; - case 227: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ -{ - Token all; - all.z = yymsp[-3].minor.yy0.z; - all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy145, &all); -} - break; - case 228: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ -{ - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy194, yymsp[-4].minor.yy332.a, yymsp[-4].minor.yy332.b, yymsp[-2].minor.yy185, yymsp[0].minor.yy72, yymsp[-10].minor.yy194, yymsp[-8].minor.yy194); - yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ -} - break; - case 229: /* trigger_time ::= BEFORE */ -{ yymsp[0].minor.yy194 = TK_BEFORE; } - break; - case 230: /* trigger_time ::= AFTER */ -{ yymsp[0].minor.yy194 = TK_AFTER; } - break; - case 231: /* trigger_time ::= INSTEAD OF */ -{ yymsp[-1].minor.yy194 = TK_INSTEAD;} - break; - case 232: /* trigger_time ::= */ -{ yymsp[1].minor.yy194 = TK_BEFORE; } - break; - case 233: /* trigger_event ::= DELETE|INSERT */ - case 234: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==234); -{yymsp[0].minor.yy332.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy332.b = 0;} - break; - case 235: /* trigger_event ::= UPDATE OF idlist */ -{yymsp[-2].minor.yy332.a = TK_UPDATE; yymsp[-2].minor.yy332.b = yymsp[0].minor.yy254;} - break; - case 236: /* when_clause ::= */ - case 255: /* key_opt ::= */ yytestcase(yyruleno==255); -{ yymsp[1].minor.yy72 = 0; } - break; - case 237: /* when_clause ::= WHEN expr */ - case 256: /* key_opt ::= KEY expr */ yytestcase(yyruleno==256); -{ yymsp[-1].minor.yy72 = yymsp[0].minor.yy190.pExpr; } - break; - case 238: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ -{ - assert( yymsp[-2].minor.yy145!=0 ); - yymsp[-2].minor.yy145->pLast->pNext = yymsp[-1].minor.yy145; - yymsp[-2].minor.yy145->pLast = yymsp[-1].minor.yy145; -} - break; - case 239: /* trigger_cmd_list ::= trigger_cmd SEMI */ -{ - assert( yymsp[-1].minor.yy145!=0 ); - yymsp[-1].minor.yy145->pLast = yymsp[-1].minor.yy145; -} - break; - case 240: /* trnm ::= nm DOT nm */ -{ - yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; - sqlite3ErrorMsg(pParse, - "qualified table names are not allowed on INSERT, UPDATE, and DELETE " - "statements within triggers"); -} - break; - case 241: /* tridxby ::= INDEXED BY nm */ -{ - sqlite3ErrorMsg(pParse, - "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " - "within triggers"); -} - break; - case 242: /* tridxby ::= NOT INDEXED */ -{ - sqlite3ErrorMsg(pParse, - "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " - "within triggers"); -} - break; - case 243: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ -{yymsp[-6].minor.yy145 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy148, yymsp[0].minor.yy72, yymsp[-5].minor.yy194);} - break; - case 244: /* trigger_cmd ::= insert_cmd INTO trnm idlist_opt select */ -{yymsp[-4].minor.yy145 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy254, yymsp[0].minor.yy243, yymsp[-4].minor.yy194);/*A-overwrites-R*/} - break; - case 245: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ -{yymsp[-4].minor.yy145 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy72);} - break; - case 246: /* trigger_cmd ::= select */ -{yymsp[0].minor.yy145 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy243); /*A-overwrites-X*/} - break; - case 247: /* expr ::= RAISE LP IGNORE RP */ -{ - spanSet(&yymsp[-3].minor.yy190,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ - yymsp[-3].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); - if( yymsp[-3].minor.yy190.pExpr ){ - yymsp[-3].minor.yy190.pExpr->affinity = OE_Ignore; - } -} - break; - case 248: /* expr ::= RAISE LP raisetype COMMA nm RP */ -{ - spanSet(&yymsp[-5].minor.yy190,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ - yymsp[-5].minor.yy190.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); - if( yymsp[-5].minor.yy190.pExpr ) { - yymsp[-5].minor.yy190.pExpr->affinity = (char)yymsp[-3].minor.yy194; - } -} - break; - case 249: /* raisetype ::= ROLLBACK */ -{yymsp[0].minor.yy194 = OE_Rollback;} - break; - case 251: /* raisetype ::= FAIL */ -{yymsp[0].minor.yy194 = OE_Fail;} - break; - case 252: /* cmd ::= DROP TRIGGER ifexists fullname */ -{ - sqlite3DropTrigger(pParse,yymsp[0].minor.yy185,yymsp[-1].minor.yy194); -} - break; - case 253: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ -{ - sqlite3Attach(pParse, yymsp[-3].minor.yy190.pExpr, yymsp[-1].minor.yy190.pExpr, yymsp[0].minor.yy72); -} - break; - case 254: /* cmd ::= DETACH database_kw_opt expr */ -{ - sqlite3Detach(pParse, yymsp[0].minor.yy190.pExpr); -} - break; - case 257: /* cmd ::= REINDEX */ -{sqlite3Reindex(pParse, 0, 0);} - break; - case 258: /* cmd ::= REINDEX nm dbnm */ -{sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} - break; - case 259: /* cmd ::= ANALYZE */ -{sqlite3Analyze(pParse, 0, 0);} - break; - case 260: /* cmd ::= ANALYZE nm dbnm */ -{sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} - break; - case 261: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ -{ - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy185,&yymsp[0].minor.yy0); -} - break; - case 262: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ -{ - yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; - sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); -} - break; - case 263: /* add_column_fullname ::= fullname */ -{ - disableLookaside(pParse); - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy185); -} - break; - case 264: /* cmd ::= create_vtab */ -{sqlite3VtabFinishParse(pParse,0);} - break; - case 265: /* cmd ::= create_vtab LP vtabarglist RP */ -{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} - break; - case 266: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ -{ - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy194); -} - break; - case 267: /* vtabarg ::= */ -{sqlite3VtabArgInit(pParse);} - break; - case 268: /* vtabargtoken ::= ANY */ - case 269: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==269); - case 270: /* lp ::= LP */ yytestcase(yyruleno==270); -{sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} - break; - case 271: /* with ::= */ -{yymsp[1].minor.yy285 = 0;} - break; - case 272: /* with ::= WITH wqlist */ -{ yymsp[-1].minor.yy285 = yymsp[0].minor.yy285; } - break; - case 273: /* with ::= WITH RECURSIVE wqlist */ -{ yymsp[-2].minor.yy285 = yymsp[0].minor.yy285; } - break; - case 274: /* wqlist ::= nm eidlist_opt AS LP select RP */ -{ - yymsp[-5].minor.yy285 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy148, yymsp[-1].minor.yy243); /*A-overwrites-X*/ -} - break; - case 275: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ -{ - yymsp[-7].minor.yy285 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy285, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy148, yymsp[-1].minor.yy243); -} - break; - default: - /* (276) input ::= cmdlist */ yytestcase(yyruleno==276); - /* (277) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==277); - /* (278) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=278); - /* (279) ecmd ::= SEMI */ yytestcase(yyruleno==279); - /* (280) ecmd ::= explain cmdx SEMI */ yytestcase(yyruleno==280); - /* (281) explain ::= */ yytestcase(yyruleno==281); - /* (282) trans_opt ::= */ yytestcase(yyruleno==282); - /* (283) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==283); - /* (284) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==284); - /* (285) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==285); - /* (286) savepoint_opt ::= */ yytestcase(yyruleno==286); - /* (287) cmd ::= create_table create_table_args */ yytestcase(yyruleno==287); - /* (288) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==288); - /* (289) columnlist ::= columnname carglist */ yytestcase(yyruleno==289); - /* (290) nm ::= ID|INDEXED */ yytestcase(yyruleno==290); - /* (291) nm ::= STRING */ yytestcase(yyruleno==291); - /* (292) nm ::= JOIN_KW */ yytestcase(yyruleno==292); - /* (293) typetoken ::= typename */ yytestcase(yyruleno==293); - /* (294) typename ::= ID|STRING */ yytestcase(yyruleno==294); - /* (295) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=295); - /* (296) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=296); - /* (297) carglist ::= carglist ccons */ yytestcase(yyruleno==297); - /* (298) carglist ::= */ yytestcase(yyruleno==298); - /* (299) ccons ::= NULL onconf */ yytestcase(yyruleno==299); - /* (300) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==300); - /* (301) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==301); - /* (302) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=302); - /* (303) tconscomma ::= */ yytestcase(yyruleno==303); - /* (304) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=304); - /* (305) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=305); - /* (306) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=306); - /* (307) oneselect ::= values */ yytestcase(yyruleno==307); - /* (308) sclp ::= selcollist COMMA */ yytestcase(yyruleno==308); - /* (309) as ::= ID|STRING */ yytestcase(yyruleno==309); - /* (310) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=310); - /* (311) exprlist ::= nexprlist */ yytestcase(yyruleno==311); - /* (312) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=312); - /* (313) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=313); - /* (314) nmnum ::= ON */ yytestcase(yyruleno==314); - /* (315) nmnum ::= DELETE */ yytestcase(yyruleno==315); - /* (316) nmnum ::= DEFAULT */ yytestcase(yyruleno==316); - /* (317) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==317); - /* (318) foreach_clause ::= */ yytestcase(yyruleno==318); - /* (319) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==319); - /* (320) trnm ::= nm */ yytestcase(yyruleno==320); - /* (321) tridxby ::= */ yytestcase(yyruleno==321); - /* (322) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==322); - /* (323) database_kw_opt ::= */ yytestcase(yyruleno==323); - /* (324) kwcolumn_opt ::= */ yytestcase(yyruleno==324); - /* (325) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==325); - /* (326) vtabarglist ::= vtabarg */ yytestcase(yyruleno==326); - /* (327) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==327); - /* (328) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==328); - /* (329) anylist ::= */ yytestcase(yyruleno==329); - /* (330) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==330); - /* (331) anylist ::= anylist ANY */ yytestcase(yyruleno==331); - break; -/********** End reduce actions ************************************************/ - }; - assert( yyrulenoYY_MAX_SHIFT ){ - yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; - } - yymsp -= yysize-1; - yypParser->yytos = yymsp; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yyTraceShift(yypParser, yyact); - }else{ - assert( yyact == YY_ACCEPT_ACTION ); - yypParser->yytos -= yysize; - yy_accept(yypParser); - } -} - -/* -** The following code executes when the parse fails -*/ -#ifndef YYNOERRORRECOVERY -static void yy_parse_failed( - yyParser *yypParser /* The parser */ -){ - sqlite3ParserARG_FETCH; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); - } -#endif - while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will be executed whenever the - ** parser fails */ -/************ Begin %parse_failure code ***************************************/ -/************ End %parse_failure code *****************************************/ - sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ -} -#endif /* YYNOERRORRECOVERY */ - -/* -** The following code executes when a syntax error first occurs. -*/ -static void yy_syntax_error( - yyParser *yypParser, /* The parser */ - int yymajor, /* The major type of the error token */ - sqlite3ParserTOKENTYPE yyminor /* The minor type of the error token */ -){ - sqlite3ParserARG_FETCH; -#define TOKEN yyminor -/************ Begin %syntax_error code ****************************************/ - - UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ - assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); -/************ End %syntax_error code ******************************************/ - sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ -} - -/* -** The following is executed when the parser accepts -*/ -static void yy_accept( - yyParser *yypParser /* The parser */ -){ - sqlite3ParserARG_FETCH; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); - } -#endif -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - assert( yypParser->yytos==yypParser->yystack ); - /* Here code is inserted which will be executed whenever the - ** parser accepts */ -/*********** Begin %parse_accept code *****************************************/ -/*********** End %parse_accept code *******************************************/ - sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ -} - -/* The main parser program. -** The first argument is a pointer to a structure obtained from -** "sqlite3ParserAlloc" which describes the current state of the parser. -** The second argument is the major token number. The third is -** the minor token. The fourth optional argument is whatever the -** user wants (and specified in the grammar) and is available for -** use by the action routines. -** -** Inputs: -**
      -**
    • A pointer to the parser (an opaque structure.) -**
    • The major token number. -**
    • The minor token number. -**
    • An option argument of a grammar-specified type. -**
    -** -** Outputs: -** None. -*/ -SQLITE_PRIVATE void sqlite3Parser( - void *yyp, /* The parser */ - int yymajor, /* The major token code number */ - sqlite3ParserTOKENTYPE yyminor /* The value for the token */ - sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ -){ - YYMINORTYPE yyminorunion; - unsigned int yyact; /* The parser action. */ -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - int yyendofinput; /* True if we are at the end of input */ -#endif -#ifdef YYERRORSYMBOL - int yyerrorhit = 0; /* True if yymajor has invoked an error */ -#endif - yyParser *yypParser; /* The parser */ - - yypParser = (yyParser*)yyp; - assert( yypParser->yytos!=0 ); -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - yyendofinput = (yymajor==0); -#endif - sqlite3ParserARG_STORE; - -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sInput '%s'\n",yyTracePrompt,yyTokenName[yymajor]); - } -#endif - - do{ - yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyact <= YY_MAX_SHIFTREDUCE ){ - yy_shift(yypParser,yyact,yymajor,yyminor); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt--; -#endif - yymajor = YYNOCODE; - }else if( yyact <= YY_MAX_REDUCE ){ - yy_reduce(yypParser,yyact-YY_MIN_REDUCE); - }else{ - assert( yyact == YY_ERROR_ACTION ); - yyminorunion.yy0 = yyminor; -#ifdef YYERRORSYMBOL - int yymx; -#endif -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); - } -#endif -#ifdef YYERRORSYMBOL - /* A syntax error has occurred. - ** The response to an error depends upon whether or not the - ** grammar defines an error token "ERROR". - ** - ** This is what we do if the grammar does define ERROR: - ** - ** * Call the %syntax_error function. - ** - ** * Begin popping the stack until we enter a state where - ** it is legal to shift the error symbol, then shift - ** the error symbol. - ** - ** * Set the error count to three. - ** - ** * Begin accepting and shifting new tokens. No new error - ** processing will occur until three tokens have been - ** shifted successfully. - ** - */ - if( yypParser->yyerrcnt<0 ){ - yy_syntax_error(yypParser,yymajor,yyminor); - } - yymx = yypParser->yytos->major; - if( yymx==YYERRORSYMBOL || yyerrorhit ){ -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sDiscard input token %s\n", - yyTracePrompt,yyTokenName[yymajor]); - } -#endif - yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); - yymajor = YYNOCODE; - }else{ - while( yypParser->yytos >= yypParser->yystack - && yymx != YYERRORSYMBOL - && (yyact = yy_find_reduce_action( - yypParser->yytos->stateno, - YYERRORSYMBOL)) >= YY_MIN_REDUCE - ){ - yy_pop_parser_stack(yypParser); - } - if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - yymajor = YYNOCODE; - }else if( yymx!=YYERRORSYMBOL ){ - yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); - } - } - yypParser->yyerrcnt = 3; - yyerrorhit = 1; -#elif defined(YYNOERRORRECOVERY) - /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to - ** do any kind of error recovery. Instead, simply invoke the syntax - ** error routine and continue going as if nothing had happened. - ** - ** Applications can set this macro (for example inside %include) if - ** they intend to abandon the parse upon the first syntax error seen. - */ - yy_syntax_error(yypParser,yymajor, yyminor); - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yymajor = YYNOCODE; - -#else /* YYERRORSYMBOL is not defined */ - /* This is what we do if the grammar does not define ERROR: - ** - ** * Report an error message, and throw away the input token. - ** - ** * If the input token is $, then fail the parse. - ** - ** As before, subsequent error messages are suppressed until - ** three input tokens have been successfully shifted. - */ - if( yypParser->yyerrcnt<=0 ){ - yy_syntax_error(yypParser,yymajor, yyminor); - } - yypParser->yyerrcnt = 3; - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - if( yyendofinput ){ - yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - } - yymajor = YYNOCODE; -#endif - } - }while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack ); -#ifndef NDEBUG - if( yyTraceFILE ){ - yyStackEntry *i; - char cDiv = '['; - fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); - for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ - fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); - cDiv = ' '; - } - fprintf(yyTraceFILE,"]\n"); - } -#endif - return; -} - -/************** End of parse.c ***********************************************/ -/************** Begin file tokenize.c ****************************************/ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** An tokenizer for SQL -** -** This file contains C code that splits an SQL input string up into -** individual tokens and sends those tokens one-by-one over to the -** parser for analysis. -*/ -/* #include "sqliteInt.h" */ -/* #include */ - -/* Character classes for tokenizing -** -** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented -** using a lookup table, whereas a switch() directly on c uses a binary search. -** The lookup table is much faster. To maximize speed, and to ensure that -** a lookup table is used, all of the classes need to be small integers and -** all of them need to be used within the switch. -*/ -#define CC_X 0 /* The letter 'x', or start of BLOB literal */ -#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ -#define CC_ID 2 /* unicode characters usable in IDs */ -#define CC_DIGIT 3 /* Digits */ -#define CC_DOLLAR 4 /* '$' */ -#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ -#define CC_VARNUM 6 /* '?'. Numeric SQL variables */ -#define CC_SPACE 7 /* Space characters */ -#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ -#define CC_QUOTE2 9 /* '['. [...] style quoted ids */ -#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ -#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ -#define CC_LT 12 /* '<'. Part of < or <= or <> */ -#define CC_GT 13 /* '>'. Part of > or >= */ -#define CC_EQ 14 /* '='. Part of = or == */ -#define CC_BANG 15 /* '!'. Part of != */ -#define CC_SLASH 16 /* '/'. / or c-style comment */ -#define CC_LP 17 /* '(' */ -#define CC_RP 18 /* ')' */ -#define CC_SEMI 19 /* ';' */ -#define CC_PLUS 20 /* '+' */ -#define CC_STAR 21 /* '*' */ -#define CC_PERCENT 22 /* '%' */ -#define CC_COMMA 23 /* ',' */ -#define CC_AND 24 /* '&' */ -#define CC_TILDA 25 /* '~' */ -#define CC_DOT 26 /* '.' */ -#define CC_ILLEGAL 27 /* Illegal character */ - -static const unsigned char aiClass[] = { -#ifdef SQLITE_ASCII -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ -/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, -/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, -/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, -/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, -/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, -/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, -/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 -#endif -#ifdef SQLITE_EBCDIC -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ -/* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27, -/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, -/* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, -/* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, -/* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 12, 17, 20, 10, -/* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27, -/* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 7, -/* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8, -/* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* 9x */ 25, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, -/* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 9, 27, 27, 27, 27, 27, -/* Cx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* Dx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* Ex */ 27, 27, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, -/* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 27, 27, 27, 27, 27, 27, -#endif -}; - -/* -** The charMap() macro maps alphabetic characters (only) into their -** lower-case ASCII equivalent. On ASCII machines, this is just -** an upper-to-lower case map. On EBCDIC machines we also need -** to adjust the encoding. The mapping is only valid for alphabetics -** which are the only characters for which this feature is used. -** -** Used by keywordhash.h -*/ -#ifdef SQLITE_ASCII -# define charMap(X) sqlite3UpperToLower[(unsigned char)X] -#endif -#ifdef SQLITE_EBCDIC -# define charMap(X) ebcdicToAscii[(unsigned char)X] -const unsigned char ebcdicToAscii[] = { -/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ - 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ - 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ - 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ - 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ - 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ - 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ -}; -#endif - -/* -** The sqlite3KeywordCode function looks up an identifier to determine if -** it is a keyword. If it is a keyword, the token code of that keyword is -** returned. If the input is not a keyword, TK_ID is returned. -** -** The implementation of this routine was generated by a program, -** mkkeywordhash.c, located in the tool subdirectory of the distribution. -** The output of the mkkeywordhash.c program is written into a file -** named keywordhash.h and then included into this source file by -** the #include below. -*/ -/************** Include keywordhash.h in the middle of tokenize.c ************/ -/************** Begin file keywordhash.h *************************************/ -/***** This file contains automatically generated code ****** -** -** The code in this file has been automatically generated by -** -** sqlite/tool/mkkeywordhash.c -** -** The code in this file implements a function that determines whether -** or not a given identifier is really an SQL keyword. The same thing -** might be implemented more directly using a hand-written hash table. -** But by using this automatically generated code, the size of the code -** is substantially reduced. This is important for embedded applications -** on platforms with limited memory. -*/ -/* Hash score: 182 */ -static int keywordCode(const char *z, int n, int *pType){ - /* zText[] encodes 834 bytes of keywords in 554 bytes */ - /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ - /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ - /* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */ - /* UNIQUERYWITHOUTERELEASEATTACHAVINGROUPDATEBEGINNERECURSIVE */ - /* BETWEENOTNULLIKECASCADELETECASECOLLATECREATECURRENT_DATEDETACH */ - /* IMMEDIATEJOINSERTMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMITWHEN */ - /* WHERENAMEAFTEREPLACEANDEFAULTAUTOINCREMENTCASTCOLUMNCOMMIT */ - /* CONFLICTCROSSCURRENT_TIMESTAMPRIMARYDEFERREDISTINCTDROPFAIL */ - /* FROMFULLGLOBYIFISNULLORDERESTRICTRIGHTROLLBACKROWUNIONUSING */ - /* VACUUMVIEWINITIALLY */ - static const char zText[553] = { - 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', - 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', - 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', - 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', - 'E','R','R','A','B','L','E','L','S','E','X','C','E','P','T','R','A','N', - 'S','A','C','T','I','O','N','A','T','U','R','A','L','T','E','R','A','I', - 'S','E','X','C','L','U','S','I','V','E','X','I','S','T','S','A','V','E', - 'P','O','I','N','T','E','R','S','E','C','T','R','I','G','G','E','R','E', - 'F','E','R','E','N','C','E','S','C','O','N','S','T','R','A','I','N','T', - 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q', - 'U','E','R','Y','W','I','T','H','O','U','T','E','R','E','L','E','A','S', - 'E','A','T','T','A','C','H','A','V','I','N','G','R','O','U','P','D','A', - 'T','E','B','E','G','I','N','N','E','R','E','C','U','R','S','I','V','E', - 'B','E','T','W','E','E','N','O','T','N','U','L','L','I','K','E','C','A', - 'S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L','A', - 'T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D','A', - 'T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E','J', - 'O','I','N','S','E','R','T','M','A','T','C','H','P','L','A','N','A','L', - 'Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U','E', - 'S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','W','H', - 'E','R','E','N','A','M','E','A','F','T','E','R','E','P','L','A','C','E', - 'A','N','D','E','F','A','U','L','T','A','U','T','O','I','N','C','R','E', - 'M','E','N','T','C','A','S','T','C','O','L','U','M','N','C','O','M','M', - 'I','T','C','O','N','F','L','I','C','T','C','R','O','S','S','C','U','R', - 'R','E','N','T','_','T','I','M','E','S','T','A','M','P','R','I','M','A', - 'R','Y','D','E','F','E','R','R','E','D','I','S','T','I','N','C','T','D', - 'R','O','P','F','A','I','L','F','R','O','M','F','U','L','L','G','L','O', - 'B','Y','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S','T', - 'R','I','C','T','R','I','G','H','T','R','O','L','L','B','A','C','K','R', - 'O','W','U','N','I','O','N','U','S','I','N','G','V','A','C','U','U','M', - 'V','I','E','W','I','N','I','T','I','A','L','L','Y', - }; - static const unsigned char aHash[127] = { - 76, 105, 117, 74, 0, 45, 0, 0, 82, 0, 77, 0, 0, - 42, 12, 78, 15, 0, 116, 85, 54, 112, 0, 19, 0, 0, - 121, 0, 119, 115, 0, 22, 93, 0, 9, 0, 0, 70, 71, - 0, 69, 6, 0, 48, 90, 102, 0, 118, 101, 0, 0, 44, - 0, 103, 24, 0, 17, 0, 122, 53, 23, 0, 5, 110, 25, - 96, 0, 0, 124, 106, 60, 123, 57, 28, 55, 0, 91, 0, - 100, 26, 0, 99, 0, 0, 0, 95, 92, 97, 88, 109, 14, - 39, 108, 0, 81, 0, 18, 89, 111, 32, 0, 120, 80, 113, - 62, 46, 84, 0, 0, 94, 40, 59, 114, 0, 36, 0, 0, - 29, 0, 86, 63, 64, 0, 20, 61, 0, 56, - }; - static const unsigned char aNext[124] = { - 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, - 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 0, 0, 50, - 0, 43, 3, 47, 0, 0, 0, 0, 30, 0, 58, 0, 38, - 0, 0, 0, 1, 66, 0, 0, 67, 0, 41, 0, 0, 0, - 0, 0, 0, 49, 65, 0, 0, 0, 0, 31, 52, 16, 34, - 10, 0, 0, 0, 0, 0, 0, 0, 11, 72, 79, 0, 8, - 0, 104, 98, 0, 107, 0, 87, 0, 75, 51, 0, 27, 37, - 73, 83, 0, 35, 68, 0, 0, - }; - static const unsigned char aLen[124] = { - 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, - 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, - 11, 6, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10, - 4, 6, 2, 3, 9, 4, 2, 6, 5, 7, 4, 5, 7, - 6, 6, 5, 6, 5, 5, 9, 7, 7, 3, 2, 4, 4, - 7, 3, 6, 4, 7, 6, 12, 6, 9, 4, 6, 5, 4, - 7, 6, 5, 6, 7, 5, 4, 5, 6, 5, 7, 3, 7, - 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, 7, 8, 8, - 2, 4, 4, 4, 4, 4, 2, 2, 6, 5, 8, 5, 8, - 3, 5, 5, 6, 4, 9, 3, - }; - static const unsigned short int aOffset[124] = { - 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, - 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, - 86, 91, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152, - 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 184, 188, 192, - 199, 204, 209, 212, 218, 221, 225, 234, 240, 240, 240, 243, 246, - 250, 251, 255, 261, 265, 272, 278, 290, 296, 305, 307, 313, 318, - 320, 327, 332, 337, 343, 349, 354, 358, 361, 367, 371, 378, 380, - 387, 389, 391, 400, 404, 410, 416, 424, 429, 429, 445, 452, 459, - 460, 467, 471, 475, 479, 483, 486, 488, 490, 496, 500, 508, 513, - 521, 524, 529, 534, 540, 544, 549, - }; - static const unsigned char aCode[124] = { - TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, - TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, - TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, - TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, - TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, - TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW, - TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_SAVEPOINT, - TK_INTERSECT, TK_TRIGGER, TK_REFERENCES, TK_CONSTRAINT, TK_INTO, - TK_OFFSET, TK_OF, TK_SET, TK_TEMP, TK_TEMP, - TK_OR, TK_UNIQUE, TK_QUERY, TK_WITHOUT, TK_WITH, - TK_JOIN_KW, TK_RELEASE, TK_ATTACH, TK_HAVING, TK_GROUP, - TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RECURSIVE, TK_BETWEEN, - TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, TK_LIKE_KW, - TK_CASCADE, TK_ASC, TK_DELETE, TK_CASE, TK_COLLATE, - TK_CREATE, TK_CTIME_KW, TK_DETACH, TK_IMMEDIATE, TK_JOIN, - TK_INSERT, TK_MATCH, TK_PLAN, TK_ANALYZE, TK_PRAGMA, - TK_ABORT, TK_VALUES, TK_VIRTUAL, TK_LIMIT, TK_WHEN, - TK_WHERE, TK_RENAME, TK_AFTER, TK_REPLACE, TK_AND, - TK_DEFAULT, TK_AUTOINCR, TK_TO, TK_IN, TK_CAST, - TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, - TK_CTIME_KW, TK_PRIMARY, TK_DEFERRED, TK_DISTINCT, TK_IS, - TK_DROP, TK_FAIL, TK_FROM, TK_JOIN_KW, TK_LIKE_KW, - TK_BY, TK_IF, TK_ISNULL, TK_ORDER, TK_RESTRICT, - TK_JOIN_KW, TK_ROLLBACK, TK_ROW, TK_UNION, TK_USING, - TK_VACUUM, TK_VIEW, TK_INITIALLY, TK_ALL, - }; - int i, j; - const char *zKW; - if( n>=2 ){ - i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) % 127; - for(i=((int)aHash[i])-1; i>=0; i=((int)aNext[i])-1){ - if( aLen[i]!=n ) continue; - j = 0; - zKW = &zText[aOffset[i]]; -#ifdef SQLITE_ASCII - while( j=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) -#endif - -/* Make the IdChar function accessible from ctime.c */ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS -SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } -#endif - - -/* -** Return the length (in bytes) of the token that begins at z[0]. -** Store the token type in *tokenType before returning. -*/ -SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ - int i, c; - switch( aiClass[*z] ){ /* Switch on the character-class of the first byte - ** of the token. See the comment on the CC_ defines - ** above. */ - case CC_SPACE: { - testcase( z[0]==' ' ); - testcase( z[0]=='\t' ); - testcase( z[0]=='\n' ); - testcase( z[0]=='\f' ); - testcase( z[0]=='\r' ); - for(i=1; sqlite3Isspace(z[i]); i++){} - *tokenType = TK_SPACE; - return i; - } - case CC_MINUS: { - if( z[1]=='-' ){ - for(i=2; (c=z[i])!=0 && c!='\n'; i++){} - *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ - return i; - } - *tokenType = TK_MINUS; - return 1; - } - case CC_LP: { - *tokenType = TK_LP; - return 1; - } - case CC_RP: { - *tokenType = TK_RP; - return 1; - } - case CC_SEMI: { - *tokenType = TK_SEMI; - return 1; - } - case CC_PLUS: { - *tokenType = TK_PLUS; - return 1; - } - case CC_STAR: { - *tokenType = TK_STAR; - return 1; - } - case CC_SLASH: { - if( z[1]!='*' || z[2]==0 ){ - *tokenType = TK_SLASH; - return 1; - } - for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} - if( c ) i++; - *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ - return i; - } - case CC_PERCENT: { - *tokenType = TK_REM; - return 1; - } - case CC_EQ: { - *tokenType = TK_EQ; - return 1 + (z[1]=='='); - } - case CC_LT: { - if( (c=z[1])=='=' ){ - *tokenType = TK_LE; - return 2; - }else if( c=='>' ){ - *tokenType = TK_NE; - return 2; - }else if( c=='<' ){ - *tokenType = TK_LSHIFT; - return 2; - }else{ - *tokenType = TK_LT; - return 1; - } - } - case CC_GT: { - if( (c=z[1])=='=' ){ - *tokenType = TK_GE; - return 2; - }else if( c=='>' ){ - *tokenType = TK_RSHIFT; - return 2; - }else{ - *tokenType = TK_GT; - return 1; - } - } - case CC_BANG: { - if( z[1]!='=' ){ - *tokenType = TK_ILLEGAL; - return 1; - }else{ - *tokenType = TK_NE; - return 2; - } - } - case CC_PIPE: { - if( z[1]!='|' ){ - *tokenType = TK_BITOR; - return 1; - }else{ - *tokenType = TK_CONCAT; - return 2; - } - } - case CC_COMMA: { - *tokenType = TK_COMMA; - return 1; - } - case CC_AND: { - *tokenType = TK_BITAND; - return 1; - } - case CC_TILDA: { - *tokenType = TK_BITNOT; - return 1; - } - case CC_QUOTE: { - int delim = z[0]; - testcase( delim=='`' ); - testcase( delim=='\'' ); - testcase( delim=='"' ); - for(i=1; (c=z[i])!=0; i++){ - if( c==delim ){ - if( z[i+1]==delim ){ - i++; - }else{ - break; - } - } - } - if( c=='\'' ){ - *tokenType = TK_STRING; - return i+1; - }else if( c!=0 ){ - *tokenType = TK_ID; - return i+1; - }else{ - *tokenType = TK_ILLEGAL; - return i; - } - } - case CC_DOT: { -#ifndef SQLITE_OMIT_FLOATING_POINT - if( !sqlite3Isdigit(z[1]) ) -#endif - { - *tokenType = TK_DOT; - return 1; - } - /* If the next character is a digit, this is a floating point - ** number that begins with ".". Fall thru into the next case */ - } - case CC_DIGIT: { - testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); - testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); - testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); - testcase( z[0]=='9' ); - *tokenType = TK_INTEGER; -#ifndef SQLITE_OMIT_HEX_INTEGER - if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ - for(i=3; sqlite3Isxdigit(z[i]); i++){} - return i; - } -#endif - for(i=0; sqlite3Isdigit(z[i]); i++){} -#ifndef SQLITE_OMIT_FLOATING_POINT - if( z[i]=='.' ){ - i++; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } - if( (z[i]=='e' || z[i]=='E') && - ( sqlite3Isdigit(z[i+1]) - || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) - ) - ){ - i += 2; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } -#endif - while( IdChar(z[i]) ){ - *tokenType = TK_ILLEGAL; - i++; - } - return i; - } - case CC_QUOTE2: { - for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} - *tokenType = c==']' ? TK_ID : TK_ILLEGAL; - return i; - } - case CC_VARNUM: { - *tokenType = TK_VARIABLE; - for(i=1; sqlite3Isdigit(z[i]); i++){} - return i; - } - case CC_DOLLAR: - case CC_VARALPHA: { - int n = 0; - testcase( z[0]=='$' ); testcase( z[0]=='@' ); - testcase( z[0]==':' ); testcase( z[0]=='#' ); - *tokenType = TK_VARIABLE; - for(i=1; (c=z[i])!=0; i++){ - if( IdChar(c) ){ - n++; -#ifndef SQLITE_OMIT_TCL_VARIABLE - }else if( c=='(' && n>0 ){ - do{ - i++; - }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); - if( c==')' ){ - i++; - }else{ - *tokenType = TK_ILLEGAL; - } - break; - }else if( c==':' && z[i+1]==':' ){ - i++; -#endif - }else{ - break; - } - } - if( n==0 ) *tokenType = TK_ILLEGAL; - return i; - } - case CC_KYWD: { - for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} - if( IdChar(z[i]) ){ - /* This token started out using characters that can appear in keywords, - ** but z[i] is a character not allowed within keywords, so this must - ** be an identifier instead */ - i++; - break; - } - *tokenType = TK_ID; - return keywordCode((char*)z, i, tokenType); - } - case CC_X: { -#ifndef SQLITE_OMIT_BLOB_LITERAL - testcase( z[0]=='x' ); testcase( z[0]=='X' ); - if( z[1]=='\'' ){ - *tokenType = TK_BLOB; - for(i=2; sqlite3Isxdigit(z[i]); i++){} - if( z[i]!='\'' || i%2 ){ - *tokenType = TK_ILLEGAL; - while( z[i] && z[i]!='\'' ){ i++; } - } - if( z[i] ) i++; - return i; - } -#endif - /* If it is not a BLOB literal, then it must be an ID, since no - ** SQL keywords start with the letter 'x'. Fall through */ - } - case CC_ID: { - i = 1; - break; - } - default: { - *tokenType = TK_ILLEGAL; - return 1; - } - } - while( IdChar(z[i]) ){ i++; } - *tokenType = TK_ID; - return i; -} - -/* -** Run the parser on the given SQL string. The parser structure is -** passed in. An SQLITE_ status code is returned. If an error occurs -** then an and attempt is made to write an error message into -** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that -** error message. -*/ -SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ - int nErr = 0; /* Number of errors encountered */ - int i; /* Loop counter */ - void *pEngine; /* The LEMON-generated LALR(1) parser */ - int tokenType; /* type of the next token */ - int lastTokenParsed = -1; /* type of the previous token */ - sqlite3 *db = pParse->db; /* The database connection */ - int mxSqlLen; /* Max length of an SQL string */ - - assert( zSql!=0 ); - mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; - if( db->nVdbeActive==0 ){ - db->u1.isInterrupted = 0; - } - pParse->rc = SQLITE_OK; - pParse->zTail = zSql; - i = 0; - assert( pzErrMsg!=0 ); - /* sqlite3ParserTrace(stdout, "parser: "); */ - pEngine = sqlite3ParserAlloc(sqlite3Malloc); - if( pEngine==0 ){ - sqlite3OomFault(db); - return SQLITE_NOMEM_BKPT; - } - assert( pParse->pNewTable==0 ); - assert( pParse->pNewTrigger==0 ); - assert( pParse->nVar==0 ); - assert( pParse->nzVar==0 ); - assert( pParse->azVar==0 ); - while( 1 ){ - assert( i>=0 ); - if( zSql[i]!=0 ){ - pParse->sLastToken.z = &zSql[i]; - pParse->sLastToken.n = sqlite3GetToken((u8*)&zSql[i],&tokenType); - i += pParse->sLastToken.n; - if( i>mxSqlLen ){ - pParse->rc = SQLITE_TOOBIG; - break; - } - }else{ - /* Upon reaching the end of input, call the parser two more times - ** with tokens TK_SEMI and 0, in that order. */ - if( lastTokenParsed==TK_SEMI ){ - tokenType = 0; - }else if( lastTokenParsed==0 ){ - break; - }else{ - tokenType = TK_SEMI; - } - } - if( tokenType>=TK_SPACE ){ - assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); - if( db->u1.isInterrupted ){ - pParse->rc = SQLITE_INTERRUPT; - break; - } - if( tokenType==TK_ILLEGAL ){ - sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", - &pParse->sLastToken); - break; - } - }else{ - sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); - lastTokenParsed = tokenType; - if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; - } - } - assert( nErr==0 ); - pParse->zTail = &zSql[i]; -#ifdef YYTRACKMAXSTACKDEPTH - sqlite3_mutex_enter(sqlite3MallocMutex()); - sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, - sqlite3ParserStackPeak(pEngine) - ); - sqlite3_mutex_leave(sqlite3MallocMutex()); -#endif /* YYDEBUG */ - sqlite3ParserFree(pEngine, sqlite3_free); - if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM_BKPT; - } - if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ - pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); - } - assert( pzErrMsg!=0 ); - if( pParse->zErrMsg ){ - *pzErrMsg = pParse->zErrMsg; - sqlite3_log(pParse->rc, "%s", *pzErrMsg); - pParse->zErrMsg = 0; - nErr++; - } - if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ - sqlite3VdbeDelete(pParse->pVdbe); - pParse->pVdbe = 0; - } -#ifndef SQLITE_OMIT_SHARED_CACHE - if( pParse->nested==0 ){ - sqlite3DbFree(db, pParse->aTableLock); - pParse->aTableLock = 0; - pParse->nTableLock = 0; - } -#endif -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3_free(pParse->apVtabLock); -#endif - - if( !IN_DECLARE_VTAB ){ - /* If the pParse->declareVtab flag is set, do not delete any table - ** structure built up in pParse->pNewTable. The calling code (see vtab.c) - ** will take responsibility for freeing the Table structure. - */ - sqlite3DeleteTable(db, pParse->pNewTable); - } - - if( pParse->pWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree); - sqlite3DeleteTrigger(db, pParse->pNewTrigger); - for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); - sqlite3DbFree(db, pParse->azVar); - while( pParse->pAinc ){ - AutoincInfo *p = pParse->pAinc; - pParse->pAinc = p->pNext; - sqlite3DbFree(db, p); - } - while( pParse->pZombieTab ){ - Table *p = pParse->pZombieTab; - pParse->pZombieTab = p->pNextZombie; - sqlite3DeleteTable(db, p); - } - assert( nErr==0 || pParse->rc!=SQLITE_OK ); - return nErr; -} - -/************** End of tokenize.c ********************************************/ -/************** Begin file complete.c ****************************************/ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** An tokenizer for SQL -** -** This file contains C code that implements the sqlite3_complete() API. -** This code used to be part of the tokenizer.c source file. But by -** separating it out, the code will be automatically omitted from -** static links that do not use it. -*/ -/* #include "sqliteInt.h" */ -#ifndef SQLITE_OMIT_COMPLETE - -/* -** This is defined in tokenize.c. We just have to import the definition. -*/ -#ifndef SQLITE_AMALGAMATION -#ifdef SQLITE_ASCII -#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) -#endif -#ifdef SQLITE_EBCDIC -SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; -#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) -#endif -#endif /* SQLITE_AMALGAMATION */ - - -/* -** Token types used by the sqlite3_complete() routine. See the header -** comments on that procedure for additional information. -*/ -#define tkSEMI 0 -#define tkWS 1 -#define tkOTHER 2 -#ifndef SQLITE_OMIT_TRIGGER -#define tkEXPLAIN 3 -#define tkCREATE 4 -#define tkTEMP 5 -#define tkTRIGGER 6 -#define tkEND 7 -#endif - -/* -** Return TRUE if the given SQL string ends in a semicolon. -** -** Special handling is require for CREATE TRIGGER statements. -** Whenever the CREATE TRIGGER keywords are seen, the statement -** must end with ";END;". -** -** This implementation uses a state machine with 8 states: -** -** (0) INVALID We have not yet seen a non-whitespace character. -** -** (1) START At the beginning or end of an SQL statement. This routine -** returns 1 if it ends in the START state and 0 if it ends -** in any other state. -** -** (2) NORMAL We are in the middle of statement which ends with a single -** semicolon. -** -** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of -** a statement. -** -** (4) CREATE The keyword CREATE has been seen at the beginning of a -** statement, possibly preceded by EXPLAIN and/or followed by -** TEMP or TEMPORARY -** -** (5) TRIGGER We are in the middle of a trigger definition that must be -** ended by a semicolon, the keyword END, and another semicolon. -** -** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at -** the end of a trigger definition. -** -** (7) END We've seen the ";END" of the ";END;" that occurs at the end -** of a trigger definition. -** -** Transitions between states above are determined by tokens extracted -** from the input. The following tokens are significant: -** -** (0) tkSEMI A semicolon. -** (1) tkWS Whitespace. -** (2) tkOTHER Any other SQL token. -** (3) tkEXPLAIN The "explain" keyword. -** (4) tkCREATE The "create" keyword. -** (5) tkTEMP The "temp" or "temporary" keyword. -** (6) tkTRIGGER The "trigger" keyword. -** (7) tkEND The "end" keyword. -** -** Whitespace never causes a state transition and is always ignored. -** This means that a SQL string of all whitespace is invalid. -** -** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed -** to recognize the end of a trigger can be omitted. All we have to do -** is look for a semicolon that is not part of an string or comment. -*/ -SQLITE_API int sqlite3_complete(const char *zSql){ - u8 state = 0; /* Current state, using numbers defined in header comment */ - u8 token; /* Value of the next token */ - -#ifndef SQLITE_OMIT_TRIGGER - /* A complex statement machine used to detect the end of a CREATE TRIGGER - ** statement. This is the normal case. - */ - static const u8 trans[8][8] = { - /* Token: */ - /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ - /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, - /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, - /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, - /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, - /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, - /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, - /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, - /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, - }; -#else - /* If triggers are not supported by this compile then the statement machine - ** used to detect the end of a statement is much simpler - */ - static const u8 trans[3][3] = { - /* Token: */ - /* State: ** SEMI WS OTHER */ - /* 0 INVALID: */ { 1, 0, 2, }, - /* 1 START: */ { 1, 1, 2, }, - /* 2 NORMAL: */ { 1, 2, 2, }, - }; -#endif /* SQLITE_OMIT_TRIGGER */ - -#ifdef SQLITE_ENABLE_API_ARMOR - if( zSql==0 ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - - while( *zSql ){ - switch( *zSql ){ - case ';': { /* A semicolon */ - token = tkSEMI; - break; - } - case ' ': - case '\r': - case '\t': - case '\n': - case '\f': { /* White space is ignored */ - token = tkWS; - break; - } - case '/': { /* C-style comments */ - if( zSql[1]!='*' ){ - token = tkOTHER; - break; - } - zSql += 2; - while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } - if( zSql[0]==0 ) return 0; - zSql++; - token = tkWS; - break; - } - case '-': { /* SQL-style comments from "--" to end of line */ - if( zSql[1]!='-' ){ - token = tkOTHER; - break; - } - while( *zSql && *zSql!='\n' ){ zSql++; } - if( *zSql==0 ) return state==1; - token = tkWS; - break; - } - case '[': { /* Microsoft-style identifiers in [...] */ - zSql++; - while( *zSql && *zSql!=']' ){ zSql++; } - if( *zSql==0 ) return 0; - token = tkOTHER; - break; - } - case '`': /* Grave-accent quoted symbols used by MySQL */ - case '"': /* single- and double-quoted strings */ - case '\'': { - int c = *zSql; - zSql++; - while( *zSql && *zSql!=c ){ zSql++; } - if( *zSql==0 ) return 0; - token = tkOTHER; - break; - } - default: { -#ifdef SQLITE_EBCDIC - unsigned char c; -#endif - if( IdChar((u8)*zSql) ){ - /* Keywords and unquoted identifiers */ - int nId; - for(nId=1; IdChar(zSql[nId]); nId++){} -#ifdef SQLITE_OMIT_TRIGGER - token = tkOTHER; -#else - switch( *zSql ){ - case 'c': case 'C': { - if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ - token = tkCREATE; - }else{ - token = tkOTHER; - } - break; - } - case 't': case 'T': { - if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ - token = tkTRIGGER; - }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ - token = tkTEMP; - }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ - token = tkTEMP; - }else{ - token = tkOTHER; - } - break; - } - case 'e': case 'E': { - if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ - token = tkEND; - }else -#ifndef SQLITE_OMIT_EXPLAIN - if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ - token = tkEXPLAIN; - }else -#endif - { - token = tkOTHER; - } - break; - } - default: { - token = tkOTHER; - break; - } - } -#endif /* SQLITE_OMIT_TRIGGER */ - zSql += nId-1; - }else{ - /* Operators and special symbols */ - token = tkOTHER; - } - break; - } - } - state = trans[state][token]; - zSql++; - } - return state==1; -} - -#ifndef SQLITE_OMIT_UTF16 -/* -** This routine is the same as the sqlite3_complete() routine described -** above, except that the parameter is required to be UTF-16 encoded, not -** UTF-8. -*/ -SQLITE_API int sqlite3_complete16(const void *zSql){ - sqlite3_value *pVal; - char const *zSql8; - int rc; - -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; -#endif - pVal = sqlite3ValueNew(0); - sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); - zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); - if( zSql8 ){ - rc = sqlite3_complete(zSql8); - }else{ - rc = SQLITE_NOMEM_BKPT; - } - sqlite3ValueFree(pVal); - return rc & 0xff; -} -#endif /* SQLITE_OMIT_UTF16 */ -#endif /* SQLITE_OMIT_COMPLETE */ - -/************** End of complete.c ********************************************/ -/************** Begin file main.c ********************************************/ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Main file for the SQLite library. The routines in this file -** implement the programmer interface to the library. Routines in -** other files are for internal use by SQLite and should not be -** accessed by users of the library. -*/ -/* #include "sqliteInt.h" */ - -#ifdef SQLITE_ENABLE_FTS3 -/************** Include fts3.h in the middle of main.c ***********************/ -/************** Begin file fts3.h ********************************************/ -/* -** 2006 Oct 10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file is used by programs that want to link against the -** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. -*/ -/* #include "sqlite3.h" */ - -#if 0 -extern "C" { -#endif /* __cplusplus */ - -SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); - -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ - -/************** End of fts3.h ************************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif -#ifdef SQLITE_ENABLE_RTREE -/************** Include rtree.h in the middle of main.c **********************/ -/************** Begin file rtree.h *******************************************/ -/* -** 2008 May 26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file is used by programs that want to link against the -** RTREE library. All it does is declare the sqlite3RtreeInit() interface. -*/ -/* #include "sqlite3.h" */ - -#if 0 -extern "C" { -#endif /* __cplusplus */ - -SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); - -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ - -/************** End of rtree.h ***********************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif -#ifdef SQLITE_ENABLE_ICU -/************** Include sqliteicu.h in the middle of main.c ******************/ -/************** Begin file sqliteicu.h ***************************************/ -/* -** 2008 May 26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file is used by programs that want to link against the -** ICU extension. All it does is declare the sqlite3IcuInit() interface. -*/ -/* #include "sqlite3.h" */ - -#if 0 -extern "C" { -#endif /* __cplusplus */ - -SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); - -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ - - -/************** End of sqliteicu.h *******************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif -#ifdef SQLITE_ENABLE_JSON1 -SQLITE_PRIVATE int sqlite3Json1Init(sqlite3*); -#endif -#ifdef SQLITE_ENABLE_FTS5 -SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*); -#endif - -#ifndef SQLITE_AMALGAMATION -/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant -** contains the text of SQLITE_VERSION macro. -*/ -SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; -#endif - -/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns -** a pointer to the to the sqlite3_version[] string constant. -*/ -SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } - -/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a -** pointer to a string constant whose value is the same as the -** SQLITE_SOURCE_ID C preprocessor macro. -*/ -SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } - -/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function -** returns an integer equal to SQLITE_VERSION_NUMBER. -*/ -SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } - -/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns -** zero if and only if SQLite was compiled with mutexing code omitted due to -** the SQLITE_THREADSAFE compile-time option being set to 0. -*/ -SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } - -/* -** When compiling the test fixture or with debugging enabled (on Win32), -** this variable being set to non-zero will cause OSTRACE macros to emit -** extra diagnostic information. -*/ -#ifdef SQLITE_HAVE_OS_TRACE -# ifndef SQLITE_DEBUG_OS_TRACE -# define SQLITE_DEBUG_OS_TRACE 0 -# endif - int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; -#endif - -#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) -/* -** If the following function pointer is not NULL and if -** SQLITE_ENABLE_IOTRACE is enabled, then messages describing -** I/O active are written using this function. These messages -** are intended for debugging activity only. -*/ -SQLITE_API void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...) = 0; -#endif - -/* -** If the following global variable points to a string which is the -** name of a directory, then that directory will be used to store -** temporary files. -** -** See also the "PRAGMA temp_store_directory" SQL command. -*/ -SQLITE_API char *sqlite3_temp_directory = 0; - -/* -** If the following global variable points to a string which is the -** name of a directory, then that directory will be used to store -** all database files specified with a relative pathname. -** -** See also the "PRAGMA data_store_directory" SQL command. -*/ -SQLITE_API char *sqlite3_data_directory = 0; - -/* -** Initialize SQLite. -** -** This routine must be called to initialize the memory allocation, -** VFS, and mutex subsystems prior to doing any serious work with -** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT -** this routine will be called automatically by key routines such as -** sqlite3_open(). -** -** This routine is a no-op except on its very first call for the process, -** or for the first call after a call to sqlite3_shutdown. -** -** The first thread to call this routine runs the initialization to -** completion. If subsequent threads call this routine before the first -** thread has finished the initialization process, then the subsequent -** threads must block until the first thread finishes with the initialization. -** -** The first thread might call this routine recursively. Recursive -** calls to this routine should not block, of course. Otherwise the -** initialization process would never complete. -** -** Let X be the first thread to enter this routine. Let Y be some other -** thread. Then while the initial invocation of this routine by X is -** incomplete, it is required that: -** -** * Calls to this routine from Y must block until the outer-most -** call by X completes. -** -** * Recursive calls to this routine from thread X return immediately -** without blocking. -*/ -SQLITE_API int sqlite3_initialize(void){ - MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ - int rc; /* Result code */ -#ifdef SQLITE_EXTRA_INIT - int bRunExtraInit = 0; /* Extra initialization needed */ -#endif - -#ifdef SQLITE_OMIT_WSD - rc = sqlite3_wsd_init(4096, 24); - if( rc!=SQLITE_OK ){ - return rc; - } -#endif - - /* If the following assert() fails on some obscure processor/compiler - ** combination, the work-around is to set the correct pointer - ** size at compile-time using -DSQLITE_PTRSIZE=n compile-time option */ - assert( SQLITE_PTRSIZE==sizeof(char*) ); - - /* If SQLite is already completely initialized, then this call - ** to sqlite3_initialize() should be a no-op. But the initialization - ** must be complete. So isInit must not be set until the very end - ** of this routine. - */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; - - /* Make sure the mutex subsystem is initialized. If unable to - ** initialize the mutex subsystem, return early with the error. - ** If the system is so sick that we are unable to allocate a mutex, - ** there is not much SQLite is going to be able to do. - ** - ** The mutex subsystem must take care of serializing its own - ** initialization. - */ - rc = sqlite3MutexInit(); - if( rc ) return rc; - - /* Initialize the malloc() system and the recursive pInitMutex mutex. - ** This operation is protected by the STATIC_MASTER mutex. Note that - ** MutexAlloc() is called for a static mutex prior to initializing the - ** malloc subsystem - this implies that the allocation of a static - ** mutex must not require support from the malloc subsystem. - */ - MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) - sqlite3_mutex_enter(pMaster); - sqlite3GlobalConfig.isMutexInit = 1; - if( !sqlite3GlobalConfig.isMallocInit ){ - rc = sqlite3MallocInit(); - } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.isMallocInit = 1; - if( !sqlite3GlobalConfig.pInitMutex ){ - sqlite3GlobalConfig.pInitMutex = - sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); - if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ - rc = SQLITE_NOMEM_BKPT; - } - } - } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.nRefInitMutex++; - } - sqlite3_mutex_leave(pMaster); - - /* If rc is not SQLITE_OK at this point, then either the malloc - ** subsystem could not be initialized or the system failed to allocate - ** the pInitMutex mutex. Return an error in either case. */ - if( rc!=SQLITE_OK ){ - return rc; - } - - /* Do the rest of the initialization under the recursive mutex so - ** that we will be able to handle recursive calls into - ** sqlite3_initialize(). The recursive calls normally come through - ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other - ** recursive calls might also be possible. - ** - ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls - ** to the xInit method, so the xInit method need not be threadsafe. - ** - ** The following mutex is what serializes access to the appdef pcache xInit - ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the - ** call to sqlite3PcacheInitialize(). - */ - sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); - if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ - sqlite3GlobalConfig.inProgress = 1; -#ifdef SQLITE_ENABLE_SQLLOG - { - extern void sqlite3_init_sqllog(void); - sqlite3_init_sqllog(); - } -#endif - memset(&sqlite3BuiltinFunctions, 0, sizeof(sqlite3BuiltinFunctions)); - sqlite3RegisterBuiltinFunctions(); - if( sqlite3GlobalConfig.isPCacheInit==0 ){ - rc = sqlite3PcacheInitialize(); - } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.isPCacheInit = 1; - rc = sqlite3OsInit(); - } - if( rc==SQLITE_OK ){ - sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, - sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); - sqlite3GlobalConfig.isInit = 1; -#ifdef SQLITE_EXTRA_INIT - bRunExtraInit = 1; -#endif - } - sqlite3GlobalConfig.inProgress = 0; - } - sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); - - /* Go back under the static mutex and clean up the recursive - ** mutex to prevent a resource leak. - */ - sqlite3_mutex_enter(pMaster); - sqlite3GlobalConfig.nRefInitMutex--; - if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ - assert( sqlite3GlobalConfig.nRefInitMutex==0 ); - sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); - sqlite3GlobalConfig.pInitMutex = 0; - } - sqlite3_mutex_leave(pMaster); - - /* The following is just a sanity check to make sure SQLite has - ** been compiled correctly. It is important to run this code, but - ** we don't want to run it too often and soak up CPU cycles for no - ** reason. So we run it once during initialization. - */ -#ifndef NDEBUG -#ifndef SQLITE_OMIT_FLOATING_POINT - /* This section of code's only "output" is via assert() statements. */ - if ( rc==SQLITE_OK ){ - u64 x = (((u64)1)<<63)-1; - double y; - assert(sizeof(x)==8); - assert(sizeof(x)==sizeof(y)); - memcpy(&y, &x, 8); - assert( sqlite3IsNaN(y) ); - } -#endif -#endif - - /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT - ** compile-time option. - */ -#ifdef SQLITE_EXTRA_INIT - if( bRunExtraInit ){ - int SQLITE_EXTRA_INIT(const char*); - rc = SQLITE_EXTRA_INIT(0); - } -#endif - - return rc; -} - -/* -** Undo the effects of sqlite3_initialize(). Must not be called while -** there are outstanding database connections or memory allocations or -** while any part of SQLite is otherwise in use in any thread. This -** routine is not threadsafe. But it is safe to invoke this routine -** on when SQLite is already shut down. If SQLite is already shut down -** when this routine is invoked, then this routine is a harmless no-op. -*/ -SQLITE_API int sqlite3_shutdown(void){ -#ifdef SQLITE_OMIT_WSD - int rc = sqlite3_wsd_init(4096, 24); - if( rc!=SQLITE_OK ){ - return rc; - } -#endif - - if( sqlite3GlobalConfig.isInit ){ -#ifdef SQLITE_EXTRA_SHUTDOWN - void SQLITE_EXTRA_SHUTDOWN(void); - SQLITE_EXTRA_SHUTDOWN(); -#endif - sqlite3_os_end(); - sqlite3_reset_auto_extension(); - sqlite3GlobalConfig.isInit = 0; - } - if( sqlite3GlobalConfig.isPCacheInit ){ - sqlite3PcacheShutdown(); - sqlite3GlobalConfig.isPCacheInit = 0; - } - if( sqlite3GlobalConfig.isMallocInit ){ - sqlite3MallocEnd(); - sqlite3GlobalConfig.isMallocInit = 0; - -#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES - /* The heap subsystem has now been shutdown and these values are supposed - ** to be NULL or point to memory that was obtained from sqlite3_malloc(), - ** which would rely on that heap subsystem; therefore, make sure these - ** values cannot refer to heap memory that was just invalidated when the - ** heap subsystem was shutdown. This is only done if the current call to - ** this function resulted in the heap subsystem actually being shutdown. - */ - sqlite3_data_directory = 0; - sqlite3_temp_directory = 0; -#endif - } - if( sqlite3GlobalConfig.isMutexInit ){ - sqlite3MutexEnd(); - sqlite3GlobalConfig.isMutexInit = 0; - } - - return SQLITE_OK; -} - -/* -** This API allows applications to modify the global configuration of -** the SQLite library at run-time. -** -** This routine should only be called when there are no outstanding -** database connections or memory allocations. This routine is not -** threadsafe. Failure to heed these warnings can lead to unpredictable -** behavior. -*/ -SQLITE_API int sqlite3_config(int op, ...){ - va_list ap; - int rc = SQLITE_OK; - - /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while - ** the SQLite library is in use. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; - - va_start(ap, op); - switch( op ){ - - /* Mutex configuration options are only available in a threadsafe - ** compile. - */ -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ - case SQLITE_CONFIG_SINGLETHREAD: { - /* EVIDENCE-OF: R-02748-19096 This option sets the threading mode to - ** Single-thread. */ - sqlite3GlobalConfig.bCoreMutex = 0; /* Disable mutex on core */ - sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ - case SQLITE_CONFIG_MULTITHREAD: { - /* EVIDENCE-OF: R-14374-42468 This option sets the threading mode to - ** Multi-thread. */ - sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ - sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ - case SQLITE_CONFIG_SERIALIZED: { - /* EVIDENCE-OF: R-41220-51800 This option sets the threading mode to - ** Serialized. */ - sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ - sqlite3GlobalConfig.bFullMutex = 1; /* Enable mutex on connections */ - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ - case SQLITE_CONFIG_MUTEX: { - /* Specify an alternative mutex implementation */ - sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ - case SQLITE_CONFIG_GETMUTEX: { - /* Retrieve the current mutex implementation */ - *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; - break; - } -#endif - - case SQLITE_CONFIG_MALLOC: { - /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a - ** single argument which is a pointer to an instance of the - ** sqlite3_mem_methods structure. The argument specifies alternative - ** low-level memory allocation routines to be used in place of the memory - ** allocation routines built into SQLite. */ - sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); - break; - } - case SQLITE_CONFIG_GETMALLOC: { - /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a - ** single argument which is a pointer to an instance of the - ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is - ** filled with the currently defined memory allocation routines. */ - if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); - *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; - break; - } - case SQLITE_CONFIG_MEMSTATUS: { - /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes - ** single argument of type int, interpreted as a boolean, which enables - ** or disables the collection of memory allocation statistics. */ - sqlite3GlobalConfig.bMemstat = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_SCRATCH: { - /* EVIDENCE-OF: R-08404-60887 There are three arguments to - ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from - ** which the scratch allocations will be drawn, the size of each scratch - ** allocation (sz), and the maximum number of scratch allocations (N). */ - sqlite3GlobalConfig.pScratch = va_arg(ap, void*); - sqlite3GlobalConfig.szScratch = va_arg(ap, int); - sqlite3GlobalConfig.nScratch = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_PAGECACHE: { - /* EVIDENCE-OF: R-18761-36601 There are three arguments to - ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory (pMem), - ** the size of each page cache line (sz), and the number of cache lines - ** (N). */ - sqlite3GlobalConfig.pPage = va_arg(ap, void*); - sqlite3GlobalConfig.szPage = va_arg(ap, int); - sqlite3GlobalConfig.nPage = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_PCACHE_HDRSZ: { - /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes - ** a single parameter which is a pointer to an integer and writes into - ** that integer the number of extra bytes per page required for each page - ** in SQLITE_CONFIG_PAGECACHE. */ - *va_arg(ap, int*) = - sqlite3HeaderSizeBtree() + - sqlite3HeaderSizePcache() + - sqlite3HeaderSizePcache1(); - break; - } - - case SQLITE_CONFIG_PCACHE: { - /* no-op */ - break; - } - case SQLITE_CONFIG_GETPCACHE: { - /* now an error */ - rc = SQLITE_ERROR; - break; - } - - case SQLITE_CONFIG_PCACHE2: { - /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a - ** single argument which is a pointer to an sqlite3_pcache_methods2 - ** object. This object specifies the interface to a custom page cache - ** implementation. */ - sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); - break; - } - case SQLITE_CONFIG_GETPCACHE2: { - /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a - ** single argument which is a pointer to an sqlite3_pcache_methods2 - ** object. SQLite copies of the current page cache implementation into - ** that object. */ - if( sqlite3GlobalConfig.pcache2.xInit==0 ){ - sqlite3PCacheSetDefault(); - } - *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; - break; - } - -/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only -** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or -** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */ -#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) - case SQLITE_CONFIG_HEAP: { - /* EVIDENCE-OF: R-19854-42126 There are three arguments to - ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the - ** number of bytes in the memory buffer, and the minimum allocation size. - */ - sqlite3GlobalConfig.pHeap = va_arg(ap, void*); - sqlite3GlobalConfig.nHeap = va_arg(ap, int); - sqlite3GlobalConfig.mnReq = va_arg(ap, int); - - if( sqlite3GlobalConfig.mnReq<1 ){ - sqlite3GlobalConfig.mnReq = 1; - }else if( sqlite3GlobalConfig.mnReq>(1<<12) ){ - /* cap min request size at 2^12 */ - sqlite3GlobalConfig.mnReq = (1<<12); - } - - if( sqlite3GlobalConfig.pHeap==0 ){ - /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) - ** is NULL, then SQLite reverts to using its default memory allocator - ** (the system malloc() implementation), undoing any prior invocation of - ** SQLITE_CONFIG_MALLOC. - ** - ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to - ** revert to its default implementation when sqlite3_initialize() is run - */ - memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); - }else{ - /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the - ** alternative memory allocator is engaged to handle all of SQLites - ** memory allocation needs. */ -#ifdef SQLITE_ENABLE_MEMSYS3 - sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); -#endif -#ifdef SQLITE_ENABLE_MEMSYS5 - sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); -#endif - } - break; - } -#endif - - case SQLITE_CONFIG_LOOKASIDE: { - sqlite3GlobalConfig.szLookaside = va_arg(ap, int); - sqlite3GlobalConfig.nLookaside = va_arg(ap, int); - break; - } - - /* Record a pointer to the logger function and its first argument. - ** The default is NULL. Logging is disabled if the function pointer is - ** NULL. - */ - case SQLITE_CONFIG_LOG: { - /* MSVC is picky about pulling func ptrs from va lists. - ** http://support.microsoft.com/kb/47961 - ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); - */ - typedef void(*LOGFUNC_t)(void*,int,const char*); - sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); - sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); - break; - } - - /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames - ** can be changed at start-time using the - ** sqlite3_config(SQLITE_CONFIG_URI,1) or - ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. - */ - case SQLITE_CONFIG_URI: { - /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single - ** argument of type int. If non-zero, then URI handling is globally - ** enabled. If the parameter is zero, then URI handling is globally - ** disabled. */ - sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); - break; - } - - case SQLITE_CONFIG_COVERING_INDEX_SCAN: { - /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN - ** option takes a single integer argument which is interpreted as a - ** boolean in order to enable or disable the use of covering indices for - ** full table scans in the query optimizer. */ - sqlite3GlobalConfig.bUseCis = va_arg(ap, int); - break; - } - -#ifdef SQLITE_ENABLE_SQLLOG - case SQLITE_CONFIG_SQLLOG: { - typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); - sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); - sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); - break; - } -#endif - - case SQLITE_CONFIG_MMAP_SIZE: { - /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit - ** integer (sqlite3_int64) values that are the default mmap size limit - ** (the default setting for PRAGMA mmap_size) and the maximum allowed - ** mmap size limit. */ - sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); - sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); - /* EVIDENCE-OF: R-53367-43190 If either argument to this option is - ** negative, then that argument is changed to its compile-time default. - ** - ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be - ** silently truncated if necessary so that it does not exceed the - ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE - ** compile-time option. - */ - if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ - mxMmap = SQLITE_MAX_MMAP_SIZE; - } - if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; - if( szMmap>mxMmap) szMmap = mxMmap; - sqlite3GlobalConfig.mxMmap = mxMmap; - sqlite3GlobalConfig.szMmap = szMmap; - break; - } - -#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ - case SQLITE_CONFIG_WIN32_HEAPSIZE: { - /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit - ** unsigned integer value that specifies the maximum size of the created - ** heap. */ - sqlite3GlobalConfig.nHeap = va_arg(ap, int); - break; - } -#endif - - case SQLITE_CONFIG_PMASZ: { - sqlite3GlobalConfig.szPma = va_arg(ap, unsigned int); - break; - } - - case SQLITE_CONFIG_STMTJRNL_SPILL: { - sqlite3GlobalConfig.nStmtSpill = va_arg(ap, int); - break; - } - - default: { - rc = SQLITE_ERROR; - break; - } - } - va_end(ap); - return rc; -} - -/* -** Set up the lookaside buffers for a database connection. -** Return SQLITE_OK on success. -** If lookaside is already active, return SQLITE_BUSY. -** -** The sz parameter is the number of bytes in each lookaside slot. -** The cnt parameter is the number of slots. If pStart is NULL the -** space for the lookaside memory is obtained from sqlite3_malloc(). -** If pStart is not NULL then it is sz*cnt bytes of memory to use for -** the lookaside memory. -*/ -static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ -#ifndef SQLITE_OMIT_LOOKASIDE - void *pStart; - if( db->lookaside.nOut ){ - return SQLITE_BUSY; - } - /* Free any existing lookaside buffer for this handle before - ** allocating a new one so we don't have to have space for - ** both at the same time. - */ - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); - } - /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger - ** than a pointer to be useful. - */ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ - if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; - if( cnt<0 ) cnt = 0; - if( sz==0 || cnt==0 ){ - sz = 0; - pStart = 0; - }else if( pBuf==0 ){ - sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ - sqlite3EndBenignMalloc(); - if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; - }else{ - pStart = pBuf; - } - db->lookaside.pStart = pStart; - db->lookaside.pFree = 0; - db->lookaside.sz = (u16)sz; - if( pStart ){ - int i; - LookasideSlot *p; - assert( sz > (int)sizeof(LookasideSlot*) ); - p = (LookasideSlot*)pStart; - for(i=cnt-1; i>=0; i--){ - p->pNext = db->lookaside.pFree; - db->lookaside.pFree = p; - p = (LookasideSlot*)&((u8*)p)[sz]; - } - db->lookaside.pEnd = p; - db->lookaside.bDisable = 0; - db->lookaside.bMalloced = pBuf==0 ?1:0; - }else{ - db->lookaside.pStart = db; - db->lookaside.pEnd = db; - db->lookaside.bDisable = 1; - db->lookaside.bMalloced = 0; - } -#endif /* SQLITE_OMIT_LOOKASIDE */ - return SQLITE_OK; -} - -/* -** Return the mutex associated with a database connection. -*/ -SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - return db->mutex; -} - -/* -** Free up as much memory as we can from the given database -** connection. -*/ -SQLITE_API int sqlite3_db_release_memory(sqlite3 *db){ - int i; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - for(i=0; inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - Pager *pPager = sqlite3BtreePager(pBt); - sqlite3PagerShrink(pPager); - } - } - sqlite3BtreeLeaveAll(db); - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} - -/* -** Flush any dirty pages in the pager-cache for any attached database -** to disk. -*/ -SQLITE_API int sqlite3_db_cacheflush(sqlite3 *db){ - int i; - int rc = SQLITE_OK; - int bSeenBusy = 0; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - for(i=0; rc==SQLITE_OK && inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt && sqlite3BtreeIsInTrans(pBt) ){ - Pager *pPager = sqlite3BtreePager(pBt); - rc = sqlite3PagerFlush(pPager); - if( rc==SQLITE_BUSY ){ - bSeenBusy = 1; - rc = SQLITE_OK; - } - } - } - sqlite3BtreeLeaveAll(db); - sqlite3_mutex_leave(db->mutex); - return ((rc==SQLITE_OK && bSeenBusy) ? SQLITE_BUSY : rc); -} - -/* -** Configuration settings for an individual database connection -*/ -SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ - va_list ap; - int rc; - va_start(ap, op); - switch( op ){ - case SQLITE_DBCONFIG_MAINDBNAME: { - db->aDb[0].zDbSName = va_arg(ap,char*); - rc = SQLITE_OK; - break; - } - case SQLITE_DBCONFIG_LOOKASIDE: { - void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ - int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ - int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ - rc = setupLookaside(db, pBuf, sz, cnt); - break; - } - default: { - static const struct { - int op; /* The opcode */ - u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ - } aFlagOp[] = { - { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, - { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, - { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, - { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, - }; - unsigned int i; - rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ - for(i=0; iflags; - if( onoff>0 ){ - db->flags |= aFlagOp[i].mask; - }else if( onoff==0 ){ - db->flags &= ~aFlagOp[i].mask; - } - if( oldFlags!=db->flags ){ - sqlite3ExpirePreparedStatements(db); - } - if( pRes ){ - *pRes = (db->flags & aFlagOp[i].mask)!=0; - } - rc = SQLITE_OK; - break; - } - } - break; - } - } - va_end(ap); - return rc; -} - - -/* -** Return true if the buffer z[0..n-1] contains all spaces. -*/ -static int allSpaces(const char *z, int n){ - while( n>0 && z[n-1]==' ' ){ n--; } - return n==0; -} - -/* -** This is the default collating function named "BINARY" which is always -** available. -** -** If the padFlag argument is not NULL then space padding at the end -** of strings is ignored. This implements the RTRIM collation. -*/ -static int binCollFunc( - void *padFlag, - int nKey1, const void *pKey1, - int nKey2, const void *pKey2 -){ - int rc, n; - n = nKey1lastRowid; -} - -/* -** Return the number of changes in the most recent call to sqlite3_exec(). -*/ -SQLITE_API int sqlite3_changes(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - return db->nChange; -} - -/* -** Return the number of changes since the database handle was opened. -*/ -SQLITE_API int sqlite3_total_changes(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - return db->nTotalChange; -} - -/* -** Close all open savepoints. This function only manipulates fields of the -** database handle object, it does not close any savepoints that may be open -** at the b-tree/pager level. -*/ -SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ - while( db->pSavepoint ){ - Savepoint *pTmp = db->pSavepoint; - db->pSavepoint = pTmp->pNext; - sqlite3DbFree(db, pTmp); - } - db->nSavepoint = 0; - db->nStatement = 0; - db->isTransactionSavepoint = 0; -} - -/* -** Invoke the destructor function associated with FuncDef p, if any. Except, -** if this is not the last copy of the function, do not invoke it. Multiple -** copies of a single function are created when create_function() is called -** with SQLITE_ANY as the encoding. -*/ -static void functionDestroy(sqlite3 *db, FuncDef *p){ - FuncDestructor *pDestructor = p->u.pDestructor; - if( pDestructor ){ - pDestructor->nRef--; - if( pDestructor->nRef==0 ){ - pDestructor->xDestroy(pDestructor->pUserData); - sqlite3DbFree(db, pDestructor); - } - } -} - -/* -** Disconnect all sqlite3_vtab objects that belong to database connection -** db. This is called when db is being closed. -*/ -static void disconnectAllVtab(sqlite3 *db){ -#ifndef SQLITE_OMIT_VIRTUALTABLE - int i; - HashElem *p; - sqlite3BtreeEnterAll(db); - for(i=0; inDb; i++){ - Schema *pSchema = db->aDb[i].pSchema; - if( db->aDb[i].pSchema ){ - for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ - Table *pTab = (Table *)sqliteHashData(p); - if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); - } - } - } - for(p=sqliteHashFirst(&db->aModule); p; p=sqliteHashNext(p)){ - Module *pMod = (Module *)sqliteHashData(p); - if( pMod->pEpoTab ){ - sqlite3VtabDisconnect(db, pMod->pEpoTab); - } - } - sqlite3VtabUnlockList(db); - sqlite3BtreeLeaveAll(db); -#else - UNUSED_PARAMETER(db); -#endif -} - -/* -** Return TRUE if database connection db has unfinalized prepared -** statements or unfinished sqlite3_backup objects. -*/ -static int connectionIsBusy(sqlite3 *db){ - int j; - assert( sqlite3_mutex_held(db->mutex) ); - if( db->pVdbe ) return 1; - for(j=0; jnDb; j++){ - Btree *pBt = db->aDb[j].pBt; - if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; - } - return 0; -} - -/* -** Close an existing SQLite database -*/ -static int sqlite3Close(sqlite3 *db, int forceZombie){ - if( !db ){ - /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or - ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ - return SQLITE_OK; - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - sqlite3_mutex_enter(db->mutex); - if( db->mTrace & SQLITE_TRACE_CLOSE ){ - db->xTrace(SQLITE_TRACE_CLOSE, db->pTraceArg, db, 0); - } - - /* Force xDisconnect calls on all virtual tables */ - disconnectAllVtab(db); - - /* If a transaction is open, the disconnectAllVtab() call above - ** will not have called the xDisconnect() method on any virtual - ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() - ** call will do so. We need to do this before the check for active - ** SQL statements below, as the v-table implementation may be storing - ** some prepared statements internally. - */ - sqlite3VtabRollback(db); - - /* Legacy behavior (sqlite3_close() behavior) is to return - ** SQLITE_BUSY if the connection can not be closed immediately. - */ - if( !forceZombie && connectionIsBusy(db) ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " - "statements or unfinished backups"); - sqlite3_mutex_leave(db->mutex); - return SQLITE_BUSY; - } - -#ifdef SQLITE_ENABLE_SQLLOG - if( sqlite3GlobalConfig.xSqllog ){ - /* Closing the handle. Fourth parameter is passed the value 2. */ - sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); - } -#endif - - /* Convert the connection into a zombie and then close it. - */ - db->magic = SQLITE_MAGIC_ZOMBIE; - sqlite3LeaveMutexAndCloseZombie(db); - return SQLITE_OK; -} - -/* -** Two variations on the public interface for closing a database -** connection. The sqlite3_close() version returns SQLITE_BUSY and -** leaves the connection option if there are unfinalized prepared -** statements or unfinished sqlite3_backups. The sqlite3_close_v2() -** version forces the connection to become a zombie if there are -** unclosed resources, and arranges for deallocation when the last -** prepare statement or sqlite3_backup closes. -*/ -SQLITE_API int sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } -SQLITE_API int sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } - - -/* -** Close the mutex on database connection db. -** -** Furthermore, if database connection db is a zombie (meaning that there -** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and -** every sqlite3_stmt has now been finalized and every sqlite3_backup has -** finished, then free all resources. -*/ -SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ - HashElem *i; /* Hash table iterator */ - int j; - - /* If there are outstanding sqlite3_stmt or sqlite3_backup objects - ** or if the connection has not yet been closed by sqlite3_close_v2(), - ** then just leave the mutex and return. - */ - if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ - sqlite3_mutex_leave(db->mutex); - return; - } - - /* If we reach this point, it means that the database connection has - ** closed all sqlite3_stmt and sqlite3_backup objects and has been - ** passed to sqlite3_close (meaning that it is a zombie). Therefore, - ** go ahead and free all resources. - */ - - /* If a transaction is open, roll it back. This also ensures that if - ** any database schemas have been modified by an uncommitted transaction - ** they are reset. And that the required b-tree mutex is held to make - ** the pager rollback and schema reset an atomic operation. */ - sqlite3RollbackAll(db, SQLITE_OK); - - /* Free any outstanding Savepoint structures. */ - sqlite3CloseSavepoints(db); - - /* Close all database connections */ - for(j=0; jnDb; j++){ - struct Db *pDb = &db->aDb[j]; - if( pDb->pBt ){ - sqlite3BtreeClose(pDb->pBt); - pDb->pBt = 0; - if( j!=1 ){ - pDb->pSchema = 0; - } - } - } - /* Clear the TEMP schema separately and last */ - if( db->aDb[1].pSchema ){ - sqlite3SchemaClear(db->aDb[1].pSchema); - } - sqlite3VtabUnlockList(db); - - /* Free up the array of auxiliary databases */ - sqlite3CollapseDatabaseArray(db); - assert( db->nDb<=2 ); - assert( db->aDb==db->aDbStatic ); - - /* Tell the code in notify.c that the connection no longer holds any - ** locks and does not require any further unlock-notify callbacks. - */ - sqlite3ConnectionClosed(db); - - for(i=sqliteHashFirst(&db->aFunc); i; i=sqliteHashNext(i)){ - FuncDef *pNext, *p; - p = sqliteHashData(i); - do{ - functionDestroy(db, p); - pNext = p->pNext; - sqlite3DbFree(db, p); - p = pNext; - }while( p ); - } - sqlite3HashClear(&db->aFunc); - for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ - CollSeq *pColl = (CollSeq *)sqliteHashData(i); - /* Invoke any destructors registered for collation sequence user data. */ - for(j=0; j<3; j++){ - if( pColl[j].xDel ){ - pColl[j].xDel(pColl[j].pUser); - } - } - sqlite3DbFree(db, pColl); - } - sqlite3HashClear(&db->aCollSeq); -#ifndef SQLITE_OMIT_VIRTUALTABLE - for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ - Module *pMod = (Module *)sqliteHashData(i); - if( pMod->xDestroy ){ - pMod->xDestroy(pMod->pAux); - } - sqlite3VtabEponymousTableClear(db, pMod); - sqlite3DbFree(db, pMod); - } - sqlite3HashClear(&db->aModule); -#endif - - sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ - sqlite3ValueFree(db->pErr); - sqlite3CloseExtensions(db); -#if SQLITE_USER_AUTHENTICATION - sqlite3_free(db->auth.zAuthUser); - sqlite3_free(db->auth.zAuthPW); -#endif - - db->magic = SQLITE_MAGIC_ERROR; - - /* The temp-database schema is allocated differently from the other schema - ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). - ** So it needs to be freed here. Todo: Why not roll the temp schema into - ** the same sqliteMalloc() as the one that allocates the database - ** structure? - */ - sqlite3DbFree(db, db->aDb[1].pSchema); - sqlite3_mutex_leave(db->mutex); - db->magic = SQLITE_MAGIC_CLOSED; - sqlite3_mutex_free(db->mutex); - assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); - } - sqlite3_free(db); -} - -/* -** Rollback all database files. If tripCode is not SQLITE_OK, then -** any write cursors are invalidated ("tripped" - as in "tripping a circuit -** breaker") and made to return tripCode if there are any further -** attempts to use that cursor. Read cursors remain open and valid -** but are "saved" in case the table pages are moved around. -*/ -SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ - int i; - int inTrans = 0; - int schemaChange; - assert( sqlite3_mutex_held(db->mutex) ); - sqlite3BeginBenignMalloc(); - - /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). - ** This is important in case the transaction being rolled back has - ** modified the database schema. If the b-tree mutexes are not taken - ** here, then another shared-cache connection might sneak in between - ** the database rollback and schema reset, which can cause false - ** corruption reports in some cases. */ - sqlite3BtreeEnterAll(db); - schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; - - for(i=0; inDb; i++){ - Btree *p = db->aDb[i].pBt; - if( p ){ - if( sqlite3BtreeIsInTrans(p) ){ - inTrans = 1; - } - sqlite3BtreeRollback(p, tripCode, !schemaChange); - } - } - sqlite3VtabRollback(db); - sqlite3EndBenignMalloc(); - - if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ - sqlite3ExpirePreparedStatements(db); - sqlite3ResetAllSchemasOfConnection(db); - } - sqlite3BtreeLeaveAll(db); - - /* Any deferred constraint violations have now been resolved. */ - db->nDeferredCons = 0; - db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; - - /* If one has been configured, invoke the rollback-hook callback */ - if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ - db->xRollbackCallback(db->pRollbackArg); - } -} - -/* -** Return a static string containing the name corresponding to the error code -** specified in the argument. -*/ -#if defined(SQLITE_NEED_ERR_NAME) -SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ - const char *zName = 0; - int i, origRc = rc; - for(i=0; i<2 && zName==0; i++, rc &= 0xff){ - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; - case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; - case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; - case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; - case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; - case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; - case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; - case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; - case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; - case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; - case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; - case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; - case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; - case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; - case SQLITE_IOERR_CHECKRESERVEDLOCK: - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; - case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; - case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; - case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; - case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; - case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; - case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; - case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; - case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; - case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; - case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; - case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; - case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; - case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; - case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; - case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; - case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; - case SQLITE_CONSTRAINT_FOREIGNKEY: - zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; - case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; - case SQLITE_CONSTRAINT_PRIMARYKEY: - zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; - case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; - case SQLITE_CONSTRAINT_COMMITHOOK: - zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; - case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; - case SQLITE_CONSTRAINT_FUNCTION: - zName = "SQLITE_CONSTRAINT_FUNCTION"; break; - case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; - case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; - case SQLITE_NOTICE_RECOVER_ROLLBACK: - zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; - case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; - case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - } - } - if( zName==0 ){ - static char zBuf[50]; - sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); - zName = zBuf; - } - return zName; -} -#endif - -/* -** Return a static string that describes the kind of error specified in the -** argument. -*/ -SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ - static const char* const aMsg[] = { - /* SQLITE_OK */ "not an error", - /* SQLITE_ERROR */ "SQL logic error or missing database", - /* SQLITE_INTERNAL */ 0, - /* SQLITE_PERM */ "access permission denied", - /* SQLITE_ABORT */ "callback requested query abort", - /* SQLITE_BUSY */ "database is locked", - /* SQLITE_LOCKED */ "database table is locked", - /* SQLITE_NOMEM */ "out of memory", - /* SQLITE_READONLY */ "attempt to write a readonly database", - /* SQLITE_INTERRUPT */ "interrupted", - /* SQLITE_IOERR */ "disk I/O error", - /* SQLITE_CORRUPT */ "database disk image is malformed", - /* SQLITE_NOTFOUND */ "unknown operation", - /* SQLITE_FULL */ "database or disk is full", - /* SQLITE_CANTOPEN */ "unable to open database file", - /* SQLITE_PROTOCOL */ "locking protocol", - /* SQLITE_EMPTY */ "table contains no data", - /* SQLITE_SCHEMA */ "database schema has changed", - /* SQLITE_TOOBIG */ "string or blob too big", - /* SQLITE_CONSTRAINT */ "constraint failed", - /* SQLITE_MISMATCH */ "datatype mismatch", - /* SQLITE_MISUSE */ "library routine called out of sequence", - /* SQLITE_NOLFS */ "large file support is disabled", - /* SQLITE_AUTH */ "authorization denied", - /* SQLITE_FORMAT */ "auxiliary database format error", - /* SQLITE_RANGE */ "bind or column index out of range", - /* SQLITE_NOTADB */ "file is encrypted or is not a database", - }; - const char *zErr = "unknown error"; - switch( rc ){ - case SQLITE_ABORT_ROLLBACK: { - zErr = "abort due to ROLLBACK"; - break; - } - default: { - rc &= 0xff; - if( ALWAYS(rc>=0) && rcbusyTimeout; - int delay, prior; - - assert( count>=0 ); - if( count < NDELAY ){ - delay = delays[count]; - prior = totals[count]; - }else{ - delay = delays[NDELAY-1]; - prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); - } - if( prior + delay > timeout ){ - delay = timeout - prior; - if( delay<=0 ) return 0; - } - sqlite3OsSleep(db->pVfs, delay*1000); - return 1; -#else - sqlite3 *db = (sqlite3 *)ptr; - int timeout = ((sqlite3 *)ptr)->busyTimeout; - if( (count+1)*1000 > timeout ){ - return 0; - } - sqlite3OsSleep(db->pVfs, 1000000); - return 1; -#endif -} - -/* -** Invoke the given busy handler. -** -** This routine is called when an operation failed with a lock. -** If this routine returns non-zero, the lock is retried. If it -** returns 0, the operation aborts with an SQLITE_BUSY error. -*/ -SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ - int rc; - if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0; - rc = p->xFunc(p->pArg, p->nBusy); - if( rc==0 ){ - p->nBusy = -1; - }else{ - p->nBusy++; - } - return rc; -} - -/* -** This routine sets the busy callback for an Sqlite database to the -** given callback function with the given argument. -*/ -SQLITE_API int sqlite3_busy_handler( - sqlite3 *db, - int (*xBusy)(void*,int), - void *pArg -){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - db->busyHandler.xFunc = xBusy; - db->busyHandler.pArg = pArg; - db->busyHandler.nBusy = 0; - db->busyTimeout = 0; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} - -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK -/* -** This routine sets the progress callback for an Sqlite database to the -** given callback function with the given argument. The progress callback will -** be invoked every nOps opcodes. -*/ -SQLITE_API void sqlite3_progress_handler( - sqlite3 *db, - int nOps, - int (*xProgress)(void*), - void *pArg -){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return; - } -#endif - sqlite3_mutex_enter(db->mutex); - if( nOps>0 ){ - db->xProgress = xProgress; - db->nProgressOps = (unsigned)nOps; - db->pProgressArg = pArg; - }else{ - db->xProgress = 0; - db->nProgressOps = 0; - db->pProgressArg = 0; - } - sqlite3_mutex_leave(db->mutex); -} -#endif - - -/* -** This routine installs a default busy handler that waits for the -** specified number of milliseconds before returning 0. -*/ -SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - if( ms>0 ){ - sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); - db->busyTimeout = ms; - }else{ - sqlite3_busy_handler(db, 0, 0); - } - return SQLITE_OK; -} - -/* -** Cause any pending operation to stop at its earliest opportunity. -*/ -SQLITE_API void sqlite3_interrupt(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return; - } -#endif - db->u1.isInterrupted = 1; -} - - -/* -** This function is exactly the same as sqlite3_create_function(), except -** that it is designed to be called by internal code. The difference is -** that if a malloc() fails in sqlite3_create_function(), an error code -** is returned and the mallocFailed flag cleared. -*/ -SQLITE_PRIVATE int sqlite3CreateFunc( - sqlite3 *db, - const char *zFunctionName, - int nArg, - int enc, - void *pUserData, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*), - FuncDestructor *pDestructor -){ - FuncDef *p; - int nName; - int extraFlags; - - assert( sqlite3_mutex_held(db->mutex) ); - if( zFunctionName==0 || - (xSFunc && (xFinal || xStep)) || - (!xSFunc && (xFinal && !xStep)) || - (!xSFunc && (!xFinal && xStep)) || - (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || - (255<(nName = sqlite3Strlen30( zFunctionName))) ){ - return SQLITE_MISUSE_BKPT; - } - - assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); - extraFlags = enc & SQLITE_DETERMINISTIC; - enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); - -#ifndef SQLITE_OMIT_UTF16 - /* If SQLITE_UTF16 is specified as the encoding type, transform this - ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the - ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. - ** - ** If SQLITE_ANY is specified, add three versions of the function - ** to the hash table. - */ - if( enc==SQLITE_UTF16 ){ - enc = SQLITE_UTF16NATIVE; - }else if( enc==SQLITE_ANY ){ - int rc; - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, - pUserData, xSFunc, xStep, xFinal, pDestructor); - if( rc==SQLITE_OK ){ - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, - pUserData, xSFunc, xStep, xFinal, pDestructor); - } - if( rc!=SQLITE_OK ){ - return rc; - } - enc = SQLITE_UTF16BE; - } -#else - enc = SQLITE_UTF8; -#endif - - /* Check if an existing function is being overridden or deleted. If so, - ** and there are active VMs, then return SQLITE_BUSY. If a function - ** is being overridden/deleted but there are no active VMs, allow the - ** operation to continue but invalidate all precompiled statements. - */ - p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 0); - if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ - if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, - "unable to delete/modify user-function due to active statements"); - assert( !db->mallocFailed ); - return SQLITE_BUSY; - }else{ - sqlite3ExpirePreparedStatements(db); - } - } - - p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 1); - assert(p || db->mallocFailed); - if( !p ){ - return SQLITE_NOMEM_BKPT; - } - - /* If an older version of the function with a configured destructor is - ** being replaced invoke the destructor function here. */ - functionDestroy(db, p); - - if( pDestructor ){ - pDestructor->nRef++; - } - p->u.pDestructor = pDestructor; - p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; - testcase( p->funcFlags & SQLITE_DETERMINISTIC ); - p->xSFunc = xSFunc ? xSFunc : xStep; - p->xFinalize = xFinal; - p->pUserData = pUserData; - p->nArg = (u16)nArg; - return SQLITE_OK; -} - -/* -** Create new user functions. -*/ -SQLITE_API int sqlite3_create_function( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*) -){ - return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xSFunc, xStep, - xFinal, 0); -} - -SQLITE_API int sqlite3_create_function_v2( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*), - void (*xDestroy)(void *) -){ - int rc = SQLITE_ERROR; - FuncDestructor *pArg = 0; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE_BKPT; - } -#endif - sqlite3_mutex_enter(db->mutex); - if( xDestroy ){ - pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); - if( !pArg ){ - xDestroy(p); - goto out; - } - pArg->xDestroy = xDestroy; - pArg->pUserData = p; - } - rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, pArg); - if( pArg && pArg->nRef==0 ){ - assert( rc!=SQLITE_OK ); - xDestroy(p); - sqlite3DbFree(db, pArg); - } - - out: - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} - -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API int sqlite3_create_function16( - sqlite3 *db, - const void *zFunctionName, - int nArg, - int eTextRep, - void *p, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), - void (*xStep)(sqlite3_context*,int,sqlite3_value**), - void (*xFinal)(sqlite3_context*) -){ - int rc; - char *zFunc8; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0); - sqlite3DbFree(db, zFunc8); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} -#endif - - -/* -** Declare that a function has been overloaded by a virtual table. -** -** If the function already exists as a regular global function, then -** this routine is a no-op. If the function does not exist, then create -** a new one that always throws a run-time error. -** -** When virtual tables intend to provide an overloaded function, they -** should call this routine to make sure the global function exists. -** A global function must exist in order for name resolution to work -** properly. -*/ -SQLITE_API int sqlite3_overload_function( - sqlite3 *db, - const char *zName, - int nArg -){ - int rc = SQLITE_OK; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ - return SQLITE_MISUSE_BKPT; - } -#endif - sqlite3_mutex_enter(db->mutex); - if( sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)==0 ){ - rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, - 0, sqlite3InvalidFunction, 0, 0, 0); - } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} - -#ifndef SQLITE_OMIT_TRACE -/* -** Register a trace function. The pArg from the previously registered trace -** is returned. -** -** A NULL trace function means that no tracing is executes. A non-NULL -** trace is a pointer to a function that is invoked at the start of each -** SQL statement. -*/ -#ifndef SQLITE_OMIT_DEPRECATED -SQLITE_API void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ - void *pOld; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - sqlite3_mutex_enter(db->mutex); - pOld = db->pTraceArg; - db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; - db->xTrace = (int(*)(u32,void*,void*,void*))xTrace; - db->pTraceArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; -} -#endif /* SQLITE_OMIT_DEPRECATED */ - -/* Register a trace callback using the version-2 interface. -*/ -SQLITE_API int sqlite3_trace_v2( - sqlite3 *db, /* Trace this connection */ - unsigned mTrace, /* Mask of events to be traced */ - int(*xTrace)(unsigned,void*,void*,void*), /* Callback to invoke */ - void *pArg /* Context */ -){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE_BKPT; - } -#endif - sqlite3_mutex_enter(db->mutex); - if( mTrace==0 ) xTrace = 0; - if( xTrace==0 ) mTrace = 0; - db->mTrace = mTrace; - db->xTrace = xTrace; - db->pTraceArg = pArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} - -#ifndef SQLITE_OMIT_DEPRECATED -/* -** Register a profile function. The pArg from the previously registered -** profile function is returned. -** -** A NULL profile function means that no profiling is executes. A non-NULL -** profile is a pointer to a function that is invoked at the conclusion of -** each SQL statement that is run. -*/ -SQLITE_API void *sqlite3_profile( - sqlite3 *db, - void (*xProfile)(void*,const char*,sqlite_uint64), - void *pArg -){ - void *pOld; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - sqlite3_mutex_enter(db->mutex); - pOld = db->pProfileArg; - db->xProfile = xProfile; - db->pProfileArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; -} -#endif /* SQLITE_OMIT_DEPRECATED */ -#endif /* SQLITE_OMIT_TRACE */ - -/* -** Register a function to be invoked when a transaction commits. -** If the invoked function returns non-zero, then the commit becomes a -** rollback. -*/ -SQLITE_API void *sqlite3_commit_hook( - sqlite3 *db, /* Attach the hook to this database */ - int (*xCallback)(void*), /* Function to invoke on each commit */ - void *pArg /* Argument to the function */ -){ - void *pOld; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - sqlite3_mutex_enter(db->mutex); - pOld = db->pCommitArg; - db->xCommitCallback = xCallback; - db->pCommitArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; -} - -/* -** Register a callback to be invoked each time a row is updated, -** inserted or deleted using this database connection. -*/ -SQLITE_API void *sqlite3_update_hook( - sqlite3 *db, /* Attach the hook to this database */ - void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), - void *pArg /* Argument to the function */ -){ - void *pRet; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - sqlite3_mutex_enter(db->mutex); - pRet = db->pUpdateArg; - db->xUpdateCallback = xCallback; - db->pUpdateArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; -} - -/* -** Register a callback to be invoked each time a transaction is rolled -** back by this database connection. -*/ -SQLITE_API void *sqlite3_rollback_hook( - sqlite3 *db, /* Attach the hook to this database */ - void (*xCallback)(void*), /* Callback function */ - void *pArg /* Argument to the function */ -){ - void *pRet; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - sqlite3_mutex_enter(db->mutex); - pRet = db->pRollbackArg; - db->xRollbackCallback = xCallback; - db->pRollbackArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; -} - -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK -/* -** Register a callback to be invoked each time a row is updated, -** inserted or deleted using this database connection. -*/ -SQLITE_API void *sqlite3_preupdate_hook( - sqlite3 *db, /* Attach the hook to this database */ - void(*xCallback)( /* Callback function */ - void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64), - void *pArg /* First callback argument */ -){ - void *pRet; - sqlite3_mutex_enter(db->mutex); - pRet = db->pPreUpdateArg; - db->xPreUpdateCallback = xCallback; - db->pPreUpdateArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; -} -#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ - -#ifndef SQLITE_OMIT_WAL -/* -** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). -** Invoke sqlite3_wal_checkpoint if the number of frames in the log file -** is greater than sqlite3.pWalArg cast to an integer (the value configured by -** wal_autocheckpoint()). -*/ -SQLITE_PRIVATE int sqlite3WalDefaultHook( - void *pClientData, /* Argument */ - sqlite3 *db, /* Connection */ - const char *zDb, /* Database */ - int nFrame /* Size of WAL */ -){ - if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ - sqlite3BeginBenignMalloc(); - sqlite3_wal_checkpoint(db, zDb); - sqlite3EndBenignMalloc(); - } - return SQLITE_OK; -} -#endif /* SQLITE_OMIT_WAL */ - -/* -** Configure an sqlite3_wal_hook() callback to automatically checkpoint -** a database after committing a transaction if there are nFrame or -** more frames in the log file. Passing zero or a negative value as the -** nFrame parameter disables automatic checkpoints entirely. -** -** The callback registered by this function replaces any existing callback -** registered using sqlite3_wal_hook(). Likewise, registering a callback -** using sqlite3_wal_hook() disables the automatic checkpoint mechanism -** configured by this function. -*/ -SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ -#ifdef SQLITE_OMIT_WAL - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(nFrame); -#else -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - if( nFrame>0 ){ - sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); - }else{ - sqlite3_wal_hook(db, 0, 0); - } -#endif - return SQLITE_OK; -} - -/* -** Register a callback to be invoked each time a transaction is written -** into the write-ahead-log by this database connection. -*/ -SQLITE_API void *sqlite3_wal_hook( - sqlite3 *db, /* Attach the hook to this db handle */ - int(*xCallback)(void *, sqlite3*, const char*, int), - void *pArg /* First argument passed to xCallback() */ -){ -#ifndef SQLITE_OMIT_WAL - void *pRet; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - sqlite3_mutex_enter(db->mutex); - pRet = db->pWalArg; - db->xWalCallback = xCallback; - db->pWalArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; -#else - return 0; -#endif -} - -/* -** Checkpoint database zDb. -*/ -SQLITE_API int sqlite3_wal_checkpoint_v2( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Name of attached database (or NULL) */ - int eMode, /* SQLITE_CHECKPOINT_* value */ - int *pnLog, /* OUT: Size of WAL log in frames */ - int *pnCkpt /* OUT: Total number of frames checkpointed */ -){ -#ifdef SQLITE_OMIT_WAL - return SQLITE_OK; -#else - int rc; /* Return code */ - int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - - /* Initialize the output variables to -1 in case an error occurs. */ - if( pnLog ) *pnLog = -1; - if( pnCkpt ) *pnCkpt = -1; - - assert( SQLITE_CHECKPOINT_PASSIVE==0 ); - assert( SQLITE_CHECKPOINT_FULL==1 ); - assert( SQLITE_CHECKPOINT_RESTART==2 ); - assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); - if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ - /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint - ** mode: */ - return SQLITE_MISUSE; - } - - sqlite3_mutex_enter(db->mutex); - if( zDb && zDb[0] ){ - iDb = sqlite3FindDbName(db, zDb); - } - if( iDb<0 ){ - rc = SQLITE_ERROR; - sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); - }else{ - db->busyHandler.nBusy = 0; - rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); - sqlite3Error(db, rc); - } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -#endif -} - - -/* -** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points -** to contains a zero-length string, all attached databases are -** checkpointed. -*/ -SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ - /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to - ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ - return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); -} - -#ifndef SQLITE_OMIT_WAL -/* -** Run a checkpoint on database iDb. This is a no-op if database iDb is -** not currently open in WAL mode. -** -** If a transaction is open on the database being checkpointed, this -** function returns SQLITE_LOCKED and a checkpoint is not attempted. If -** an error occurs while running the checkpoint, an SQLite error code is -** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. -** -** The mutex on database handle db should be held by the caller. The mutex -** associated with the specific b-tree being checkpointed is taken by -** this function while the checkpoint is running. -** -** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are -** checkpointed. If an error is encountered it is returned immediately - -** no attempt is made to checkpoint any remaining databases. -** -** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. -*/ -SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ - int rc = SQLITE_OK; /* Return code */ - int i; /* Used to iterate through attached dbs */ - int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ - - assert( sqlite3_mutex_held(db->mutex) ); - assert( !pnLog || *pnLog==-1 ); - assert( !pnCkpt || *pnCkpt==-1 ); - - for(i=0; inDb && rc==SQLITE_OK; i++){ - if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ - rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); - pnLog = 0; - pnCkpt = 0; - if( rc==SQLITE_BUSY ){ - bBusy = 1; - rc = SQLITE_OK; - } - } - } - - return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; -} -#endif /* SQLITE_OMIT_WAL */ - -/* -** This function returns true if main-memory should be used instead of -** a temporary file for transient pager files and statement journals. -** The value returned depends on the value of db->temp_store (runtime -** parameter) and the compile time value of SQLITE_TEMP_STORE. The -** following table describes the relationship between these two values -** and this functions return value. -** -** SQLITE_TEMP_STORE db->temp_store Location of temporary database -** ----------------- -------------- ------------------------------ -** 0 any file (return 0) -** 1 1 file (return 0) -** 1 2 memory (return 1) -** 1 0 file (return 0) -** 2 1 file (return 0) -** 2 2 memory (return 1) -** 2 0 memory (return 1) -** 3 any memory (return 1) -*/ -SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ -#if SQLITE_TEMP_STORE==1 - return ( db->temp_store==2 ); -#endif -#if SQLITE_TEMP_STORE==2 - return ( db->temp_store!=1 ); -#endif -#if SQLITE_TEMP_STORE==3 - UNUSED_PARAMETER(db); - return 1; -#endif -#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 - UNUSED_PARAMETER(db); - return 0; -#endif -} - -/* -** Return UTF-8 encoded English language explanation of the most recent -** error. -*/ -SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ - const char *z; - if( !db ){ - return sqlite3ErrStr(SQLITE_NOMEM_BKPT); - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return sqlite3ErrStr(SQLITE_MISUSE_BKPT); - } - sqlite3_mutex_enter(db->mutex); - if( db->mallocFailed ){ - z = sqlite3ErrStr(SQLITE_NOMEM_BKPT); - }else{ - testcase( db->pErr==0 ); - z = (char*)sqlite3_value_text(db->pErr); - assert( !db->mallocFailed ); - if( z==0 ){ - z = sqlite3ErrStr(db->errCode); - } - } - sqlite3_mutex_leave(db->mutex); - return z; -} - -#ifndef SQLITE_OMIT_UTF16 -/* -** Return UTF-16 encoded English language explanation of the most recent -** error. -*/ -SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ - static const u16 outOfMem[] = { - 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 - }; - static const u16 misuse[] = { - 'l', 'i', 'b', 'r', 'a', 'r', 'y', ' ', - 'r', 'o', 'u', 't', 'i', 'n', 'e', ' ', - 'c', 'a', 'l', 'l', 'e', 'd', ' ', - 'o', 'u', 't', ' ', - 'o', 'f', ' ', - 's', 'e', 'q', 'u', 'e', 'n', 'c', 'e', 0 - }; - - const void *z; - if( !db ){ - return (void *)outOfMem; - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return (void *)misuse; - } - sqlite3_mutex_enter(db->mutex); - if( db->mallocFailed ){ - z = (void *)outOfMem; - }else{ - z = sqlite3_value_text16(db->pErr); - if( z==0 ){ - sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); - z = sqlite3_value_text16(db->pErr); - } - /* A malloc() may have failed within the call to sqlite3_value_text16() - ** above. If this is the case, then the db->mallocFailed flag needs to - ** be cleared before returning. Do this directly, instead of via - ** sqlite3ApiExit(), to avoid setting the database handle error message. - */ - sqlite3OomClear(db); - } - sqlite3_mutex_leave(db->mutex); - return z; -} -#endif /* SQLITE_OMIT_UTF16 */ - -/* -** Return the most recent error code generated by an SQLite routine. If NULL is -** passed to this function, we assume a malloc() failed during sqlite3_open(). -*/ -SQLITE_API int sqlite3_errcode(sqlite3 *db){ - if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - if( !db || db->mallocFailed ){ - return SQLITE_NOMEM_BKPT; - } - return db->errCode & db->errMask; -} -SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ - if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - if( !db || db->mallocFailed ){ - return SQLITE_NOMEM_BKPT; - } - return db->errCode; -} -SQLITE_API int sqlite3_system_errno(sqlite3 *db){ - return db ? db->iSysErrno : 0; -} - -/* -** Return a string that describes the kind of error specified in the -** argument. For now, this simply calls the internal sqlite3ErrStr() -** function. -*/ -SQLITE_API const char *sqlite3_errstr(int rc){ - return sqlite3ErrStr(rc); -} - -/* -** Create a new collating function for database "db". The name is zName -** and the encoding is enc. -*/ -static int createCollation( - sqlite3* db, - const char *zName, - u8 enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*), - void(*xDel)(void*) -){ - CollSeq *pColl; - int enc2; - - assert( sqlite3_mutex_held(db->mutex) ); - - /* If SQLITE_UTF16 is specified as the encoding type, transform this - ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the - ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. - */ - enc2 = enc; - testcase( enc2==SQLITE_UTF16 ); - testcase( enc2==SQLITE_UTF16_ALIGNED ); - if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ - enc2 = SQLITE_UTF16NATIVE; - } - if( enc2SQLITE_UTF16BE ){ - return SQLITE_MISUSE_BKPT; - } - - /* Check if this call is removing or replacing an existing collation - ** sequence. If so, and there are active VMs, return busy. If there - ** are no active VMs, invalidate any pre-compiled statements. - */ - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); - if( pColl && pColl->xCmp ){ - if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, - "unable to delete/modify collation sequence due to active statements"); - return SQLITE_BUSY; - } - sqlite3ExpirePreparedStatements(db); - - /* If collation sequence pColl was created directly by a call to - ** sqlite3_create_collation, and not generated by synthCollSeq(), - ** then any copies made by synthCollSeq() need to be invalidated. - ** Also, collation destructor - CollSeq.xDel() - function may need - ** to be called. - */ - if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ - CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); - int j; - for(j=0; j<3; j++){ - CollSeq *p = &aColl[j]; - if( p->enc==pColl->enc ){ - if( p->xDel ){ - p->xDel(p->pUser); - } - p->xCmp = 0; - } - } - } - } - - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); - if( pColl==0 ) return SQLITE_NOMEM_BKPT; - pColl->xCmp = xCompare; - pColl->pUser = pCtx; - pColl->xDel = xDel; - pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - sqlite3Error(db, SQLITE_OK); - return SQLITE_OK; -} - - -/* -** This array defines hard upper bounds on limit values. The -** initializer must be kept in sync with the SQLITE_LIMIT_* -** #defines in sqlite3.h. -*/ -static const int aHardLimit[] = { - SQLITE_MAX_LENGTH, - SQLITE_MAX_SQL_LENGTH, - SQLITE_MAX_COLUMN, - SQLITE_MAX_EXPR_DEPTH, - SQLITE_MAX_COMPOUND_SELECT, - SQLITE_MAX_VDBE_OP, - SQLITE_MAX_FUNCTION_ARG, - SQLITE_MAX_ATTACHED, - SQLITE_MAX_LIKE_PATTERN_LENGTH, - SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ - SQLITE_MAX_TRIGGER_DEPTH, - SQLITE_MAX_WORKER_THREADS, -}; - -/* -** Make sure the hard limits are set to reasonable values -*/ -#if SQLITE_MAX_LENGTH<100 -# error SQLITE_MAX_LENGTH must be at least 100 -#endif -#if SQLITE_MAX_SQL_LENGTH<100 -# error SQLITE_MAX_SQL_LENGTH must be at least 100 -#endif -#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH -# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH -#endif -#if SQLITE_MAX_COMPOUND_SELECT<2 -# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 -#endif -#if SQLITE_MAX_VDBE_OP<40 -# error SQLITE_MAX_VDBE_OP must be at least 40 -#endif -#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127 -# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127 -#endif -#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 -# error SQLITE_MAX_ATTACHED must be between 0 and 125 -#endif -#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 -# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 -#endif -#if SQLITE_MAX_COLUMN>32767 -# error SQLITE_MAX_COLUMN must not exceed 32767 -#endif -#if SQLITE_MAX_TRIGGER_DEPTH<1 -# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 -#endif -#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 -# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 -#endif - - -/* -** Change the value of a limit. Report the old value. -** If an invalid limit index is supplied, report -1. -** Make no changes but still report the old value if the -** new limit is negative. -** -** A new lower limit does not shrink existing constructs. -** It merely prevents new constructs that exceed the limit -** from forming. -*/ -SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ - int oldLimit; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return -1; - } -#endif - - /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME - ** there is a hard upper bound set at compile-time by a C preprocessor - ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to - ** "_MAX_".) - */ - assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); - assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); - assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); - assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); - assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); - assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); - assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== - SQLITE_MAX_LIKE_PATTERN_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); - assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); - assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); - assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); - - - if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ - return -1; - } - oldLimit = db->aLimit[limitId]; - if( newLimit>=0 ){ /* IMP: R-52476-28732 */ - if( newLimit>aHardLimit[limitId] ){ - newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ - } - db->aLimit[limitId] = newLimit; - } - return oldLimit; /* IMP: R-53341-35419 */ -} - -/* -** This function is used to parse both URIs and non-URI filenames passed by the -** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database -** URIs specified as part of ATTACH statements. -** -** The first argument to this function is the name of the VFS to use (or -** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" -** query parameter. The second argument contains the URI (or non-URI filename) -** itself. When this function is called the *pFlags variable should contain -** the default flags to open the database handle with. The value stored in -** *pFlags may be updated before returning if the URI filename contains -** "cache=xxx" or "mode=xxx" query parameters. -** -** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to -** the VFS that should be used to open the database file. *pzFile is set to -** point to a buffer containing the name of the file to open. It is the -** responsibility of the caller to eventually call sqlite3_free() to release -** this buffer. -** -** If an error occurs, then an SQLite error code is returned and *pzErrMsg -** may be set to point to a buffer containing an English language error -** message. It is the responsibility of the caller to eventually release -** this buffer by calling sqlite3_free(). -*/ -SQLITE_PRIVATE int sqlite3ParseUri( - const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ - const char *zUri, /* Nul-terminated URI to parse */ - unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ - sqlite3_vfs **ppVfs, /* OUT: VFS to use */ - char **pzFile, /* OUT: Filename component of URI */ - char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ -){ - int rc = SQLITE_OK; - unsigned int flags = *pFlags; - const char *zVfs = zDefaultVfs; - char *zFile; - char c; - int nUri = sqlite3Strlen30(zUri); - - assert( *pzErrMsg==0 ); - - if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ - || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ - && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ - ){ - char *zOpt; - int eState; /* Parser state when parsing URI */ - int iIn; /* Input character index */ - int iOut = 0; /* Output character index */ - u64 nByte = nUri+2; /* Bytes of space to allocate */ - - /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen - ** method that there may be extra parameters following the file-name. */ - flags |= SQLITE_OPEN_URI; - - for(iIn=0; iIn=0 && octet<256 ); - if( octet==0 ){ - /* This branch is taken when "%00" appears within the URI. In this - ** case we ignore all text in the remainder of the path, name or - ** value currently being parsed. So ignore the current character - ** and skip to the next "?", "=" or "&", as appropriate. */ - while( (c = zUri[iIn])!=0 && c!='#' - && (eState!=0 || c!='?') - && (eState!=1 || (c!='=' && c!='&')) - && (eState!=2 || c!='&') - ){ - iIn++; - } - continue; - } - c = octet; - }else if( eState==1 && (c=='&' || c=='=') ){ - if( zFile[iOut-1]==0 ){ - /* An empty option name. Ignore this option altogether. */ - while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; - continue; - } - if( c=='&' ){ - zFile[iOut++] = '\0'; - }else{ - eState = 2; - } - c = 0; - }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ - c = 0; - eState = 1; - } - zFile[iOut++] = c; - } - if( eState==1 ) zFile[iOut++] = '\0'; - zFile[iOut++] = '\0'; - zFile[iOut++] = '\0'; - - /* Check if there were any options specified that should be interpreted - ** here. Options that are interpreted here include "vfs" and those that - ** correspond to flags that may be passed to the sqlite3_open_v2() - ** method. */ - zOpt = &zFile[sqlite3Strlen30(zFile)+1]; - while( zOpt[0] ){ - int nOpt = sqlite3Strlen30(zOpt); - char *zVal = &zOpt[nOpt+1]; - int nVal = sqlite3Strlen30(zVal); - - if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ - zVfs = zVal; - }else{ - struct OpenMode { - const char *z; - int mode; - } *aMode = 0; - char *zModeType = 0; - int mask = 0; - int limit = 0; - - if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ - static struct OpenMode aCacheMode[] = { - { "shared", SQLITE_OPEN_SHAREDCACHE }, - { "private", SQLITE_OPEN_PRIVATECACHE }, - { 0, 0 } - }; - - mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; - aMode = aCacheMode; - limit = mask; - zModeType = "cache"; - } - if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ - static struct OpenMode aOpenMode[] = { - { "ro", SQLITE_OPEN_READONLY }, - { "rw", SQLITE_OPEN_READWRITE }, - { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, - { "memory", SQLITE_OPEN_MEMORY }, - { 0, 0 } - }; - - mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE - | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; - aMode = aOpenMode; - limit = mask & flags; - zModeType = "access"; - } - - if( aMode ){ - int i; - int mode = 0; - for(i=0; aMode[i].z; i++){ - const char *z = aMode[i].z; - if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ - mode = aMode[i].mode; - break; - } - } - if( mode==0 ){ - *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); - rc = SQLITE_ERROR; - goto parse_uri_out; - } - if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ - *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", - zModeType, zVal); - rc = SQLITE_PERM; - goto parse_uri_out; - } - flags = (flags & ~mask) | mode; - } - } - - zOpt = &zVal[nVal+1]; - } - - }else{ - zFile = sqlite3_malloc64(nUri+2); - if( !zFile ) return SQLITE_NOMEM_BKPT; - memcpy(zFile, zUri, nUri); - zFile[nUri] = '\0'; - zFile[nUri+1] = '\0'; - flags &= ~SQLITE_OPEN_URI; - } - - *ppVfs = sqlite3_vfs_find(zVfs); - if( *ppVfs==0 ){ - *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); - rc = SQLITE_ERROR; - } - parse_uri_out: - if( rc!=SQLITE_OK ){ - sqlite3_free(zFile); - zFile = 0; - } - *pFlags = flags; - *pzFile = zFile; - return rc; -} - - -/* -** This routine does the work of opening a database on behalf of -** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" -** is UTF-8 encoded. -*/ -static int openDatabase( - const char *zFilename, /* Database filename UTF-8 encoded */ - sqlite3 **ppDb, /* OUT: Returned database handle */ - unsigned int flags, /* Operational flags */ - const char *zVfs /* Name of the VFS to use */ -){ - sqlite3 *db; /* Store allocated handle here */ - int rc; /* Return code */ - int isThreadsafe; /* True for threadsafe connections */ - char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ - char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ - -#ifdef SQLITE_ENABLE_API_ARMOR - if( ppDb==0 ) return SQLITE_MISUSE_BKPT; -#endif - *ppDb = 0; -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; -#endif - - /* Only allow sensible combinations of bits in the flags argument. - ** Throw an error if any non-sense combination is used. If we - ** do not block illegal combinations here, it could trigger - ** assert() statements in deeper layers. Sensible combinations - ** are: - ** - ** 1: SQLITE_OPEN_READONLY - ** 2: SQLITE_OPEN_READWRITE - ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE - */ - assert( SQLITE_OPEN_READONLY == 0x01 ); - assert( SQLITE_OPEN_READWRITE == 0x02 ); - assert( SQLITE_OPEN_CREATE == 0x04 ); - testcase( (1<<(flags&7))==0x02 ); /* READONLY */ - testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ - testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ - if( ((1<<(flags&7)) & 0x46)==0 ){ - return SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */ - } - - if( sqlite3GlobalConfig.bCoreMutex==0 ){ - isThreadsafe = 0; - }else if( flags & SQLITE_OPEN_NOMUTEX ){ - isThreadsafe = 0; - }else if( flags & SQLITE_OPEN_FULLMUTEX ){ - isThreadsafe = 1; - }else{ - isThreadsafe = sqlite3GlobalConfig.bFullMutex; - } - if( flags & SQLITE_OPEN_PRIVATECACHE ){ - flags &= ~SQLITE_OPEN_SHAREDCACHE; - }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ - flags |= SQLITE_OPEN_SHAREDCACHE; - } - - /* Remove harmful bits from the flags parameter - ** - ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were - ** dealt with in the previous code block. Besides these, the only - ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, - ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, - ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask - ** off all other flags. - */ - flags &= ~( SQLITE_OPEN_DELETEONCLOSE | - SQLITE_OPEN_EXCLUSIVE | - SQLITE_OPEN_MAIN_DB | - SQLITE_OPEN_TEMP_DB | - SQLITE_OPEN_TRANSIENT_DB | - SQLITE_OPEN_MAIN_JOURNAL | - SQLITE_OPEN_TEMP_JOURNAL | - SQLITE_OPEN_SUBJOURNAL | - SQLITE_OPEN_MASTER_JOURNAL | - SQLITE_OPEN_NOMUTEX | - SQLITE_OPEN_FULLMUTEX | - SQLITE_OPEN_WAL - ); - - /* Allocate the sqlite data structure */ - db = sqlite3MallocZero( sizeof(sqlite3) ); - if( db==0 ) goto opendb_out; - if( isThreadsafe ){ - db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); - if( db->mutex==0 ){ - sqlite3_free(db); - db = 0; - goto opendb_out; - } - } - sqlite3_mutex_enter(db->mutex); - db->errMask = 0xff; - db->nDb = 2; - db->magic = SQLITE_MAGIC_BUSY; - db->aDb = db->aDbStatic; - - assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); - memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); - db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; - db->autoCommit = 1; - db->nextAutovac = -1; - db->szMmap = sqlite3GlobalConfig.szMmap; - db->nextPagesize = 0; - db->nMaxSorterMmap = 0x7FFFFFFF; - db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill -#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX - | SQLITE_AutoIndex -#endif -#if SQLITE_DEFAULT_CKPTFULLFSYNC - | SQLITE_CkptFullFSync -#endif -#if SQLITE_DEFAULT_FILE_FORMAT<4 - | SQLITE_LegacyFileFmt -#endif -#ifdef SQLITE_ENABLE_LOAD_EXTENSION - | SQLITE_LoadExtension -#endif -#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS - | SQLITE_RecTriggers -#endif -#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS - | SQLITE_ForeignKeys -#endif -#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) - | SQLITE_ReverseOrder -#endif -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) - | SQLITE_CellSizeCk -#endif -#if defined(SQLITE_ENABLE_FTS3_TOKENIZER) - | SQLITE_Fts3Tokenizer -#endif - ; - sqlite3HashInit(&db->aCollSeq); -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3HashInit(&db->aModule); -#endif - - /* Add the default collation sequence BINARY. BINARY works for both UTF-8 - ** and UTF-16, so add a version for each to avoid any unnecessary - ** conversions. The only error that can occur here is a malloc() failure. - ** - ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating - ** functions: - */ - createCollation(db, sqlite3StrBINARY, SQLITE_UTF8, 0, binCollFunc, 0); - createCollation(db, sqlite3StrBINARY, SQLITE_UTF16BE, 0, binCollFunc, 0); - createCollation(db, sqlite3StrBINARY, SQLITE_UTF16LE, 0, binCollFunc, 0); - createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); - createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); - if( db->mallocFailed ){ - goto opendb_out; - } - /* EVIDENCE-OF: R-08308-17224 The default collating function for all - ** strings is BINARY. - */ - db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, sqlite3StrBINARY, 0); - assert( db->pDfltColl!=0 ); - - /* Parse the filename/URI argument. */ - db->openFlags = flags; - rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); - sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); - sqlite3_free(zErrMsg); - goto opendb_out; - } - - /* Open the backend database driver */ - rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, - flags | SQLITE_OPEN_MAIN_DB); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM_BKPT; - } - sqlite3Error(db, rc); - goto opendb_out; - } - sqlite3BtreeEnter(db->aDb[0].pBt); - db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); - if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db); - sqlite3BtreeLeave(db->aDb[0].pBt); - db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); - - /* The default safety_level for the main database is FULL; for the temp - ** database it is OFF. This matches the pager layer defaults. - */ - db->aDb[0].zDbSName = "main"; - db->aDb[0].safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; - db->aDb[1].zDbSName = "temp"; - db->aDb[1].safety_level = PAGER_SYNCHRONOUS_OFF; - - db->magic = SQLITE_MAGIC_OPEN; - if( db->mallocFailed ){ - goto opendb_out; - } - - /* Register all built-in functions, but do not attempt to read the - ** database schema yet. This is delayed until the first time the database - ** is accessed. - */ - sqlite3Error(db, SQLITE_OK); - sqlite3RegisterPerConnectionBuiltinFunctions(db); - rc = sqlite3_errcode(db); - -#ifdef SQLITE_ENABLE_FTS5 - /* Register any built-in FTS5 module before loading the automatic - ** extensions. This allows automatic extensions to register FTS5 - ** tokenizers and auxiliary functions. */ - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3Fts5Init(db); - } -#endif - - /* Load automatic extensions - extensions that have been registered - ** using the sqlite3_automatic_extension() API. - */ - if( rc==SQLITE_OK ){ - sqlite3AutoLoadExtensions(db); - rc = sqlite3_errcode(db); - if( rc!=SQLITE_OK ){ - goto opendb_out; - } - } - -#ifdef SQLITE_ENABLE_FTS1 - if( !db->mallocFailed ){ - extern int sqlite3Fts1Init(sqlite3*); - rc = sqlite3Fts1Init(db); - } -#endif - -#ifdef SQLITE_ENABLE_FTS2 - if( !db->mallocFailed && rc==SQLITE_OK ){ - extern int sqlite3Fts2Init(sqlite3*); - rc = sqlite3Fts2Init(db); - } -#endif - -#ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */ - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3Fts3Init(db); - } -#endif - -#ifdef SQLITE_ENABLE_ICU - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3IcuInit(db); - } -#endif - -#ifdef SQLITE_ENABLE_RTREE - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3RtreeInit(db); - } -#endif - -#ifdef SQLITE_ENABLE_DBSTAT_VTAB - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3DbstatRegister(db); - } -#endif - -#ifdef SQLITE_ENABLE_JSON1 - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3Json1Init(db); - } -#endif - - /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking - ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking - ** mode. Doing nothing at all also makes NORMAL the default. - */ -#ifdef SQLITE_DEFAULT_LOCKING_MODE - db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; - sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), - SQLITE_DEFAULT_LOCKING_MODE); -#endif - - if( rc ) sqlite3Error(db, rc); - - /* Enable the lookaside-malloc subsystem */ - setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, - sqlite3GlobalConfig.nLookaside); - - sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); - -opendb_out: - if( db ){ - assert( db->mutex!=0 || isThreadsafe==0 - || sqlite3GlobalConfig.bFullMutex==0 ); - sqlite3_mutex_leave(db->mutex); - } - rc = sqlite3_errcode(db); - assert( db!=0 || rc==SQLITE_NOMEM ); - if( rc==SQLITE_NOMEM ){ - sqlite3_close(db); - db = 0; - }else if( rc!=SQLITE_OK ){ - db->magic = SQLITE_MAGIC_SICK; - } - *ppDb = db; -#ifdef SQLITE_ENABLE_SQLLOG - if( sqlite3GlobalConfig.xSqllog ){ - /* Opening a db handle. Fourth parameter is passed 0. */ - void *pArg = sqlite3GlobalConfig.pSqllogArg; - sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); - } -#endif -#if defined(SQLITE_HAS_CODEC) - if( rc==SQLITE_OK ){ - const char *zHexKey = sqlite3_uri_parameter(zOpen, "hexkey"); - if( zHexKey && zHexKey[0] ){ - u8 iByte; - int i; - char zKey[40]; - for(i=0, iByte=0; imutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} - -#ifndef SQLITE_OMIT_UTF16 -/* -** Register a new collation sequence with the database handle db. -*/ -SQLITE_API int sqlite3_create_collation16( - sqlite3* db, - const void *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*) -){ - int rc = SQLITE_OK; - char *zName8; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); - if( zName8 ){ - rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); - sqlite3DbFree(db, zName8); - } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} -#endif /* SQLITE_OMIT_UTF16 */ - -/* -** Register a collation sequence factory callback with the database handle -** db. Replace any previously installed collation sequence factory. -*/ -SQLITE_API int sqlite3_collation_needed( - sqlite3 *db, - void *pCollNeededArg, - void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) -){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - db->xCollNeeded = xCollNeeded; - db->xCollNeeded16 = 0; - db->pCollNeededArg = pCollNeededArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} - -#ifndef SQLITE_OMIT_UTF16 -/* -** Register a collation sequence factory callback with the database handle -** db. Replace any previously installed collation sequence factory. -*/ -SQLITE_API int sqlite3_collation_needed16( - sqlite3 *db, - void *pCollNeededArg, - void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) -){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - db->xCollNeeded = 0; - db->xCollNeeded16 = xCollNeeded16; - db->pCollNeededArg = pCollNeededArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} -#endif /* SQLITE_OMIT_UTF16 */ - -#ifndef SQLITE_OMIT_DEPRECATED -/* -** This function is now an anachronism. It used to be used to recover from a -** malloc() failure, but SQLite now does this automatically. -*/ -SQLITE_API int sqlite3_global_recover(void){ - return SQLITE_OK; -} -#endif - -/* -** Test to see whether or not the database connection is in autocommit -** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on -** by default. Autocommit is disabled by a BEGIN statement and reenabled -** by the next COMMIT or ROLLBACK. -*/ -SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - return db->autoCommit; -} - -/* -** The following routines are substitutes for constants SQLITE_CORRUPT, -** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_NOMEM and possibly other error -** constants. They serve two purposes: -** -** 1. Serve as a convenient place to set a breakpoint in a debugger -** to detect when version error conditions occurs. -** -** 2. Invoke sqlite3_log() to provide the source code location where -** a low-level error is first detected. -*/ -static int reportError(int iErr, int lineno, const char *zType){ - sqlite3_log(iErr, "%s at line %d of [%.10s]", - zType, lineno, 20+sqlite3_sourceid()); - return iErr; -} -SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_CORRUPT, lineno, "database corruption"); -} -SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_MISUSE, lineno, "misuse"); -} -SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_CANTOPEN, lineno, "cannot open file"); -} -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3NomemError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_NOMEM, lineno, "OOM"); -} -SQLITE_PRIVATE int sqlite3IoerrnomemError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); -} -#endif - -#ifndef SQLITE_OMIT_DEPRECATED -/* -** This is a convenience routine that makes sure that all thread-specific -** data for this thread has been deallocated. -** -** SQLite no longer uses thread-specific data so this routine is now a -** no-op. It is retained for historical compatibility. -*/ -SQLITE_API void sqlite3_thread_cleanup(void){ -} -#endif - -/* -** Return meta information about a specific column of a database table. -** See comment in sqlite3.h (sqlite.h.in) for details. -*/ -SQLITE_API int sqlite3_table_column_metadata( - sqlite3 *db, /* Connection handle */ - const char *zDbName, /* Database name or NULL */ - const char *zTableName, /* Table name */ - const char *zColumnName, /* Column name */ - char const **pzDataType, /* OUTPUT: Declared data type */ - char const **pzCollSeq, /* OUTPUT: Collation sequence name */ - int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ - int *pPrimaryKey, /* OUTPUT: True if column part of PK */ - int *pAutoinc /* OUTPUT: True if column is auto-increment */ -){ - int rc; - char *zErrMsg = 0; - Table *pTab = 0; - Column *pCol = 0; - int iCol = 0; - char const *zDataType = 0; - char const *zCollSeq = 0; - int notnull = 0; - int primarykey = 0; - int autoinc = 0; - - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zTableName==0 ){ - return SQLITE_MISUSE_BKPT; - } -#endif - - /* Ensure the database schema has been loaded */ - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - rc = sqlite3Init(db, &zErrMsg); - if( SQLITE_OK!=rc ){ - goto error_out; - } - - /* Locate the table in question */ - pTab = sqlite3FindTable(db, zTableName, zDbName); - if( !pTab || pTab->pSelect ){ - pTab = 0; - goto error_out; - } - - /* Find the column for which info is requested */ - if( zColumnName==0 ){ - /* Query for existance of table only */ - }else{ - for(iCol=0; iColnCol; iCol++){ - pCol = &pTab->aCol[iCol]; - if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ - break; - } - } - if( iCol==pTab->nCol ){ - if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ - iCol = pTab->iPKey; - pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; - }else{ - pTab = 0; - goto error_out; - } - } - } - - /* The following block stores the meta information that will be returned - ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey - ** and autoinc. At this point there are two possibilities: - ** - ** 1. The specified column name was rowid", "oid" or "_rowid_" - ** and there is no explicitly declared IPK column. - ** - ** 2. The table is not a view and the column name identified an - ** explicitly declared column. Copy meta information from *pCol. - */ - if( pCol ){ - zDataType = sqlite3ColumnType(pCol,0); - zCollSeq = pCol->zColl; - notnull = pCol->notNull!=0; - primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; - autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; - }else{ - zDataType = "INTEGER"; - primarykey = 1; - } - if( !zCollSeq ){ - zCollSeq = sqlite3StrBINARY; - } - -error_out: - sqlite3BtreeLeaveAll(db); - - /* Whether the function call succeeded or failed, set the output parameters - ** to whatever their local counterparts contain. If an error did occur, - ** this has the effect of zeroing all output parameters. - */ - if( pzDataType ) *pzDataType = zDataType; - if( pzCollSeq ) *pzCollSeq = zCollSeq; - if( pNotNull ) *pNotNull = notnull; - if( pPrimaryKey ) *pPrimaryKey = primarykey; - if( pAutoinc ) *pAutoinc = autoinc; - - if( SQLITE_OK==rc && !pTab ){ - sqlite3DbFree(db, zErrMsg); - zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, - zColumnName); - rc = SQLITE_ERROR; - } - sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); - sqlite3DbFree(db, zErrMsg); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} - -/* -** Sleep for a little while. Return the amount of time slept. -*/ -SQLITE_API int sqlite3_sleep(int ms){ - sqlite3_vfs *pVfs; - int rc; - pVfs = sqlite3_vfs_find(0); - if( pVfs==0 ) return 0; - - /* This function works in milliseconds, but the underlying OsSleep() - ** API uses microseconds. Hence the 1000's. - */ - rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); - return rc; -} - -/* -** Enable or disable the extended result codes. -*/ -SQLITE_API int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - db->errMask = onoff ? 0xffffffff : 0xff; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} - -/* -** Invoke the xFileControl method on a particular database. -*/ -SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ - int rc = SQLITE_ERROR; - Btree *pBtree; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - pBtree = sqlite3DbNameToBtree(db, zDbName); - if( pBtree ){ - Pager *pPager; - sqlite3_file *fd; - sqlite3BtreeEnter(pBtree); - pPager = sqlite3BtreePager(pBtree); - assert( pPager!=0 ); - fd = sqlite3PagerFile(pPager); - assert( fd!=0 ); - if( op==SQLITE_FCNTL_FILE_POINTER ){ - *(sqlite3_file**)pArg = fd; - rc = SQLITE_OK; - }else if( op==SQLITE_FCNTL_VFS_POINTER ){ - *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); - rc = SQLITE_OK; - }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ - *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); - rc = SQLITE_OK; - }else if( fd->pMethods ){ - rc = sqlite3OsFileControl(fd, op, pArg); - }else{ - rc = SQLITE_NOTFOUND; - } - sqlite3BtreeLeave(pBtree); - } - sqlite3_mutex_leave(db->mutex); - return rc; -} - -/* -** Interface to the testing logic. -*/ -SQLITE_API int sqlite3_test_control(int op, ...){ - int rc = 0; -#ifdef SQLITE_OMIT_BUILTIN_TEST - UNUSED_PARAMETER(op); -#else - va_list ap; - va_start(ap, op); - switch( op ){ - - /* - ** Save the current state of the PRNG. - */ - case SQLITE_TESTCTRL_PRNG_SAVE: { - sqlite3PrngSaveState(); - break; - } - - /* - ** Restore the state of the PRNG to the last state saved using - ** PRNG_SAVE. If PRNG_SAVE has never before been called, then - ** this verb acts like PRNG_RESET. - */ - case SQLITE_TESTCTRL_PRNG_RESTORE: { - sqlite3PrngRestoreState(); - break; - } - - /* - ** Reset the PRNG back to its uninitialized state. The next call - ** to sqlite3_randomness() will reseed the PRNG using a single call - ** to the xRandomness method of the default VFS. - */ - case SQLITE_TESTCTRL_PRNG_RESET: { - sqlite3_randomness(0,0); - break; - } - - /* - ** sqlite3_test_control(BITVEC_TEST, size, program) - ** - ** Run a test against a Bitvec object of size. The program argument - ** is an array of integers that defines the test. Return -1 on a - ** memory allocation error, 0 on success, or non-zero for an error. - ** See the sqlite3BitvecBuiltinTest() for additional information. - */ - case SQLITE_TESTCTRL_BITVEC_TEST: { - int sz = va_arg(ap, int); - int *aProg = va_arg(ap, int*); - rc = sqlite3BitvecBuiltinTest(sz, aProg); - break; - } - - /* - ** sqlite3_test_control(FAULT_INSTALL, xCallback) - ** - ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, - ** if xCallback is not NULL. - ** - ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) - ** is called immediately after installing the new callback and the return - ** value from sqlite3FaultSim(0) becomes the return from - ** sqlite3_test_control(). - */ - case SQLITE_TESTCTRL_FAULT_INSTALL: { - /* MSVC is picky about pulling func ptrs from va lists. - ** http://support.microsoft.com/kb/47961 - ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); - */ - typedef int(*TESTCALLBACKFUNC_t)(int); - sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); - rc = sqlite3FaultSim(0); - break; - } - - /* - ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) - ** - ** Register hooks to call to indicate which malloc() failures - ** are benign. - */ - case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { - typedef void (*void_function)(void); - void_function xBenignBegin; - void_function xBenignEnd; - xBenignBegin = va_arg(ap, void_function); - xBenignEnd = va_arg(ap, void_function); - sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); - break; - } - - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) - ** - ** Set the PENDING byte to the value in the argument, if X>0. - ** Make no changes if X==0. Return the value of the pending byte - ** as it existing before this routine was called. - ** - ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in - ** an incompatible database file format. Changing the PENDING byte - ** while any database connection is open results in undefined and - ** deleterious behavior. - */ - case SQLITE_TESTCTRL_PENDING_BYTE: { - rc = PENDING_BYTE; -#ifndef SQLITE_OMIT_WSD - { - unsigned int newVal = va_arg(ap, unsigned int); - if( newVal ) sqlite3PendingByte = newVal; - } -#endif - break; - } - - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) - ** - ** This action provides a run-time test to see whether or not - ** assert() was enabled at compile-time. If X is true and assert() - ** is enabled, then the return value is true. If X is true and - ** assert() is disabled, then the return value is zero. If X is - ** false and assert() is enabled, then the assertion fires and the - ** process aborts. If X is false and assert() is disabled, then the - ** return value is zero. - */ - case SQLITE_TESTCTRL_ASSERT: { - volatile int x = 0; - assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 ); - rc = x; - break; - } - - - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) - ** - ** This action provides a run-time test to see how the ALWAYS and - ** NEVER macros were defined at compile-time. - ** - ** The return value is ALWAYS(X). - ** - ** The recommended test is X==2. If the return value is 2, that means - ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the - ** default setting. If the return value is 1, then ALWAYS() is either - ** hard-coded to true or else it asserts if its argument is false. - ** The first behavior (hard-coded to true) is the case if - ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second - ** behavior (assert if the argument to ALWAYS() is false) is the case if - ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. - ** - ** The run-time test procedure might look something like this: - ** - ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ - ** // ALWAYS() and NEVER() are no-op pass-through macros - ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ - ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. - ** }else{ - ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. - ** } - */ - case SQLITE_TESTCTRL_ALWAYS: { - int x = va_arg(ap,int); - rc = ALWAYS(x); - break; - } - - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); - ** - ** The integer returned reveals the byte-order of the computer on which - ** SQLite is running: - ** - ** 1 big-endian, determined at run-time - ** 10 little-endian, determined at run-time - ** 432101 big-endian, determined at compile-time - ** 123410 little-endian, determined at compile-time - */ - case SQLITE_TESTCTRL_BYTEORDER: { - rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; - break; - } - - /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) - ** - ** Set the nReserve size to N for the main database on the database - ** connection db. - */ - case SQLITE_TESTCTRL_RESERVE: { - sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); - sqlite3_mutex_leave(db->mutex); - break; - } - - /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) - ** - ** Enable or disable various optimizations for testing purposes. The - ** argument N is a bitmask of optimizations to be disabled. For normal - ** operation N should be 0. The idea is that a test program (like the - ** SQL Logic Test or SLT test module) can run the same SQL multiple times - ** with various optimizations disabled to verify that the same answer - ** is obtained in every case. - */ - case SQLITE_TESTCTRL_OPTIMIZATIONS: { - sqlite3 *db = va_arg(ap, sqlite3*); - db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); - break; - } - -#ifdef SQLITE_N_KEYWORD - /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) - ** - ** If zWord is a keyword recognized by the parser, then return the - ** number of keywords. Or if zWord is not a keyword, return 0. - ** - ** This test feature is only available in the amalgamation since - ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite - ** is built using separate source files. - */ - case SQLITE_TESTCTRL_ISKEYWORD: { - const char *zWord = va_arg(ap, const char*); - int n = sqlite3Strlen30(zWord); - rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; - break; - } -#endif - - /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); - ** - ** Pass pFree into sqlite3ScratchFree(). - ** If sz>0 then allocate a scratch buffer into pNew. - */ - case SQLITE_TESTCTRL_SCRATCHMALLOC: { - void *pFree, **ppNew; - int sz; - sz = va_arg(ap, int); - ppNew = va_arg(ap, void**); - pFree = va_arg(ap, void*); - if( sz ) *ppNew = sqlite3ScratchMalloc(sz); - sqlite3ScratchFree(pFree); - break; - } - - /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); - ** - ** If parameter onoff is non-zero, configure the wrappers so that all - ** subsequent calls to localtime() and variants fail. If onoff is zero, - ** undo this setting. - */ - case SQLITE_TESTCTRL_LOCALTIME_FAULT: { - sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); - break; - } - - /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); - ** - ** Set or clear a flag that indicates that the database file is always well- - ** formed and never corrupt. This flag is clear by default, indicating that - ** database files might have arbitrary corruption. Setting the flag during - ** testing causes certain assert() statements in the code to be activated - ** that demonstrat invariants on well-formed database files. - */ - case SQLITE_TESTCTRL_NEVER_CORRUPT: { - sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); - break; - } - - /* Set the threshold at which OP_Once counters reset back to zero. - ** By default this is 0x7ffffffe (over 2 billion), but that value is - ** too big to test in a reasonable amount of time, so this control is - ** provided to set a small and easily reachable reset value. - */ - case SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD: { - sqlite3GlobalConfig.iOnceResetThreshold = va_arg(ap, int); - break; - } - - /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); - ** - ** Set the VDBE coverage callback function to xCallback with context - ** pointer ptr. - */ - case SQLITE_TESTCTRL_VDBE_COVERAGE: { -#ifdef SQLITE_VDBE_COVERAGE - typedef void (*branch_callback)(void*,int,u8,u8); - sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); - sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); -#endif - break; - } - - /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ - case SQLITE_TESTCTRL_SORTER_MMAP: { - sqlite3 *db = va_arg(ap, sqlite3*); - db->nMaxSorterMmap = va_arg(ap, int); - break; - } - - /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); - ** - ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if - ** not. - */ - case SQLITE_TESTCTRL_ISINIT: { - if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; - break; - } - - /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, onOff, tnum); - ** - ** This test control is used to create imposter tables. "db" is a pointer - ** to the database connection. dbName is the database name (ex: "main" or - ** "temp") which will receive the imposter. "onOff" turns imposter mode on - ** or off. "tnum" is the root page of the b-tree to which the imposter - ** table should connect. - ** - ** Enable imposter mode only when the schema has already been parsed. Then - ** run a single CREATE TABLE statement to construct the imposter table in - ** the parsed schema. Then turn imposter mode back off again. - ** - ** If onOff==0 and tnum>0 then reset the schema for all databases, causing - ** the schema to be reparsed the next time it is needed. This has the - ** effect of erasing all imposter tables. - */ - case SQLITE_TESTCTRL_IMPOSTER: { - sqlite3 *db = va_arg(ap, sqlite3*); - sqlite3_mutex_enter(db->mutex); - db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); - db->init.busy = db->init.imposterTable = va_arg(ap,int); - db->init.newTnum = va_arg(ap,int); - if( db->init.busy==0 && db->init.newTnum>0 ){ - sqlite3ResetAllSchemasOfConnection(db); - } - sqlite3_mutex_leave(db->mutex); - break; - } - } - va_end(ap); -#endif /* SQLITE_OMIT_BUILTIN_TEST */ - return rc; -} - -/* -** This is a utility routine, useful to VFS implementations, that checks -** to see if a database file was a URI that contained a specific query -** parameter, and if so obtains the value of the query parameter. -** -** The zFilename argument is the filename pointer passed into the xOpen() -** method of a VFS implementation. The zParam argument is the name of the -** query parameter we seek. This routine returns the value of the zParam -** parameter if it exists. If the parameter does not exist, this routine -** returns a NULL pointer. -*/ -SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ - if( zFilename==0 || zParam==0 ) return 0; - zFilename += sqlite3Strlen30(zFilename) + 1; - while( zFilename[0] ){ - int x = strcmp(zFilename, zParam); - zFilename += sqlite3Strlen30(zFilename) + 1; - if( x==0 ) return zFilename; - zFilename += sqlite3Strlen30(zFilename) + 1; - } - return 0; -} - -/* -** Return a boolean value for a query parameter. -*/ -SQLITE_API int sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ - const char *z = sqlite3_uri_parameter(zFilename, zParam); - bDflt = bDflt!=0; - return z ? sqlite3GetBoolean(z, bDflt) : bDflt; -} - -/* -** Return a 64-bit integer value for a query parameter. -*/ -SQLITE_API sqlite3_int64 sqlite3_uri_int64( - const char *zFilename, /* Filename as passed to xOpen */ - const char *zParam, /* URI parameter sought */ - sqlite3_int64 bDflt /* return if parameter is missing */ -){ - const char *z = sqlite3_uri_parameter(zFilename, zParam); - sqlite3_int64 v; - if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ - bDflt = v; - } - return bDflt; -} - -/* -** Return the Btree pointer identified by zDbName. Return NULL if not found. -*/ -SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ - int i; - for(i=0; inDb; i++){ - if( db->aDb[i].pBt - && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zDbSName)==0) - ){ - return db->aDb[i].pBt; - } - } - return 0; -} - -/* -** Return the filename of the database associated with a database -** connection. -*/ -SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ - Btree *pBt; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - pBt = sqlite3DbNameToBtree(db, zDbName); - return pBt ? sqlite3BtreeGetFilename(pBt) : 0; -} - -/* -** Return 1 if database is read-only or 0 if read/write. Return -1 if -** no such database exists. -*/ -SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ - Btree *pBt; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return -1; - } -#endif - pBt = sqlite3DbNameToBtree(db, zDbName); - return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; -} - -#ifdef SQLITE_ENABLE_SNAPSHOT -/* -** Obtain a snapshot handle for the snapshot of database zDb currently -** being read by handle db. -*/ -SQLITE_API int sqlite3_snapshot_get( - sqlite3 *db, - const char *zDb, - sqlite3_snapshot **ppSnapshot -){ - int rc = SQLITE_ERROR; -#ifndef SQLITE_OMIT_WAL - int iDb; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE_BKPT; - } -#endif - sqlite3_mutex_enter(db->mutex); - - iDb = sqlite3FindDbName(db, zDb); - if( iDb==0 || iDb>1 ){ - Btree *pBt = db->aDb[iDb].pBt; - if( 0==sqlite3BtreeIsInTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); - } - } - } - - sqlite3_mutex_leave(db->mutex); -#endif /* SQLITE_OMIT_WAL */ - return rc; -} - -/* -** Open a read-transaction on the snapshot idendified by pSnapshot. -*/ -SQLITE_API int sqlite3_snapshot_open( - sqlite3 *db, - const char *zDb, - sqlite3_snapshot *pSnapshot -){ - int rc = SQLITE_ERROR; -#ifndef SQLITE_OMIT_WAL - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE_BKPT; - } -#endif - sqlite3_mutex_enter(db->mutex); - if( db->autoCommit==0 ){ - int iDb; - iDb = sqlite3FindDbName(db, zDb); - if( iDb==0 || iDb>1 ){ - Btree *pBt = db->aDb[iDb].pBt; - if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ - rc = sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), pSnapshot); - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); - sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), 0); - } - } - } - } - - sqlite3_mutex_leave(db->mutex); -#endif /* SQLITE_OMIT_WAL */ - return rc; -} - -/* -** Free a snapshot handle obtained from sqlite3_snapshot_get(). -*/ -SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){ - sqlite3_free(pSnapshot); -} -#endif /* SQLITE_ENABLE_SNAPSHOT */ - -/************** End of main.c ************************************************/ -/************** Begin file notify.c ******************************************/ -/* -** 2009 March 3 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains the implementation of the sqlite3_unlock_notify() -** API method and its associated functionality. -*/ -/* #include "sqliteInt.h" */ -/* #include "btreeInt.h" */ - -/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */ -#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY - -/* -** Public interfaces: -** -** sqlite3ConnectionBlocked() -** sqlite3ConnectionUnlocked() -** sqlite3ConnectionClosed() -** sqlite3_unlock_notify() -*/ - -#define assertMutexHeld() \ - assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ) - -/* -** Head of a linked list of all sqlite3 objects created by this process -** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection -** is not NULL. This variable may only accessed while the STATIC_MASTER -** mutex is held. -*/ -static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; - -#ifndef NDEBUG -/* -** This function is a complex assert() that verifies the following -** properties of the blocked connections list: -** -** 1) Each entry in the list has a non-NULL value for either -** pUnlockConnection or pBlockingConnection, or both. -** -** 2) All entries in the list that share a common value for -** xUnlockNotify are grouped together. -** -** 3) If the argument db is not NULL, then none of the entries in the -** blocked connections list have pUnlockConnection or pBlockingConnection -** set to db. This is used when closing connection db. -*/ -static void checkListProperties(sqlite3 *db){ - sqlite3 *p; - for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ - int seen = 0; - sqlite3 *p2; - - /* Verify property (1) */ - assert( p->pUnlockConnection || p->pBlockingConnection ); - - /* Verify property (2) */ - for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ - if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; - assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); - assert( db==0 || p->pUnlockConnection!=db ); - assert( db==0 || p->pBlockingConnection!=db ); - } - } -} -#else -# define checkListProperties(x) -#endif - -/* -** Remove connection db from the blocked connections list. If connection -** db is not currently a part of the list, this function is a no-op. -*/ -static void removeFromBlockedList(sqlite3 *db){ - sqlite3 **pp; - assertMutexHeld(); - for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ - if( *pp==db ){ - *pp = (*pp)->pNextBlocked; - break; - } - } -} - -/* -** Add connection db to the blocked connections list. It is assumed -** that it is not already a part of the list. -*/ -static void addToBlockedList(sqlite3 *db){ - sqlite3 **pp; - assertMutexHeld(); - for( - pp=&sqlite3BlockedList; - *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; - pp=&(*pp)->pNextBlocked - ); - db->pNextBlocked = *pp; - *pp = db; -} - -/* -** Obtain the STATIC_MASTER mutex. -*/ -static void enterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - checkListProperties(0); -} - -/* -** Release the STATIC_MASTER mutex. -*/ -static void leaveMutex(void){ - assertMutexHeld(); - checkListProperties(0); - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); -} - -/* -** Register an unlock-notify callback. -** -** This is called after connection "db" has attempted some operation -** but has received an SQLITE_LOCKED error because another connection -** (call it pOther) in the same process was busy using the same shared -** cache. pOther is found by looking at db->pBlockingConnection. -** -** If there is no blocking connection, the callback is invoked immediately, -** before this routine returns. -** -** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate -** a deadlock. -** -** Otherwise, make arrangements to invoke xNotify when pOther drops -** its locks. -** -** Each call to this routine overrides any prior callbacks registered -** on the same "db". If xNotify==0 then any prior callbacks are immediately -** cancelled. -*/ -SQLITE_API int sqlite3_unlock_notify( - sqlite3 *db, - void (*xNotify)(void **, int), - void *pArg -){ - int rc = SQLITE_OK; - - sqlite3_mutex_enter(db->mutex); - enterMutex(); - - if( xNotify==0 ){ - removeFromBlockedList(db); - db->pBlockingConnection = 0; - db->pUnlockConnection = 0; - db->xUnlockNotify = 0; - db->pUnlockArg = 0; - }else if( 0==db->pBlockingConnection ){ - /* The blocking transaction has been concluded. Or there never was a - ** blocking transaction. In either case, invoke the notify callback - ** immediately. - */ - xNotify(&pArg, 1); - }else{ - sqlite3 *p; - - for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} - if( p ){ - rc = SQLITE_LOCKED; /* Deadlock detected. */ - }else{ - db->pUnlockConnection = db->pBlockingConnection; - db->xUnlockNotify = xNotify; - db->pUnlockArg = pArg; - removeFromBlockedList(db); - addToBlockedList(db); - } - } - - leaveMutex(); - assert( !db->mallocFailed ); - sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); - sqlite3_mutex_leave(db->mutex); - return rc; -} - -/* -** This function is called while stepping or preparing a statement -** associated with connection db. The operation will return SQLITE_LOCKED -** to the user because it requires a lock that will not be available -** until connection pBlocker concludes its current transaction. -*/ -SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ - enterMutex(); - if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ - addToBlockedList(db); - } - db->pBlockingConnection = pBlocker; - leaveMutex(); -} - -/* -** This function is called when -** the transaction opened by database db has just finished. Locks held -** by database connection db have been released. -** -** This function loops through each entry in the blocked connections -** list and does the following: -** -** 1) If the sqlite3.pBlockingConnection member of a list entry is -** set to db, then set pBlockingConnection=0. -** -** 2) If the sqlite3.pUnlockConnection member of a list entry is -** set to db, then invoke the configured unlock-notify callback and -** set pUnlockConnection=0. -** -** 3) If the two steps above mean that pBlockingConnection==0 and -** pUnlockConnection==0, remove the entry from the blocked connections -** list. -*/ -SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ - void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ - int nArg = 0; /* Number of entries in aArg[] */ - sqlite3 **pp; /* Iterator variable */ - void **aArg; /* Arguments to the unlock callback */ - void **aDyn = 0; /* Dynamically allocated space for aArg[] */ - void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ - - aArg = aStatic; - enterMutex(); /* Enter STATIC_MASTER mutex */ - - /* This loop runs once for each entry in the blocked-connections list. */ - for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ - sqlite3 *p = *pp; - - /* Step 1. */ - if( p->pBlockingConnection==db ){ - p->pBlockingConnection = 0; - } - - /* Step 2. */ - if( p->pUnlockConnection==db ){ - assert( p->xUnlockNotify ); - if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ - xUnlockNotify(aArg, nArg); - nArg = 0; - } - - sqlite3BeginBenignMalloc(); - assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); - assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); - if( (!aDyn && nArg==(int)ArraySize(aStatic)) - || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) - ){ - /* The aArg[] array needs to grow. */ - void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); - if( pNew ){ - memcpy(pNew, aArg, nArg*sizeof(void *)); - sqlite3_free(aDyn); - aDyn = aArg = pNew; - }else{ - /* This occurs when the array of context pointers that need to - ** be passed to the unlock-notify callback is larger than the - ** aStatic[] array allocated on the stack and the attempt to - ** allocate a larger array from the heap has failed. - ** - ** This is a difficult situation to handle. Returning an error - ** code to the caller is insufficient, as even if an error code - ** is returned the transaction on connection db will still be - ** closed and the unlock-notify callbacks on blocked connections - ** will go unissued. This might cause the application to wait - ** indefinitely for an unlock-notify callback that will never - ** arrive. - ** - ** Instead, invoke the unlock-notify callback with the context - ** array already accumulated. We can then clear the array and - ** begin accumulating any further context pointers without - ** requiring any dynamic allocation. This is sub-optimal because - ** it means that instead of one callback with a large array of - ** context pointers the application will receive two or more - ** callbacks with smaller arrays of context pointers, which will - ** reduce the applications ability to prioritize multiple - ** connections. But it is the best that can be done under the - ** circumstances. - */ - xUnlockNotify(aArg, nArg); - nArg = 0; - } - } - sqlite3EndBenignMalloc(); - - aArg[nArg++] = p->pUnlockArg; - xUnlockNotify = p->xUnlockNotify; - p->pUnlockConnection = 0; - p->xUnlockNotify = 0; - p->pUnlockArg = 0; - } - - /* Step 3. */ - if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ - /* Remove connection p from the blocked connections list. */ - *pp = p->pNextBlocked; - p->pNextBlocked = 0; - }else{ - pp = &p->pNextBlocked; - } - } - - if( nArg!=0 ){ - xUnlockNotify(aArg, nArg); - } - sqlite3_free(aDyn); - leaveMutex(); /* Leave STATIC_MASTER mutex */ -} - -/* -** This is called when the database connection passed as an argument is -** being closed. The connection is removed from the blocked list. -*/ -SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ - sqlite3ConnectionUnlocked(db); - enterMutex(); - removeFromBlockedList(db); - checkListProperties(db); - leaveMutex(); -} -#endif - -/************** End of notify.c **********************************************/ -/************** Begin file fts3.c ********************************************/ -/* -** 2006 Oct 10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This is an SQLite module implementing full-text search. -*/ - -/* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or -** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). -*/ - -/* The full-text index is stored in a series of b+tree (-like) -** structures called segments which map terms to doclists. The -** structures are like b+trees in layout, but are constructed from the -** bottom up in optimal fashion and are not updatable. Since trees -** are built from the bottom up, things will be described from the -** bottom up. -** -** -**** Varints **** -** The basic unit of encoding is a variable-length integer called a -** varint. We encode variable-length integers in little-endian order -** using seven bits * per byte as follows: -** -** KEY: -** A = 0xxxxxxx 7 bits of data and one flag bit -** B = 1xxxxxxx 7 bits of data and one flag bit -** -** 7 bits - A -** 14 bits - BA -** 21 bits - BBA -** and so on. -** -** This is similar in concept to how sqlite encodes "varints" but -** the encoding is not the same. SQLite varints are big-endian -** are are limited to 9 bytes in length whereas FTS3 varints are -** little-endian and can be up to 10 bytes in length (in theory). -** -** Example encodings: -** -** 1: 0x01 -** 127: 0x7f -** 128: 0x81 0x00 -** -** -**** Document lists **** -** A doclist (document list) holds a docid-sorted list of hits for a -** given term. Doclists hold docids and associated token positions. -** A docid is the unique integer identifier for a single document. -** A position is the index of a word within the document. The first -** word of the document has a position of 0. -** -** FTS3 used to optionally store character offsets using a compile-time -** option. But that functionality is no longer supported. -** -** A doclist is stored like this: -** -** array { -** varint docid; (delta from previous doclist) -** array { (position list for column 0) -** varint position; (2 more than the delta from previous position) -** } -** array { -** varint POS_COLUMN; (marks start of position list for new column) -** varint column; (index of new column) -** array { -** varint position; (2 more than the delta from previous position) -** } -** } -** varint POS_END; (marks end of positions for this document. -** } -** -** Here, array { X } means zero or more occurrences of X, adjacent in -** memory. A "position" is an index of a token in the token stream -** generated by the tokenizer. Note that POS_END and POS_COLUMN occur -** in the same logical place as the position element, and act as sentinals -** ending a position list array. POS_END is 0. POS_COLUMN is 1. -** The positions numbers are not stored literally but rather as two more -** than the difference from the prior position, or the just the position plus -** 2 for the first position. Example: -** -** label: A B C D E F G H I J K -** value: 123 5 9 1 1 14 35 0 234 72 0 -** -** The 123 value is the first docid. For column zero in this document -** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 -** at D signals the start of a new column; the 1 at E indicates that the -** new column is column number 1. There are two positions at 12 and 45 -** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The -** 234 at I is the delta to next docid (357). It has one position 70 -** (72-2) and then terminates with the 0 at K. -** -** A "position-list" is the list of positions for multiple columns for -** a single docid. A "column-list" is the set of positions for a single -** column. Hence, a position-list consists of one or more column-lists, -** a document record consists of a docid followed by a position-list and -** a doclist consists of one or more document records. -** -** A bare doclist omits the position information, becoming an -** array of varint-encoded docids. -** -**** Segment leaf nodes **** -** Segment leaf nodes store terms and doclists, ordered by term. Leaf -** nodes are written using LeafWriter, and read using LeafReader (to -** iterate through a single leaf node's data) and LeavesReader (to -** iterate through a segment's entire leaf layer). Leaf nodes have -** the format: -** -** varint iHeight; (height from leaf level, always 0) -** varint nTerm; (length of first term) -** char pTerm[nTerm]; (content of first term) -** varint nDoclist; (length of term's associated doclist) -** char pDoclist[nDoclist]; (content of doclist) -** array { -** (further terms are delta-encoded) -** varint nPrefix; (length of prefix shared with previous term) -** varint nSuffix; (length of unshared suffix) -** char pTermSuffix[nSuffix];(unshared suffix of next term) -** varint nDoclist; (length of term's associated doclist) -** char pDoclist[nDoclist]; (content of doclist) -** } -** -** Here, array { X } means zero or more occurrences of X, adjacent in -** memory. -** -** Leaf nodes are broken into blocks which are stored contiguously in -** the %_segments table in sorted order. This means that when the end -** of a node is reached, the next term is in the node with the next -** greater node id. -** -** New data is spilled to a new leaf node when the current node -** exceeds LEAF_MAX bytes (default 2048). New data which itself is -** larger than STANDALONE_MIN (default 1024) is placed in a standalone -** node (a leaf node with a single term and doclist). The goal of -** these settings is to pack together groups of small doclists while -** making it efficient to directly access large doclists. The -** assumption is that large doclists represent terms which are more -** likely to be query targets. -** -** TODO(shess) It may be useful for blocking decisions to be more -** dynamic. For instance, it may make more sense to have a 2.5k leaf -** node rather than splitting into 2k and .5k nodes. My intuition is -** that this might extend through 2x or 4x the pagesize. -** -** -**** Segment interior nodes **** -** Segment interior nodes store blockids for subtree nodes and terms -** to describe what data is stored by the each subtree. Interior -** nodes are written using InteriorWriter, and read using -** InteriorReader. InteriorWriters are created as needed when -** SegmentWriter creates new leaf nodes, or when an interior node -** itself grows too big and must be split. The format of interior -** nodes: -** -** varint iHeight; (height from leaf level, always >0) -** varint iBlockid; (block id of node's leftmost subtree) -** optional { -** varint nTerm; (length of first term) -** char pTerm[nTerm]; (content of first term) -** array { -** (further terms are delta-encoded) -** varint nPrefix; (length of shared prefix with previous term) -** varint nSuffix; (length of unshared suffix) -** char pTermSuffix[nSuffix]; (unshared suffix of next term) -** } -** } -** -** Here, optional { X } means an optional element, while array { X } -** means zero or more occurrences of X, adjacent in memory. -** -** An interior node encodes n terms separating n+1 subtrees. The -** subtree blocks are contiguous, so only the first subtree's blockid -** is encoded. The subtree at iBlockid will contain all terms less -** than the first term encoded (or all terms if no term is encoded). -** Otherwise, for terms greater than or equal to pTerm[i] but less -** than pTerm[i+1], the subtree for that term will be rooted at -** iBlockid+i. Interior nodes only store enough term data to -** distinguish adjacent children (if the rightmost term of the left -** child is "something", and the leftmost term of the right child is -** "wicked", only "w" is stored). -** -** New data is spilled to a new interior node at the same height when -** the current node exceeds INTERIOR_MAX bytes (default 2048). -** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing -** interior nodes and making the tree too skinny. The interior nodes -** at a given height are naturally tracked by interior nodes at -** height+1, and so on. -** -** -**** Segment directory **** -** The segment directory in table %_segdir stores meta-information for -** merging and deleting segments, and also the root node of the -** segment's tree. -** -** The root node is the top node of the segment's tree after encoding -** the entire segment, restricted to ROOT_MAX bytes (default 1024). -** This could be either a leaf node or an interior node. If the top -** node requires more than ROOT_MAX bytes, it is flushed to %_segments -** and a new root interior node is generated (which should always fit -** within ROOT_MAX because it only needs space for 2 varints, the -** height and the blockid of the previous root). -** -** The meta-information in the segment directory is: -** level - segment level (see below) -** idx - index within level -** - (level,idx uniquely identify a segment) -** start_block - first leaf node -** leaves_end_block - last leaf node -** end_block - last block (including interior nodes) -** root - contents of root node -** -** If the root node is a leaf node, then start_block, -** leaves_end_block, and end_block are all 0. -** -** -**** Segment merging **** -** To amortize update costs, segments are grouped into levels and -** merged in batches. Each increase in level represents exponentially -** more documents. -** -** New documents (actually, document updates) are tokenized and -** written individually (using LeafWriter) to a level 0 segment, with -** incrementing idx. When idx reaches MERGE_COUNT (default 16), all -** level 0 segments are merged into a single level 1 segment. Level 1 -** is populated like level 0, and eventually MERGE_COUNT level 1 -** segments are merged to a single level 2 segment (representing -** MERGE_COUNT^2 updates), and so on. -** -** A segment merge traverses all segments at a given level in -** parallel, performing a straightforward sorted merge. Since segment -** leaf nodes are written in to the %_segments table in order, this -** merge traverses the underlying sqlite disk structures efficiently. -** After the merge, all segment blocks from the merged level are -** deleted. -** -** MERGE_COUNT controls how often we merge segments. 16 seems to be -** somewhat of a sweet spot for insertion performance. 32 and 64 show -** very similar performance numbers to 16 on insertion, though they're -** a tiny bit slower (perhaps due to more overhead in merge-time -** sorting). 8 is about 20% slower than 16, 4 about 50% slower than -** 16, 2 about 66% slower than 16. -** -** At query time, high MERGE_COUNT increases the number of segments -** which need to be scanned and merged. For instance, with 100k docs -** inserted: -** -** MERGE_COUNT segments -** 16 25 -** 8 12 -** 4 10 -** 2 6 -** -** This appears to have only a moderate impact on queries for very -** frequent terms (which are somewhat dominated by segment merge -** costs), and infrequent and non-existent terms still seem to be fast -** even with many segments. -** -** TODO(shess) That said, it would be nice to have a better query-side -** argument for MERGE_COUNT of 16. Also, it is possible/likely that -** optimizations to things like doclist merging will swing the sweet -** spot around. -** -** -** -**** Handling of deletions and updates **** -** Since we're using a segmented structure, with no docid-oriented -** index into the term index, we clearly cannot simply update the term -** index when a document is deleted or updated. For deletions, we -** write an empty doclist (varint(docid) varint(POS_END)), for updates -** we simply write the new doclist. Segment merges overwrite older -** data for a particular docid with newer data, so deletes or updates -** will eventually overtake the earlier data and knock it out. The -** query logic likewise merges doclists so that newer data knocks out -** older data. -*/ - -/************** Include fts3Int.h in the middle of fts3.c ********************/ -/************** Begin file fts3Int.h *****************************************/ -/* -** 2009 Nov 12 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -*/ -#ifndef _FTSINT_H -#define _FTSINT_H - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - -/* FTS3/FTS4 require virtual tables */ -#ifdef SQLITE_OMIT_VIRTUALTABLE -# undef SQLITE_ENABLE_FTS3 -# undef SQLITE_ENABLE_FTS4 -#endif - -/* -** FTS4 is really an extension for FTS3. It is enabled using the -** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all -** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. -*/ -#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) -# define SQLITE_ENABLE_FTS3 -#endif - -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* If not building as part of the core, include sqlite3ext.h. */ -#ifndef SQLITE_CORE -/* # include "sqlite3ext.h" */ -SQLITE_EXTENSION_INIT3 -#endif - -/* #include "sqlite3.h" */ -/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ -/************** Begin file fts3_tokenizer.h **********************************/ -/* -** 2006 July 10 -** -** The author disclaims copyright to this source code. -** -************************************************************************* -** Defines the interface to tokenizers used by fulltext-search. There -** are three basic components: -** -** sqlite3_tokenizer_module is a singleton defining the tokenizer -** interface functions. This is essentially the class structure for -** tokenizers. -** -** sqlite3_tokenizer is used to define a particular tokenizer, perhaps -** including customization information defined at creation time. -** -** sqlite3_tokenizer_cursor is generated by a tokenizer to generate -** tokens from a particular input. -*/ -#ifndef _FTS3_TOKENIZER_H_ -#define _FTS3_TOKENIZER_H_ - -/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. -** If tokenizers are to be allowed to call sqlite3_*() functions, then -** we will need a way to register the API consistently. -*/ -/* #include "sqlite3.h" */ - -/* -** Structures used by the tokenizer interface. When a new tokenizer -** implementation is registered, the caller provides a pointer to -** an sqlite3_tokenizer_module containing pointers to the callback -** functions that make up an implementation. -** -** When an fts3 table is created, it passes any arguments passed to -** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the -** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer -** implementation. The xCreate() function in turn returns an -** sqlite3_tokenizer structure representing the specific tokenizer to -** be used for the fts3 table (customized by the tokenizer clause arguments). -** -** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() -** method is called. It returns an sqlite3_tokenizer_cursor object -** that may be used to tokenize a specific input buffer based on -** the tokenization rules supplied by a specific sqlite3_tokenizer -** object. -*/ -typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; -typedef struct sqlite3_tokenizer sqlite3_tokenizer; -typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; - -struct sqlite3_tokenizer_module { - - /* - ** Structure version. Should always be set to 0 or 1. - */ - int iVersion; - - /* - ** Create a new tokenizer. The values in the argv[] array are the - ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL - ** TABLE statement that created the fts3 table. For example, if - ** the following SQL is executed: - ** - ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) - ** - ** then argc is set to 2, and the argv[] array contains pointers - ** to the strings "arg1" and "arg2". - ** - ** This method should return either SQLITE_OK (0), or an SQLite error - ** code. If SQLITE_OK is returned, then *ppTokenizer should be set - ** to point at the newly created tokenizer structure. The generic - ** sqlite3_tokenizer.pModule variable should not be initialized by - ** this callback. The caller will do so. - */ - int (*xCreate)( - int argc, /* Size of argv array */ - const char *const*argv, /* Tokenizer argument strings */ - sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ - ); - - /* - ** Destroy an existing tokenizer. The fts3 module calls this method - ** exactly once for each successful call to xCreate(). - */ - int (*xDestroy)(sqlite3_tokenizer *pTokenizer); - - /* - ** Create a tokenizer cursor to tokenize an input buffer. The caller - ** is responsible for ensuring that the input buffer remains valid - ** until the cursor is closed (using the xClose() method). - */ - int (*xOpen)( - sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ - const char *pInput, int nBytes, /* Input buffer */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ - ); - - /* - ** Destroy an existing tokenizer cursor. The fts3 module calls this - ** method exactly once for each successful call to xOpen(). - */ - int (*xClose)(sqlite3_tokenizer_cursor *pCursor); - - /* - ** Retrieve the next token from the tokenizer cursor pCursor. This - ** method should either return SQLITE_OK and set the values of the - ** "OUT" variables identified below, or SQLITE_DONE to indicate that - ** the end of the buffer has been reached, or an SQLite error code. - ** - ** *ppToken should be set to point at a buffer containing the - ** normalized version of the token (i.e. after any case-folding and/or - ** stemming has been performed). *pnBytes should be set to the length - ** of this buffer in bytes. The input text that generated the token is - ** identified by the byte offsets returned in *piStartOffset and - ** *piEndOffset. *piStartOffset should be set to the index of the first - ** byte of the token in the input buffer. *piEndOffset should be set - ** to the index of the first byte just past the end of the token in - ** the input buffer. - ** - ** The buffer *ppToken is set to point at is managed by the tokenizer - ** implementation. It is only required to be valid until the next call - ** to xNext() or xClose(). - */ - /* TODO(shess) current implementation requires pInput to be - ** nul-terminated. This should either be fixed, or pInput/nBytes - ** should be converted to zInput. - */ - int (*xNext)( - sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ - const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ - int *piStartOffset, /* OUT: Byte offset of token in input buffer */ - int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ - int *piPosition /* OUT: Number of tokens returned before this one */ - ); - - /*********************************************************************** - ** Methods below this point are only available if iVersion>=1. - */ - - /* - ** Configure the language id of a tokenizer cursor. - */ - int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); -}; - -struct sqlite3_tokenizer { - const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ - /* Tokenizer implementations will typically add additional fields */ -}; - -struct sqlite3_tokenizer_cursor { - sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ - /* Tokenizer implementations will typically add additional fields */ -}; - -int fts3_global_term_cnt(int iTerm, int iCol); -int fts3_term_cnt(int iTerm, int iCol); - - -#endif /* _FTS3_TOKENIZER_H_ */ - -/************** End of fts3_tokenizer.h **************************************/ -/************** Continuing where we left off in fts3Int.h ********************/ -/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ -/************** Begin file fts3_hash.h ***************************************/ -/* -** 2001 September 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This is the header file for the generic hash-table implementation -** used in SQLite. We've modified it slightly to serve as a standalone -** hash table implementation for the full-text indexing module. -** -*/ -#ifndef _FTS3_HASH_H_ -#define _FTS3_HASH_H_ - -/* Forward declarations of structures. */ -typedef struct Fts3Hash Fts3Hash; -typedef struct Fts3HashElem Fts3HashElem; - -/* A complete hash table is an instance of the following structure. -** The internals of this structure are intended to be opaque -- client -** code should not attempt to access or modify the fields of this structure -** directly. Change this structure only by using the routines below. -** However, many of the "procedures" and "functions" for modifying and -** accessing this structure are really macros, so we can't really make -** this structure opaque. -*/ -struct Fts3Hash { - char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ - char copyKey; /* True if copy of key made on insert */ - int count; /* Number of entries in this table */ - Fts3HashElem *first; /* The first element of the array */ - int htsize; /* Number of buckets in the hash table */ - struct _fts3ht { /* the hash table */ - int count; /* Number of entries with this hash */ - Fts3HashElem *chain; /* Pointer to first entry with this hash */ - } *ht; -}; - -/* Each element in the hash table is an instance of the following -** structure. All elements are stored on a single doubly-linked list. -** -** Again, this structure is intended to be opaque, but it can't really -** be opaque because it is used by macros. -*/ -struct Fts3HashElem { - Fts3HashElem *next, *prev; /* Next and previous elements in the table */ - void *data; /* Data associated with this element */ - void *pKey; int nKey; /* Key associated with this element */ -}; - -/* -** There are 2 different modes of operation for a hash table: -** -** FTS3_HASH_STRING pKey points to a string that is nKey bytes long -** (including the null-terminator, if any). Case -** is respected in comparisons. -** -** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. -** memcmp() is used to compare keys. -** -** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. -*/ -#define FTS3_HASH_STRING 1 -#define FTS3_HASH_BINARY 2 - -/* -** Access routines. To delete, insert a NULL pointer. -*/ -SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); -SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); -SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); -SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); -SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); - -/* -** Shorthand for the functions above -*/ -#define fts3HashInit sqlite3Fts3HashInit -#define fts3HashInsert sqlite3Fts3HashInsert -#define fts3HashFind sqlite3Fts3HashFind -#define fts3HashClear sqlite3Fts3HashClear -#define fts3HashFindElem sqlite3Fts3HashFindElem - -/* -** Macros for looping over all elements of a hash table. The idiom is -** like this: -** -** Fts3Hash h; -** Fts3HashElem *p; -** ... -** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ -** SomeStructure *pData = fts3HashData(p); -** // do something with pData -** } -*/ -#define fts3HashFirst(H) ((H)->first) -#define fts3HashNext(E) ((E)->next) -#define fts3HashData(E) ((E)->data) -#define fts3HashKey(E) ((E)->pKey) -#define fts3HashKeysize(E) ((E)->nKey) - -/* -** Number of entries in a hash table -*/ -#define fts3HashCount(H) ((H)->count) - -#endif /* _FTS3_HASH_H_ */ - -/************** End of fts3_hash.h *******************************************/ -/************** Continuing where we left off in fts3Int.h ********************/ - -/* -** This constant determines the maximum depth of an FTS expression tree -** that the library will create and use. FTS uses recursion to perform -** various operations on the query tree, so the disadvantage of a large -** limit is that it may allow very large queries to use large amounts -** of stack space (perhaps causing a stack overflow). -*/ -#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH -# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 -#endif - - -/* -** This constant controls how often segments are merged. Once there are -** FTS3_MERGE_COUNT segments of level N, they are merged into a single -** segment of level N+1. -*/ -#define FTS3_MERGE_COUNT 16 - -/* -** This is the maximum amount of data (in bytes) to store in the -** Fts3Table.pendingTerms hash table. Normally, the hash table is -** populated as documents are inserted/updated/deleted in a transaction -** and used to create a new segment when the transaction is committed. -** However if this limit is reached midway through a transaction, a new -** segment is created and the hash table cleared immediately. -*/ -#define FTS3_MAX_PENDING_DATA (1*1024*1024) - -/* -** Macro to return the number of elements in an array. SQLite has a -** similar macro called ArraySize(). Use a different name to avoid -** a collision when building an amalgamation with built-in FTS3. -*/ -#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) - - -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) -#endif -#ifndef MAX -# define MAX(x,y) ((x)>(y)?(x):(y)) -#endif - -/* -** Maximum length of a varint encoded integer. The varint format is different -** from that used by SQLite, so the maximum length is 10, not 9. -*/ -#define FTS3_VARINT_MAX 10 - -/* -** FTS4 virtual tables may maintain multiple indexes - one index of all terms -** in the document set and zero or more prefix indexes. All indexes are stored -** as one or more b+-trees in the %_segments and %_segdir tables. -** -** It is possible to determine which index a b+-tree belongs to based on the -** value stored in the "%_segdir.level" column. Given this value L, the index -** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with -** level values between 0 and 1023 (inclusive) belong to index 0, all levels -** between 1024 and 2047 to index 1, and so on. -** -** It is considered impossible for an index to use more than 1024 levels. In -** theory though this may happen, but only after at least -** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. -*/ -#define FTS3_SEGDIR_MAXLEVEL 1024 -#define FTS3_SEGDIR_MAXLEVEL_STR "1024" - -/* -** The testcase() macro is only used by the amalgamation. If undefined, -** make it a no-op. -*/ -#ifndef testcase -# define testcase(X) -#endif - -/* -** Terminator values for position-lists and column-lists. -*/ -#define POS_COLUMN (1) /* Column-list terminator */ -#define POS_END (0) /* Position-list terminator */ - -/* -** This section provides definitions to allow the -** FTS3 extension to be compiled outside of the -** amalgamation. -*/ -#ifndef SQLITE_AMALGAMATION -/* -** Macros indicating that conditional expressions are always true or -** false. -*/ -#ifdef SQLITE_COVERAGE_TEST -# define ALWAYS(x) (1) -# define NEVER(X) (0) -#elif defined(SQLITE_DEBUG) -# define ALWAYS(x) sqlite3Fts3Always((x)!=0) -# define NEVER(x) sqlite3Fts3Never((x)!=0) -SQLITE_PRIVATE int sqlite3Fts3Always(int b); -SQLITE_PRIVATE int sqlite3Fts3Never(int b); -#else -# define ALWAYS(x) (x) -# define NEVER(x) (x) -#endif - -/* -** Internal types used by SQLite. -*/ -typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ -typedef short int i16; /* 2-byte (or larger) signed integer */ -typedef unsigned int u32; /* 4-byte unsigned integer */ -typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ -typedef sqlite3_int64 i64; /* 8-byte signed integer */ - -/* -** Macro used to suppress compiler warnings for unused parameters. -*/ -#define UNUSED_PARAMETER(x) (void)(x) - -/* -** Activate assert() only if SQLITE_TEST is enabled. -*/ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - -/* -** The TESTONLY macro is used to enclose variable declarations or -** other bits of code that are needed to support the arguments -** within testcase() and assert() macros. -*/ -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) -# define TESTONLY(X) X -#else -# define TESTONLY(X) -#endif - -#endif /* SQLITE_AMALGAMATION */ - -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); -# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() -#else -# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB -#endif - -typedef struct Fts3Table Fts3Table; -typedef struct Fts3Cursor Fts3Cursor; -typedef struct Fts3Expr Fts3Expr; -typedef struct Fts3Phrase Fts3Phrase; -typedef struct Fts3PhraseToken Fts3PhraseToken; - -typedef struct Fts3Doclist Fts3Doclist; -typedef struct Fts3SegFilter Fts3SegFilter; -typedef struct Fts3DeferredToken Fts3DeferredToken; -typedef struct Fts3SegReader Fts3SegReader; -typedef struct Fts3MultiSegReader Fts3MultiSegReader; - -typedef struct MatchinfoBuffer MatchinfoBuffer; - -/* -** A connection to a fulltext index is an instance of the following -** structure. The xCreate and xConnect methods create an instance -** of this structure and xDestroy and xDisconnect free that instance. -** All other methods receive a pointer to the structure as one of their -** arguments. -*/ -struct Fts3Table { - sqlite3_vtab base; /* Base class used by SQLite core */ - sqlite3 *db; /* The database connection */ - const char *zDb; /* logical database name */ - const char *zName; /* virtual table name */ - int nColumn; /* number of named columns in virtual table */ - char **azColumn; /* column names. malloced */ - u8 *abNotindexed; /* True for 'notindexed' columns */ - sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ - char *zContentTbl; /* content=xxx option, or NULL */ - char *zLanguageid; /* languageid=xxx option, or NULL */ - int nAutoincrmerge; /* Value configured by 'automerge' */ - u32 nLeafAdd; /* Number of leaf blocks added this trans */ - - /* Precompiled statements used by the implementation. Each of these - ** statements is run and reset within a single virtual table API call. - */ - sqlite3_stmt *aStmt[40]; - - char *zReadExprlist; - char *zWriteExprlist; - - int nNodeSize; /* Soft limit for node size */ - u8 bFts4; /* True for FTS4, false for FTS3 */ - u8 bHasStat; /* True if %_stat table exists (2==unknown) */ - u8 bHasDocsize; /* True if %_docsize table exists */ - u8 bDescIdx; /* True if doclists are in reverse order */ - u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ - int nPgsz; /* Page size for host database */ - char *zSegmentsTbl; /* Name of %_segments table */ - sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - - /* - ** The following array of hash tables is used to buffer pending index - ** updates during transactions. All pending updates buffered at any one - ** time must share a common language-id (see the FTS4 langid= feature). - ** The current language id is stored in variable iPrevLangid. - ** - ** A single FTS4 table may have multiple full-text indexes. For each index - ** there is an entry in the aIndex[] array. Index 0 is an index of all the - ** terms that appear in the document set. Each subsequent index in aIndex[] - ** is an index of prefixes of a specific length. - ** - ** Variable nPendingData contains an estimate the memory consumed by the - ** pending data structures, including hash table overhead, but not including - ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash - ** tables are flushed to disk. Variable iPrevDocid is the docid of the most - ** recently inserted record. - */ - int nIndex; /* Size of aIndex[] */ - struct Fts3Index { - int nPrefix; /* Prefix length (0 for main terms index) */ - Fts3Hash hPending; /* Pending terms table for this index */ - } *aIndex; - int nMaxPendingData; /* Max pending data before flush to disk */ - int nPendingData; /* Current bytes of pending data */ - sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ - int iPrevLangid; /* Langid of recently inserted document */ - int bPrevDelete; /* True if last operation was a delete */ - -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) - /* State variables used for validating that the transaction control - ** methods of the virtual table are called at appropriate times. These - ** values do not contribute to FTS functionality; they are used for - ** verifying the operation of the SQLite core. - */ - int inTransaction; /* True after xBegin but before xCommit/xRollback */ - int mxSavepoint; /* Largest valid xSavepoint integer */ -#endif - -#ifdef SQLITE_TEST - /* True to disable the incremental doclist optimization. This is controled - ** by special insert command 'test-no-incr-doclist'. */ - int bNoIncrDoclist; -#endif -}; - -/* -** When the core wants to read from the virtual table, it creates a -** virtual table cursor (an instance of the following structure) using -** the xOpen method. Cursors are destroyed using the xClose method. -*/ -struct Fts3Cursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - i16 eSearch; /* Search strategy (see below) */ - u8 isEof; /* True if at End Of Results */ - u8 isRequireSeek; /* True if must seek pStmt to %_content row */ - sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ - Fts3Expr *pExpr; /* Parsed MATCH query string */ - int iLangid; /* Language being queried for */ - int nPhrase; /* Number of matchable phrases in query */ - Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ - sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ - char *pNextId; /* Pointer into the body of aDoclist */ - char *aDoclist; /* List of docids for full-text queries */ - int nDoclist; /* Size of buffer at aDoclist */ - u8 bDesc; /* True to sort in descending order */ - int eEvalmode; /* An FTS3_EVAL_XX constant */ - int nRowAvg; /* Average size of database rows, in pages */ - sqlite3_int64 nDoc; /* Documents in table */ - i64 iMinDocid; /* Minimum docid to return */ - i64 iMaxDocid; /* Maximum docid to return */ - int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ - MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ -}; - -#define FTS3_EVAL_FILTER 0 -#define FTS3_EVAL_NEXT 1 -#define FTS3_EVAL_MATCHINFO 2 - -/* -** The Fts3Cursor.eSearch member is always set to one of the following. -** Actualy, Fts3Cursor.eSearch can be greater than or equal to -** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index -** of the column to be searched. For example, in -** -** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); -** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; -** -** Because the LHS of the MATCH operator is 2nd column "b", -** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, -** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" -** indicating that all columns should be searched, -** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. -*/ -#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ -#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ -#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ - -/* -** The lower 16-bits of the sqlite3_index_info.idxNum value set by -** the xBestIndex() method contains the Fts3Cursor.eSearch value described -** above. The upper 16-bits contain a combination of the following -** bits, used to describe extra constraints on full-text searches. -*/ -#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ -#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ -#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ - -struct Fts3Doclist { - char *aAll; /* Array containing doclist (or NULL) */ - int nAll; /* Size of a[] in bytes */ - char *pNextDocid; /* Pointer to next docid */ - - sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ - int bFreeList; /* True if pList should be sqlite3_free()d */ - char *pList; /* Pointer to position list following iDocid */ - int nList; /* Length of position list */ -}; - -/* -** A "phrase" is a sequence of one or more tokens that must match in -** sequence. A single token is the base case and the most common case. -** For a sequence of tokens contained in double-quotes (i.e. "one two three") -** nToken will be the number of tokens in the string. -*/ -struct Fts3PhraseToken { - char *z; /* Text of the token */ - int n; /* Number of bytes in buffer z */ - int isPrefix; /* True if token ends with a "*" character */ - int bFirst; /* True if token must appear at position 0 */ - - /* Variables above this point are populated when the expression is - ** parsed (by code in fts3_expr.c). Below this point the variables are - ** used when evaluating the expression. */ - Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ - Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ -}; - -struct Fts3Phrase { - /* Cache of doclist for this phrase. */ - Fts3Doclist doclist; - int bIncr; /* True if doclist is loaded incrementally */ - int iDoclistToken; - - /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an - ** OR condition. */ - char *pOrPoslist; - i64 iOrDocid; - - /* Variables below this point are populated by fts3_expr.c when parsing - ** a MATCH expression. Everything above is part of the evaluation phase. - */ - int nToken; /* Number of tokens in the phrase */ - int iColumn; /* Index of column this phrase must match */ - Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ -}; - -/* -** A tree of these objects forms the RHS of a MATCH operator. -** -** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist -** points to a malloced buffer, size nDoclist bytes, containing the results -** of this phrase query in FTS3 doclist format. As usual, the initial -** "Length" field found in doclists stored on disk is omitted from this -** buffer. -** -** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global -** matchinfo data. If it is not NULL, it points to an array of size nCol*3, -** where nCol is the number of columns in the queried FTS table. The array -** is populated as follows: -** -** aMI[iCol*3 + 0] = Undefined -** aMI[iCol*3 + 1] = Number of occurrences -** aMI[iCol*3 + 2] = Number of rows containing at least one instance -** -** The aMI array is allocated using sqlite3_malloc(). It should be freed -** when the expression node is. -*/ -struct Fts3Expr { - int eType; /* One of the FTSQUERY_XXX values defined below */ - int nNear; /* Valid if eType==FTSQUERY_NEAR */ - Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ - Fts3Expr *pLeft; /* Left operand */ - Fts3Expr *pRight; /* Right operand */ - Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ - - /* The following are used by the fts3_eval.c module. */ - sqlite3_int64 iDocid; /* Current docid */ - u8 bEof; /* True this expression is at EOF already */ - u8 bStart; /* True if iDocid is valid */ - u8 bDeferred; /* True if this expression is entirely deferred */ - - /* The following are used by the fts3_snippet.c module. */ - int iPhrase; /* Index of this phrase in matchinfo() results */ - u32 *aMI; /* See above */ -}; - -/* -** Candidate values for Fts3Query.eType. Note that the order of the first -** four values is in order of precedence when parsing expressions. For -** example, the following: -** -** "a OR b AND c NOT d NEAR e" -** -** is equivalent to: -** -** "a OR (b AND (c NOT (d NEAR e)))" -*/ -#define FTSQUERY_NEAR 1 -#define FTSQUERY_NOT 2 -#define FTSQUERY_AND 3 -#define FTSQUERY_OR 4 -#define FTSQUERY_PHRASE 5 - - -/* fts3_write.c */ -SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); -SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); -SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, - sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); -SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( - Fts3Table*,int,const char*,int,int,Fts3SegReader**); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); - -SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); - -#ifndef SQLITE_DISABLE_FTS4_DEFERRED -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); -SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); -SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); -SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); -#else -# define sqlite3Fts3FreeDeferredTokens(x) -# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK -# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK -# define sqlite3Fts3FreeDeferredDoclists(x) -# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK -#endif - -SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); - -/* Special values interpreted by sqlite3SegReaderCursor() */ -#define FTS3_SEGCURSOR_PENDING -1 -#define FTS3_SEGCURSOR_ALL -2 - -SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); -SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); - -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, - int, int, int, const char *, int, int, int, Fts3MultiSegReader *); - -/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ -#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 -#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 -#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 -#define FTS3_SEGMENT_PREFIX 0x00000008 -#define FTS3_SEGMENT_SCAN 0x00000010 -#define FTS3_SEGMENT_FIRST 0x00000020 - -/* Type passed as 4th argument to SegmentReaderIterate() */ -struct Fts3SegFilter { - const char *zTerm; - int nTerm; - int iCol; - int flags; -}; - -struct Fts3MultiSegReader { - /* Used internally by sqlite3Fts3SegReaderXXX() calls */ - Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ - int nSegment; /* Size of apSegment array */ - int nAdvance; /* How many seg-readers to advance */ - Fts3SegFilter *pFilter; /* Pointer to filter object */ - char *aBuffer; /* Buffer to merge doclists in */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ - - int iColFilter; /* If >=0, filter for this column */ - int bRestart; - - /* Used by fts3.c only. */ - int nCost; /* Cost of running iterator */ - int bLookup; /* True if a lookup of a single entry. */ - - /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ - char *zTerm; /* Pointer to term buffer */ - int nTerm; /* Size of zTerm in bytes */ - char *aDoclist; /* Pointer to doclist buffer */ - int nDoclist; /* Size of aDoclist[] in bytes */ -}; - -SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); - -#define fts3GetVarint32(p, piVal) ( \ - (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ -) - -/* fts3.c */ -SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); -SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); -SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); -SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); -SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); -SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); -SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); -SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); -SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); -SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); -SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); - -/* fts3_tokenizer.c */ -SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); -SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); -SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, - sqlite3_tokenizer **, char ** -); -SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); - -/* fts3_snippet.c */ -SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); -SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, - const char *, const char *, int, int -); -SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); -SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); - -/* fts3_expr.c */ -SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, - char **, int, int, int, const char *, int, Fts3Expr **, char ** -); -SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); -#ifdef SQLITE_TEST -SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); -SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); -#endif - -SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, - sqlite3_tokenizer_cursor ** -); - -/* fts3_aux.c */ -SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); - -SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); - -SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( - Fts3Table*, Fts3MultiSegReader*, int, const char*, int); -SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( - Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); -SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); -SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); -SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); - -/* fts3_tokenize_vtab.c */ -SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); - -/* fts3_unicode2.c (functions generated by parsing unicode text files) */ -#ifndef SQLITE_DISABLE_FTS3_UNICODE -SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); -SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); -SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); -#endif - -#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ -#endif /* _FTSINT_H */ - -/************** End of fts3Int.h *********************************************/ -/************** Continuing where we left off in fts3.c ***********************/ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) -# define SQLITE_CORE 1 -#endif - -/* #include */ -/* #include */ -/* #include */ -/* #include */ -/* #include */ -/* #include */ - -/* #include "fts3.h" */ -#ifndef SQLITE_CORE -/* # include "sqlite3ext.h" */ - SQLITE_EXTENSION_INIT1 -#endif - -static int fts3EvalNext(Fts3Cursor *pCsr); -static int fts3EvalStart(Fts3Cursor *pCsr); -static int fts3TermSegReaderCursor( - Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); - -#ifndef SQLITE_AMALGAMATION -# if defined(SQLITE_DEBUG) -SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; } -SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; } -# endif -#endif - -/* -** Write a 64-bit variable-length integer to memory starting at p[0]. -** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. -** The number of bytes written is returned. -*/ -SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ - unsigned char *q = (unsigned char *) p; - sqlite_uint64 vu = v; - do{ - *q++ = (unsigned char) ((vu & 0x7f) | 0x80); - vu >>= 7; - }while( vu!=0 ); - q[-1] &= 0x7f; /* turn off high bit in final byte */ - assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); - return (int) (q - (unsigned char *)p); -} - -#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ - v = (v & mask1) | ( (*ptr++) << shift ); \ - if( (v & mask2)==0 ){ var = v; return ret; } -#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ - v = (*ptr++); \ - if( (v & mask2)==0 ){ var = v; return ret; } - -/* -** Read a 64-bit variable-length integer from memory starting at p[0]. -** Return the number of bytes read, or 0 on error. -** The value is stored in *v. -*/ -SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ - const char *pStart = p; - u32 a; - u64 b; - int shift; - - GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); - GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); - GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); - GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); - b = (a & 0x0FFFFFFF ); - - for(shift=28; shift<=63; shift+=7){ - u64 c = *p++; - b += (c&0x7F) << shift; - if( (c & 0x80)==0 ) break; - } - *v = b; - return (int)(p - pStart); -} - -/* -** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a -** 32-bit integer before it is returned. -*/ -SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){ - u32 a; - -#ifndef fts3GetVarint32 - GETVARINT_INIT(a, p, 0, 0x00, 0x80, *pi, 1); -#else - a = (*p++); - assert( a & 0x80 ); -#endif - - GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *pi, 2); - GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *pi, 3); - GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4); - a = (a & 0x0FFFFFFF ); - *pi = (int)(a | ((u32)(*p & 0x0F) << 28)); - return 5; -} - -/* -** Return the number of bytes required to encode v as a varint -*/ -SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){ - int i = 0; - do{ - i++; - v >>= 7; - }while( v!=0 ); - return i; -} - -/* -** Convert an SQL-style quoted string into a normal string by removing -** the quote characters. The conversion is done in-place. If the -** input does not begin with a quote character, then this routine -** is a no-op. -** -** Examples: -** -** "abc" becomes abc -** 'xyz' becomes xyz -** [pqr] becomes pqr -** `mno` becomes mno -** -*/ -SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ - char quote; /* Quote character (if any ) */ - - quote = z[0]; - if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ - int iIn = 1; /* Index of next byte to read from input */ - int iOut = 0; /* Index of next byte to write to output */ - - /* If the first byte was a '[', then the close-quote character is a ']' */ - if( quote=='[' ) quote = ']'; - - while( z[iIn] ){ - if( z[iIn]==quote ){ - if( z[iIn+1]!=quote ) break; - z[iOut++] = quote; - iIn += 2; - }else{ - z[iOut++] = z[iIn++]; - } - } - z[iOut] = '\0'; - } -} - -/* -** Read a single varint from the doclist at *pp and advance *pp to point -** to the first byte past the end of the varint. Add the value of the varint -** to *pVal. -*/ -static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ - sqlite3_int64 iVal; - *pp += sqlite3Fts3GetVarint(*pp, &iVal); - *pVal += iVal; -} - -/* -** When this function is called, *pp points to the first byte following a -** varint that is part of a doclist (or position-list, or any other list -** of varints). This function moves *pp to point to the start of that varint, -** and sets *pVal by the varint value. -** -** Argument pStart points to the first byte of the doclist that the -** varint is part of. -*/ -static void fts3GetReverseVarint( - char **pp, - char *pStart, - sqlite3_int64 *pVal -){ - sqlite3_int64 iVal; - char *p; - - /* Pointer p now points at the first byte past the varint we are - ** interested in. So, unless the doclist is corrupt, the 0x80 bit is - ** clear on character p[-1]. */ - for(p = (*pp)-2; p>=pStart && *p&0x80; p--); - p++; - *pp = p; - - sqlite3Fts3GetVarint(p, &iVal); - *pVal = iVal; -} - -/* -** The xDisconnect() virtual table method. -*/ -static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table *)pVtab; - int i; - - assert( p->nPendingData==0 ); - assert( p->pSegments==0 ); - - /* Free any prepared statements held */ - for(i=0; iaStmt); i++){ - sqlite3_finalize(p->aStmt[i]); - } - sqlite3_free(p->zSegmentsTbl); - sqlite3_free(p->zReadExprlist); - sqlite3_free(p->zWriteExprlist); - sqlite3_free(p->zContentTbl); - sqlite3_free(p->zLanguageid); - - /* Invoke the tokenizer destructor to free the tokenizer. */ - p->pTokenizer->pModule->xDestroy(p->pTokenizer); - - sqlite3_free(p); - return SQLITE_OK; -} - -/* -** Write an error message into *pzErr -*/ -SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ - va_list ap; - sqlite3_free(*pzErr); - va_start(ap, zFormat); - *pzErr = sqlite3_vmprintf(zFormat, ap); - va_end(ap); -} - -/* -** Construct one or more SQL statements from the format string given -** and then evaluate those statements. The success code is written -** into *pRc. -** -** If *pRc is initially non-zero then this routine is a no-op. -*/ -static void fts3DbExec( - int *pRc, /* Success code */ - sqlite3 *db, /* Database in which to run SQL */ - const char *zFormat, /* Format string for SQL */ - ... /* Arguments to the format string */ -){ - va_list ap; - char *zSql; - if( *pRc ) return; - va_start(ap, zFormat); - zSql = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( zSql==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - *pRc = sqlite3_exec(db, zSql, 0, 0, 0); - sqlite3_free(zSql); - } -} - -/* -** The xDestroy() virtual table method. -*/ -static int fts3DestroyMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table *)pVtab; - int rc = SQLITE_OK; /* Return code */ - const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ - sqlite3 *db = p->db; /* Database handle */ - - /* Drop the shadow tables */ - if( p->zContentTbl==0 ){ - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); - } - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); - - /* If everything has worked, invoke fts3DisconnectMethod() to free the - ** memory associated with the Fts3Table structure and return SQLITE_OK. - ** Otherwise, return an SQLite error code. - */ - return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); -} - - -/* -** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table -** passed as the first argument. This is done as part of the xConnect() -** and xCreate() methods. -** -** If *pRc is non-zero when this function is called, it is a no-op. -** Otherwise, if an error occurs, an SQLite error code is stored in *pRc -** before returning. -*/ -static void fts3DeclareVtab(int *pRc, Fts3Table *p){ - if( *pRc==SQLITE_OK ){ - int i; /* Iterator variable */ - int rc; /* Return code */ - char *zSql; /* SQL statement passed to declare_vtab() */ - char *zCols; /* List of user defined columns */ - const char *zLanguageid; - - zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); - sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); - - /* Create a list of user columns for the virtual table */ - zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); - for(i=1; zCols && inColumn; i++){ - zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); - } - - /* Create the whole "CREATE TABLE" statement to pass to SQLite */ - zSql = sqlite3_mprintf( - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", - zCols, p->zName, zLanguageid - ); - if( !zCols || !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_declare_vtab(p->db, zSql); - } - - sqlite3_free(zSql); - sqlite3_free(zCols); - *pRc = rc; - } -} - -/* -** Create the %_stat table if it does not already exist. -*/ -SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ - fts3DbExec(pRc, p->db, - "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" - "(id INTEGER PRIMARY KEY, value BLOB);", - p->zDb, p->zName - ); - if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; -} - -/* -** Create the backing store tables (%_content, %_segments and %_segdir) -** required by the FTS3 table passed as the only argument. This is done -** as part of the vtab xCreate() method. -** -** If the p->bHasDocsize boolean is true (indicating that this is an -** FTS4 table, not an FTS3 table) then also create the %_docsize and -** %_stat tables required by FTS4. -*/ -static int fts3CreateTables(Fts3Table *p){ - int rc = SQLITE_OK; /* Return code */ - int i; /* Iterator variable */ - sqlite3 *db = p->db; /* The database connection */ - - if( p->zContentTbl==0 ){ - const char *zLanguageid = p->zLanguageid; - char *zContentCols; /* Columns of %_content table */ - - /* Create a list of user columns for the content table */ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); - } - if( zLanguageid && zContentCols ){ - zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); - } - if( zContentCols==0 ) rc = SQLITE_NOMEM; - - /* Create the content table */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_content'(%s)", - p->zDb, p->zName, zContentCols - ); - sqlite3_free(zContentCols); - } - - /* Create other tables */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", - p->zDb, p->zName - ); - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_segdir'(" - "level INTEGER," - "idx INTEGER," - "start_block INTEGER," - "leaves_end_block INTEGER," - "end_block INTEGER," - "root BLOB," - "PRIMARY KEY(level, idx)" - ");", - p->zDb, p->zName - ); - if( p->bHasDocsize ){ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", - p->zDb, p->zName - ); - } - assert( p->bHasStat==p->bFts4 ); - if( p->bHasStat ){ - sqlite3Fts3CreateStatTable(&rc, p); - } - return rc; -} - -/* -** Store the current database page-size in bytes in p->nPgsz. -** -** If *pRc is non-zero when this function is called, it is a no-op. -** Otherwise, if an error occurs, an SQLite error code is stored in *pRc -** before returning. -*/ -static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ - if( *pRc==SQLITE_OK ){ - int rc; /* Return code */ - char *zSql; /* SQL text "PRAGMA %Q.page_size" */ - sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ - - zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_step(pStmt); - p->nPgsz = sqlite3_column_int(pStmt, 0); - rc = sqlite3_finalize(pStmt); - }else if( rc==SQLITE_AUTH ){ - p->nPgsz = 1024; - rc = SQLITE_OK; - } - } - assert( p->nPgsz>0 || rc!=SQLITE_OK ); - sqlite3_free(zSql); - *pRc = rc; - } -} - -/* -** "Special" FTS4 arguments are column specifications of the following form: -** -** = -** -** There may not be whitespace surrounding the "=" character. The -** term may be quoted, but the may not. -*/ -static int fts3IsSpecialColumn( - const char *z, - int *pnKey, - char **pzValue -){ - char *zValue; - const char *zCsr = z; - - while( *zCsr!='=' ){ - if( *zCsr=='\0' ) return 0; - zCsr++; - } - - *pnKey = (int)(zCsr-z); - zValue = sqlite3_mprintf("%s", &zCsr[1]); - if( zValue ){ - sqlite3Fts3Dequote(zValue); - } - *pzValue = zValue; - return 1; -} - -/* -** Append the output of a printf() style formatting to an existing string. -*/ -static void fts3Appendf( - int *pRc, /* IN/OUT: Error code */ - char **pz, /* IN/OUT: Pointer to string buffer */ - const char *zFormat, /* Printf format string to append */ - ... /* Arguments for printf format string */ -){ - if( *pRc==SQLITE_OK ){ - va_list ap; - char *z; - va_start(ap, zFormat); - z = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( z && *pz ){ - char *z2 = sqlite3_mprintf("%s%s", *pz, z); - sqlite3_free(z); - z = z2; - } - if( z==0 ) *pRc = SQLITE_NOMEM; - sqlite3_free(*pz); - *pz = z; - } -} - -/* -** Return a copy of input string zInput enclosed in double-quotes (") and -** with all double quote characters escaped. For example: -** -** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" -** -** The pointer returned points to memory obtained from sqlite3_malloc(). It -** is the callers responsibility to call sqlite3_free() to release this -** memory. -*/ -static char *fts3QuoteId(char const *zInput){ - int nRet; - char *zRet; - nRet = 2 + (int)strlen(zInput)*2 + 1; - zRet = sqlite3_malloc(nRet); - if( zRet ){ - int i; - char *z = zRet; - *(z++) = '"'; - for(i=0; zInput[i]; i++){ - if( zInput[i]=='"' ) *(z++) = '"'; - *(z++) = zInput[i]; - } - *(z++) = '"'; - *(z++) = '\0'; - } - return zRet; -} - -/* -** Return a list of comma separated SQL expressions and a FROM clause that -** could be used in a SELECT statement such as the following: -** -** SELECT FROM %_content AS x ... -** -** to return the docid, followed by each column of text data in order -** from left to write. If parameter zFunc is not NULL, then instead of -** being returned directly each column of text data is passed to an SQL -** function named zFunc first. For example, if zFunc is "unzip" and the -** table has the three user-defined columns "a", "b", and "c", the following -** string is returned: -** -** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" -** -** The pointer returned points to a buffer allocated by sqlite3_malloc(). It -** is the responsibility of the caller to eventually free it. -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and -** a NULL pointer is returned). Otherwise, if an OOM error is encountered -** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If -** no error occurs, *pRc is left unmodified. -*/ -static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ - char *zRet = 0; - char *zFree = 0; - char *zFunction; - int i; - - if( p->zContentTbl==0 ){ - if( !zFunc ){ - zFunction = ""; - }else{ - zFree = zFunction = fts3QuoteId(zFunc); - } - fts3Appendf(pRc, &zRet, "docid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); - } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); - } - sqlite3_free(zFree); - }else{ - fts3Appendf(pRc, &zRet, "rowid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); - } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); - } - } - fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", - p->zDb, - (p->zContentTbl ? p->zContentTbl : p->zName), - (p->zContentTbl ? "" : "_content") - ); - return zRet; -} - -/* -** Return a list of N comma separated question marks, where N is the number -** of columns in the %_content table (one for the docid plus one for each -** user-defined text column). -** -** If argument zFunc is not NULL, then all but the first question mark -** is preceded by zFunc and an open bracket, and followed by a closed -** bracket. For example, if zFunc is "zip" and the FTS3 table has three -** user-defined text columns, the following string is returned: -** -** "?, zip(?), zip(?), zip(?)" -** -** The pointer returned points to a buffer allocated by sqlite3_malloc(). It -** is the responsibility of the caller to eventually free it. -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and -** a NULL pointer is returned). Otherwise, if an OOM error is encountered -** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If -** no error occurs, *pRc is left unmodified. -*/ -static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ - char *zRet = 0; - char *zFree = 0; - char *zFunction; - int i; - - if( !zFunc ){ - zFunction = ""; - }else{ - zFree = zFunction = fts3QuoteId(zFunc); - } - fts3Appendf(pRc, &zRet, "?"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); - } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", ?"); - } - sqlite3_free(zFree); - return zRet; -} - -/* -** This function interprets the string at (*pp) as a non-negative integer -** value. It reads the integer and sets *pnOut to the value read, then -** sets *pp to point to the byte immediately following the last byte of -** the integer value. -** -** Only decimal digits ('0'..'9') may be part of an integer value. -** -** If *pp does not being with a decimal digit SQLITE_ERROR is returned and -** the output value undefined. Otherwise SQLITE_OK is returned. -** -** This function is used when parsing the "prefix=" FTS4 parameter. -*/ -static int fts3GobbleInt(const char **pp, int *pnOut){ - const int MAX_NPREFIX = 10000000; - const char *p; /* Iterator pointer */ - int nInt = 0; /* Output value */ - - for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ - nInt = nInt * 10 + (p[0] - '0'); - if( nInt>MAX_NPREFIX ){ - nInt = 0; - break; - } - } - if( p==*pp ) return SQLITE_ERROR; - *pnOut = nInt; - *pp = p; - return SQLITE_OK; -} - -/* -** This function is called to allocate an array of Fts3Index structures -** representing the indexes maintained by the current FTS table. FTS tables -** always maintain the main "terms" index, but may also maintain one or -** more "prefix" indexes, depending on the value of the "prefix=" parameter -** (if any) specified as part of the CREATE VIRTUAL TABLE statement. -** -** Argument zParam is passed the value of the "prefix=" option if one was -** specified, or NULL otherwise. -** -** If no error occurs, SQLITE_OK is returned and *apIndex set to point to -** the allocated array. *pnIndex is set to the number of elements in the -** array. If an error does occur, an SQLite error code is returned. -** -** Regardless of whether or not an error is returned, it is the responsibility -** of the caller to call sqlite3_free() on the output array to free it. -*/ -static int fts3PrefixParameter( - const char *zParam, /* ABC in prefix=ABC parameter to parse */ - int *pnIndex, /* OUT: size of *apIndex[] array */ - struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ -){ - struct Fts3Index *aIndex; /* Allocated array */ - int nIndex = 1; /* Number of entries in array */ - - if( zParam && zParam[0] ){ - const char *p; - nIndex++; - for(p=zParam; *p; p++){ - if( *p==',' ) nIndex++; - } - } - - aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); - *apIndex = aIndex; - if( !aIndex ){ - return SQLITE_NOMEM; - } - - memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); - if( zParam ){ - const char *p = zParam; - int i; - for(i=1; i=0 ); - if( nPrefix==0 ){ - nIndex--; - i--; - }else{ - aIndex[i].nPrefix = nPrefix; - } - p++; - } - } - - *pnIndex = nIndex; - return SQLITE_OK; -} - -/* -** This function is called when initializing an FTS4 table that uses the -** content=xxx option. It determines the number of and names of the columns -** of the new FTS4 table. -** -** The third argument passed to this function is the value passed to the -** config=xxx option (i.e. "xxx"). This function queries the database for -** a table of that name. If found, the output variables are populated -** as follows: -** -** *pnCol: Set to the number of columns table xxx has, -** -** *pnStr: Set to the total amount of space required to store a copy -** of each columns name, including the nul-terminator. -** -** *pazCol: Set to point to an array of *pnCol strings. Each string is -** the name of the corresponding column in table xxx. The array -** and its contents are allocated using a single allocation. It -** is the responsibility of the caller to free this allocation -** by eventually passing the *pazCol value to sqlite3_free(). -** -** If the table cannot be found, an error code is returned and the output -** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is -** returned (and the output variables are undefined). -*/ -static int fts3ContentColumns( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ - const char *zTbl, /* Name of content table */ - const char ***pazCol, /* OUT: Malloc'd array of column names */ - int *pnCol, /* OUT: Size of array *pazCol */ - int *pnStr, /* OUT: Bytes of string content */ - char **pzErr /* OUT: error message */ -){ - int rc = SQLITE_OK; /* Return code */ - char *zSql; /* "SELECT *" statement on zTbl */ - sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ - - zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); - } - } - sqlite3_free(zSql); - - if( rc==SQLITE_OK ){ - const char **azCol; /* Output array */ - int nStr = 0; /* Size of all column names (incl. 0x00) */ - int nCol; /* Number of table columns */ - int i; /* Used to iterate through columns */ - - /* Loop through the returned columns. Set nStr to the number of bytes of - ** space required to store a copy of each column name, including the - ** nul-terminator byte. */ - nCol = sqlite3_column_count(pStmt); - for(i=0; i module name ("fts3" or "fts4") -** argv[1] -> database name -** argv[2] -> table name -** argv[...] -> "column name" and other module argument fields. -*/ -static int fts3InitVtab( - int isCreate, /* True for xCreate, false for xConnect */ - sqlite3 *db, /* The SQLite database connection */ - void *pAux, /* Hash table containing tokenizers */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ - char **pzErr /* Write any error message here */ -){ - Fts3Hash *pHash = (Fts3Hash *)pAux; - Fts3Table *p = 0; /* Pointer to allocated vtab */ - int rc = SQLITE_OK; /* Return code */ - int i; /* Iterator variable */ - int nByte; /* Size of allocation used for *p */ - int iCol; /* Column index */ - int nString = 0; /* Bytes required to hold all column names */ - int nCol = 0; /* Number of columns in the FTS table */ - char *zCsr; /* Space for holding column names */ - int nDb; /* Bytes required to hold database name */ - int nName; /* Bytes required to hold table name */ - int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ - const char **aCol; /* Array of column names */ - sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ - - int nIndex = 0; /* Size of aIndex[] array */ - struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ - - /* The results of parsing supported FTS4 key=value options: */ - int bNoDocsize = 0; /* True to omit %_docsize table */ - int bDescIdx = 0; /* True to store descending indexes */ - char *zPrefix = 0; /* Prefix parameter value (or NULL) */ - char *zCompress = 0; /* compress=? parameter (or NULL) */ - char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ - char *zContent = 0; /* content=? parameter (or NULL) */ - char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ - char **azNotindexed = 0; /* The set of notindexed= columns */ - int nNotindexed = 0; /* Size of azNotindexed[] array */ - - assert( strlen(argv[0])==4 ); - assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) - || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) - ); - - nDb = (int)strlen(argv[1]) + 1; - nName = (int)strlen(argv[2]) + 1; - - nByte = sizeof(const char *) * (argc-2); - aCol = (const char **)sqlite3_malloc(nByte); - if( aCol ){ - memset((void*)aCol, 0, nByte); - azNotindexed = (char **)sqlite3_malloc(nByte); - } - if( azNotindexed ){ - memset(azNotindexed, 0, nByte); - } - if( !aCol || !azNotindexed ){ - rc = SQLITE_NOMEM; - goto fts3_init_out; - } - - /* Loop through all of the arguments passed by the user to the FTS3/4 - ** module (i.e. all the column names and special arguments). This loop - ** does the following: - ** - ** + Figures out the number of columns the FTSX table will have, and - ** the number of bytes of space that must be allocated to store copies - ** of the column names. - ** - ** + If there is a tokenizer specification included in the arguments, - ** initializes the tokenizer pTokenizer. - */ - for(i=3; rc==SQLITE_OK && i8 - && 0==sqlite3_strnicmp(z, "tokenize", 8) - && 0==sqlite3Fts3IsIdChar(z[8]) - ){ - rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); - } - - /* Check if it is an FTS4 special argument. */ - else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ - struct Fts4Option { - const char *zOpt; - int nOpt; - } aFts4Opt[] = { - { "matchinfo", 9 }, /* 0 -> MATCHINFO */ - { "prefix", 6 }, /* 1 -> PREFIX */ - { "compress", 8 }, /* 2 -> COMPRESS */ - { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ - { "order", 5 }, /* 4 -> ORDER */ - { "content", 7 }, /* 5 -> CONTENT */ - { "languageid", 10 }, /* 6 -> LANGUAGEID */ - { "notindexed", 10 } /* 7 -> NOTINDEXED */ - }; - - int iOpt; - if( !zVal ){ - rc = SQLITE_NOMEM; - }else{ - for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ - break; - } - } - if( iOpt==SizeofArray(aFts4Opt) ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); - rc = SQLITE_ERROR; - }else{ - switch( iOpt ){ - case 0: /* MATCHINFO */ - if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; - } - bNoDocsize = 1; - break; - - case 1: /* PREFIX */ - sqlite3_free(zPrefix); - zPrefix = zVal; - zVal = 0; - break; - - case 2: /* COMPRESS */ - sqlite3_free(zCompress); - zCompress = zVal; - zVal = 0; - break; - - case 3: /* UNCOMPRESS */ - sqlite3_free(zUncompress); - zUncompress = zVal; - zVal = 0; - break; - - case 4: /* ORDER */ - if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) - && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) - ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); - rc = SQLITE_ERROR; - } - bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); - break; - - case 5: /* CONTENT */ - sqlite3_free(zContent); - zContent = zVal; - zVal = 0; - break; - - case 6: /* LANGUAGEID */ - assert( iOpt==6 ); - sqlite3_free(zLanguageid); - zLanguageid = zVal; - zVal = 0; - break; - - case 7: /* NOTINDEXED */ - azNotindexed[nNotindexed++] = zVal; - zVal = 0; - break; - } - } - sqlite3_free(zVal); - } - } - - /* Otherwise, the argument is a column name. */ - else { - nString += (int)(strlen(z) + 1); - aCol[nCol++] = z; - } - } - - /* If a content=xxx option was specified, the following: - ** - ** 1. Ignore any compress= and uncompress= options. - ** - ** 2. If no column names were specified as part of the CREATE VIRTUAL - ** TABLE statement, use all columns from the content table. - */ - if( rc==SQLITE_OK && zContent ){ - sqlite3_free(zCompress); - sqlite3_free(zUncompress); - zCompress = 0; - zUncompress = 0; - if( nCol==0 ){ - sqlite3_free((void*)aCol); - aCol = 0; - rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); - - /* If a languageid= option was specified, remove the language id - ** column from the aCol[] array. */ - if( rc==SQLITE_OK && zLanguageid ){ - int j; - for(j=0; jdb = db; - p->nColumn = nCol; - p->nPendingData = 0; - p->azColumn = (char **)&p[1]; - p->pTokenizer = pTokenizer; - p->nMaxPendingData = FTS3_MAX_PENDING_DATA; - p->bHasDocsize = (isFts4 && bNoDocsize==0); - p->bHasStat = isFts4; - p->bFts4 = isFts4; - p->bDescIdx = bDescIdx; - p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ - p->zContentTbl = zContent; - p->zLanguageid = zLanguageid; - zContent = 0; - zLanguageid = 0; - TESTONLY( p->inTransaction = -1 ); - TESTONLY( p->mxSavepoint = -1 ); - - p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; - memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); - p->nIndex = nIndex; - for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); - } - p->abNotindexed = (u8 *)&p->aIndex[nIndex]; - - /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->abNotindexed[nCol]; - p->zName = zCsr; - memcpy(zCsr, argv[2], nName); - zCsr += nName; - p->zDb = zCsr; - memcpy(zCsr, argv[1], nDb); - zCsr += nDb; - - /* Fill in the azColumn array */ - for(iCol=0; iColazColumn[iCol] = zCsr; - zCsr += n+1; - assert( zCsr <= &((char *)p)[nByte] ); - } - - /* Fill in the abNotindexed array */ - for(iCol=0; iColazColumn[iCol]); - for(i=0; iazColumn[iCol], zNot, n) - ){ - p->abNotindexed[iCol] = 1; - sqlite3_free(zNot); - azNotindexed[i] = 0; - } - } - } - for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); - p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); - if( rc!=SQLITE_OK ) goto fts3_init_out; - - /* If this is an xCreate call, create the underlying tables in the - ** database. TODO: For xConnect(), it could verify that said tables exist. - */ - if( isCreate ){ - rc = fts3CreateTables(p); - } - - /* Check to see if a legacy fts3 table has been "upgraded" by the - ** addition of a %_stat table so that it can use incremental merge. - */ - if( !isFts4 && !isCreate ){ - p->bHasStat = 2; - } - - /* Figure out the page-size for the database. This is required in order to - ** estimate the cost of loading large doclists from the database. */ - fts3DatabasePageSize(&rc, p); - p->nNodeSize = p->nPgsz-35; - - /* Declare the table schema to SQLite. */ - fts3DeclareVtab(&rc, p); - -fts3_init_out: - sqlite3_free(zPrefix); - sqlite3_free(aIndex); - sqlite3_free(zCompress); - sqlite3_free(zUncompress); - sqlite3_free(zContent); - sqlite3_free(zLanguageid); - for(i=0; ipModule->xDestroy(pTokenizer); - } - }else{ - assert( p->pSegments==0 ); - *ppVTab = &p->base; - } - return rc; -} - -/* -** The xConnect() and xCreate() methods for the virtual table. All the -** work is done in function fts3InitVtab(). -*/ -static int fts3ConnectMethod( - sqlite3 *db, /* Database connection */ - void *pAux, /* Pointer to tokenizer hash table */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ -){ - return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); -} -static int fts3CreateMethod( - sqlite3 *db, /* Database connection */ - void *pAux, /* Pointer to tokenizer hash table */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ -){ - return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); -} - -/* -** Set the pIdxInfo->estimatedRows variable to nRow. Unless this -** extension is currently being used by a version of SQLite too old to -** support estimatedRows. In that case this function is a no-op. -*/ -static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ -#if SQLITE_VERSION_NUMBER>=3008002 - if( sqlite3_libversion_number()>=3008002 ){ - pIdxInfo->estimatedRows = nRow; - } -#endif -} - -/* -** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this -** extension is currently being used by a version of SQLite too old to -** support index-info flags. In that case this function is a no-op. -*/ -static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ -#if SQLITE_VERSION_NUMBER>=3008012 - if( sqlite3_libversion_number()>=3008012 ){ - pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; - } -#endif -} - -/* -** Implementation of the xBestIndex method for FTS3 tables. There -** are three possible strategies, in order of preference: -** -** 1. Direct lookup by rowid or docid. -** 2. Full-text search using a MATCH operator on a non-docid column. -** 3. Linear scan of %_content table. -*/ -static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ - Fts3Table *p = (Fts3Table *)pVTab; - int i; /* Iterator variable */ - int iCons = -1; /* Index of constraint to use */ - - int iLangidCons = -1; /* Index of langid=x constraint, if present */ - int iDocidGe = -1; /* Index of docid>=x constraint, if present */ - int iDocidLe = -1; /* Index of docid<=x constraint, if present */ - int iIdx; - - /* By default use a full table scan. This is an expensive option, - ** so search through the constraints to see if a more efficient - ** strategy is possible. - */ - pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 5000000; - for(i=0; inConstraint; i++){ - int bDocid; /* True if this constraint is on docid */ - struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; - if( pCons->usable==0 ){ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ - /* There exists an unusable MATCH constraint. This means that if - ** the planner does elect to use the results of this call as part - ** of the overall query plan the user will see an "unable to use - ** function MATCH in the requested context" error. To discourage - ** this, return a very high cost here. */ - pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 1e50; - fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); - return SQLITE_OK; - } - continue; - } - - bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); - - /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ - if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ - pInfo->idxNum = FTS3_DOCID_SEARCH; - pInfo->estimatedCost = 1.0; - iCons = i; - } - - /* A MATCH constraint. Use a full-text search. - ** - ** If there is more than one MATCH constraint available, use the first - ** one encountered. If there is both a MATCH constraint and a direct - ** rowid/docid lookup, prefer the MATCH strategy. This is done even - ** though the rowid/docid lookup is faster than a MATCH query, selecting - ** it would lead to an "unable to use function MATCH in the requested - ** context" error. - */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH - && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn - ){ - pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; - pInfo->estimatedCost = 2.0; - iCons = i; - } - - /* Equality constraint on the langid column */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ - && pCons->iColumn==p->nColumn + 2 - ){ - iLangidCons = i; - } - - if( bDocid ){ - switch( pCons->op ){ - case SQLITE_INDEX_CONSTRAINT_GE: - case SQLITE_INDEX_CONSTRAINT_GT: - iDocidGe = i; - break; - - case SQLITE_INDEX_CONSTRAINT_LE: - case SQLITE_INDEX_CONSTRAINT_LT: - iDocidLe = i; - break; - } - } - } - - /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ - if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); - - iIdx = 1; - if( iCons>=0 ){ - pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; - pInfo->aConstraintUsage[iCons].omit = 1; - } - if( iLangidCons>=0 ){ - pInfo->idxNum |= FTS3_HAVE_LANGID; - pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; - } - if( iDocidGe>=0 ){ - pInfo->idxNum |= FTS3_HAVE_DOCID_GE; - pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; - } - if( iDocidLe>=0 ){ - pInfo->idxNum |= FTS3_HAVE_DOCID_LE; - pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; - } - - /* Regardless of the strategy selected, FTS can deliver rows in rowid (or - ** docid) order. Both ascending and descending are possible. - */ - if( pInfo->nOrderBy==1 ){ - struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; - if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ - if( pOrder->desc ){ - pInfo->idxStr = "DESC"; - }else{ - pInfo->idxStr = "ASC"; - } - pInfo->orderByConsumed = 1; - } - } - - assert( p->pSegments==0 ); - return SQLITE_OK; -} - -/* -** Implementation of xOpen method. -*/ -static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ - - UNUSED_PARAMETER(pVTab); - - /* Allocate a buffer large enough for an Fts3Cursor structure. If the - ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, - ** if the allocation fails, return SQLITE_NOMEM. - */ - *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); - if( !pCsr ){ - return SQLITE_NOMEM; - } - memset(pCsr, 0, sizeof(Fts3Cursor)); - return SQLITE_OK; -} - -/* -** Close the cursor. For additional information see the documentation -** on the xClose method of the virtual table interface. -*/ -static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_finalize(pCsr->pStmt); - sqlite3Fts3ExprFree(pCsr->pExpr); - sqlite3Fts3FreeDeferredTokens(pCsr); - sqlite3_free(pCsr->aDoclist); - sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_free(pCsr); - return SQLITE_OK; -} - -/* -** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then -** compose and prepare an SQL statement of the form: -** -** "SELECT FROM %_content WHERE rowid = ?" -** -** (or the equivalent for a content=xxx table) and set pCsr->pStmt to -** it. If an error occurs, return an SQLite error code. -** -** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. -*/ -static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ - int rc = SQLITE_OK; - if( pCsr->pStmt==0 ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - char *zSql; - zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); - } - *ppStmt = pCsr->pStmt; - return rc; -} - -/* -** Position the pCsr->pStmt statement so that it is on the row -** of the %_content table that contains the last match. Return -** SQLITE_OK on success. -*/ -static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ - int rc = SQLITE_OK; - if( pCsr->isRequireSeek ){ - sqlite3_stmt *pStmt = 0; - - rc = fts3CursorSeekStmt(pCsr, &pStmt); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); - pCsr->isRequireSeek = 0; - if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ - return SQLITE_OK; - }else{ - rc = sqlite3_reset(pCsr->pStmt); - if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ - /* If no row was found and no error has occurred, then the %_content - ** table is missing a row that is present in the full-text index. - ** The data structures are corrupt. */ - rc = FTS_CORRUPT_VTAB; - pCsr->isEof = 1; - } - } - } - } - - if( rc!=SQLITE_OK && pContext ){ - sqlite3_result_error_code(pContext, rc); - } - return rc; -} - -/* -** This function is used to process a single interior node when searching -** a b-tree for a term or term prefix. The node data is passed to this -** function via the zNode/nNode parameters. The term to search for is -** passed in zTerm/nTerm. -** -** If piFirst is not NULL, then this function sets *piFirst to the blockid -** of the child node that heads the sub-tree that may contain the term. -** -** If piLast is not NULL, then *piLast is set to the right-most child node -** that heads a sub-tree that may contain a term for which zTerm/nTerm is -** a prefix. -** -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. -*/ -static int fts3ScanInteriorNode( - const char *zTerm, /* Term to select leaves for */ - int nTerm, /* Size of term zTerm in bytes */ - const char *zNode, /* Buffer containing segment interior node */ - int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piFirst, /* OUT: Selected child node */ - sqlite3_int64 *piLast /* OUT: Selected child node */ -){ - int rc = SQLITE_OK; /* Return code */ - const char *zCsr = zNode; /* Cursor to iterate through node */ - const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ - char *zBuffer = 0; /* Buffer to load terms into */ - int nAlloc = 0; /* Size of allocated buffer */ - int isFirstTerm = 1; /* True when processing first term on page */ - sqlite3_int64 iChild; /* Block id of child node to descend to */ - - /* Skip over the 'height' varint that occurs at the start of every - ** interior node. Then load the blockid of the left-child of the b-tree - ** node into variable iChild. - ** - ** Even if the data structure on disk is corrupted, this (reading two - ** varints from the buffer) does not risk an overread. If zNode is a - ** root node, then the buffer comes from a SELECT statement. SQLite does - ** not make this guarantee explicitly, but in practice there are always - ** either more than 20 bytes of allocated space following the nNode bytes of - ** contents, or two zero bytes. Or, if the node is read from the %_segments - ** table, then there are always 20 bytes of zeroed padding following the - ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). - */ - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - if( zCsr>zEnd ){ - return FTS_CORRUPT_VTAB; - } - - while( zCsrzEnd ){ - rc = FTS_CORRUPT_VTAB; - goto finish_scan; - } - if( nPrefix+nSuffix>nAlloc ){ - char *zNew; - nAlloc = (nPrefix+nSuffix) * 2; - zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); - if( !zNew ){ - rc = SQLITE_NOMEM; - goto finish_scan; - } - zBuffer = zNew; - } - assert( zBuffer ); - memcpy(&zBuffer[nPrefix], zCsr, nSuffix); - nBuffer = nPrefix + nSuffix; - zCsr += nSuffix; - - /* Compare the term we are searching for with the term just loaded from - ** the interior node. If the specified term is greater than or equal - ** to the term from the interior node, then all terms on the sub-tree - ** headed by node iChild are smaller than zTerm. No need to search - ** iChild. - ** - ** If the interior node term is larger than the specified term, then - ** the tree headed by iChild may contain the specified term. - */ - cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); - if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ - *piFirst = iChild; - piFirst = 0; - } - - if( piLast && cmp<0 ){ - *piLast = iChild; - piLast = 0; - } - - iChild++; - }; - - if( piFirst ) *piFirst = iChild; - if( piLast ) *piLast = iChild; - - finish_scan: - sqlite3_free(zBuffer); - return rc; -} - - -/* -** The buffer pointed to by argument zNode (size nNode bytes) contains an -** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) -** contains a term. This function searches the sub-tree headed by the zNode -** node for the range of leaf nodes that may contain the specified term -** or terms for which the specified term is a prefix. -** -** If piLeaf is not NULL, then *piLeaf is set to the blockid of the -** left-most leaf node in the tree that may contain the specified term. -** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the -** right-most leaf node that may contain a term for which the specified -** term is a prefix. -** -** It is possible that the range of returned leaf nodes does not contain -** the specified term or any terms for which it is a prefix. However, if the -** segment does contain any such terms, they are stored within the identified -** range. Because this function only inspects interior segment nodes (and -** never loads leaf nodes into memory), it is not possible to be sure. -** -** If an error occurs, an error code other than SQLITE_OK is returned. -*/ -static int fts3SelectLeaf( - Fts3Table *p, /* Virtual table handle */ - const char *zTerm, /* Term to select leaves for */ - int nTerm, /* Size of term zTerm in bytes */ - const char *zNode, /* Buffer containing segment interior node */ - int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piLeaf, /* Selected leaf node */ - sqlite3_int64 *piLeaf2 /* Selected leaf node */ -){ - int rc = SQLITE_OK; /* Return code */ - int iHeight; /* Height of this node in tree */ - - assert( piLeaf || piLeaf2 ); - - fts3GetVarint32(zNode, &iHeight); - rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); - assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); - - if( rc==SQLITE_OK && iHeight>1 ){ - char *zBlob = 0; /* Blob read from %_segments table */ - int nBlob = 0; /* Size of zBlob in bytes */ - - if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ - rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); - if( rc==SQLITE_OK ){ - rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); - } - sqlite3_free(zBlob); - piLeaf = 0; - zBlob = 0; - } - - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); - } - if( rc==SQLITE_OK ){ - rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); - } - sqlite3_free(zBlob); - } - - return rc; -} - -/* -** This function is used to create delta-encoded serialized lists of FTS3 -** varints. Each call to this function appends a single varint to a list. -*/ -static void fts3PutDeltaVarint( - char **pp, /* IN/OUT: Output pointer */ - sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ - sqlite3_int64 iVal /* Write this value to the list */ -){ - assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); - *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); - *piPrev = iVal; -} - -/* -** When this function is called, *ppPoslist is assumed to point to the -** start of a position-list. After it returns, *ppPoslist points to the -** first byte after the position-list. -** -** A position list is list of positions (delta encoded) and columns for -** a single document record of a doclist. So, in other words, this -** routine advances *ppPoslist so that it points to the next docid in -** the doclist, or to the first byte past the end of the doclist. -** -** If pp is not NULL, then the contents of the position list are copied -** to *pp. *pp is set to point to the first byte past the last byte copied -** before this function returns. -*/ -static void fts3PoslistCopy(char **pp, char **ppPoslist){ - char *pEnd = *ppPoslist; - char c = 0; - - /* The end of a position list is marked by a zero encoded as an FTS3 - ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by - ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail - ** of some other, multi-byte, value. - ** - ** The following while-loop moves pEnd to point to the first byte that is not - ** immediately preceded by a byte with the 0x80 bit set. Then increments - ** pEnd once more so that it points to the byte immediately following the - ** last byte in the position-list. - */ - while( *pEnd | c ){ - c = *pEnd++ & 0x80; - testcase( c!=0 && (*pEnd)==0 ); - } - pEnd++; /* Advance past the POS_END terminator byte */ - - if( pp ){ - int n = (int)(pEnd - *ppPoslist); - char *p = *pp; - memcpy(p, *ppPoslist, n); - p += n; - *pp = p; - } - *ppPoslist = pEnd; -} - -/* -** When this function is called, *ppPoslist is assumed to point to the -** start of a column-list. After it returns, *ppPoslist points to the -** to the terminator (POS_COLUMN or POS_END) byte of the column-list. -** -** A column-list is list of delta-encoded positions for a single column -** within a single document within a doclist. -** -** The column-list is terminated either by a POS_COLUMN varint (1) or -** a POS_END varint (0). This routine leaves *ppPoslist pointing to -** the POS_COLUMN or POS_END that terminates the column-list. -** -** If pp is not NULL, then the contents of the column-list are copied -** to *pp. *pp is set to point to the first byte past the last byte copied -** before this function returns. The POS_COLUMN or POS_END terminator -** is not copied into *pp. -*/ -static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ - char *pEnd = *ppPoslist; - char c = 0; - - /* A column-list is terminated by either a 0x01 or 0x00 byte that is - ** not part of a multi-byte varint. - */ - while( 0xFE & (*pEnd | c) ){ - c = *pEnd++ & 0x80; - testcase( c!=0 && ((*pEnd)&0xfe)==0 ); - } - if( pp ){ - int n = (int)(pEnd - *ppPoslist); - char *p = *pp; - memcpy(p, *ppPoslist, n); - p += n; - *pp = p; - } - *ppPoslist = pEnd; -} - -/* -** Value used to signify the end of an position-list. This is safe because -** it is not possible to have a document with 2^31 terms. -*/ -#define POSITION_LIST_END 0x7fffffff - -/* -** This function is used to help parse position-lists. When this function is -** called, *pp may point to the start of the next varint in the position-list -** being parsed, or it may point to 1 byte past the end of the position-list -** (in which case **pp will be a terminator bytes POS_END (0) or -** (1)). -** -** If *pp points past the end of the current position-list, set *pi to -** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, -** increment the current value of *pi by the value read, and set *pp to -** point to the next value before returning. -** -** Before calling this routine *pi must be initialized to the value of -** the previous position, or zero if we are reading the first position -** in the position-list. Because positions are delta-encoded, the value -** of the previous position is needed in order to compute the value of -** the next position. -*/ -static void fts3ReadNextPos( - char **pp, /* IN/OUT: Pointer into position-list buffer */ - sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ -){ - if( (**pp)&0xFE ){ - fts3GetDeltaVarint(pp, pi); - *pi -= 2; - }else{ - *pi = POSITION_LIST_END; - } -} - -/* -** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by -** the value of iCol encoded as a varint to *pp. This will start a new -** column list. -** -** Set *pp to point to the byte just after the last byte written before -** returning (do not modify it if iCol==0). Return the total number of bytes -** written (0 if iCol==0). -*/ -static int fts3PutColNumber(char **pp, int iCol){ - int n = 0; /* Number of bytes written */ - if( iCol ){ - char *p = *pp; /* Output pointer */ - n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); - *p = 0x01; - *pp = &p[n]; - } - return n; -} - -/* -** Compute the union of two position lists. The output written -** into *pp contains all positions of both *pp1 and *pp2 in sorted -** order and with any duplicates removed. All pointers are -** updated appropriately. The caller is responsible for insuring -** that there is enough space in *pp to hold the complete output. -*/ -static void fts3PoslistMerge( - char **pp, /* Output buffer */ - char **pp1, /* Left input list */ - char **pp2 /* Right input list */ -){ - char *p = *pp; - char *p1 = *pp1; - char *p2 = *pp2; - - while( *p1 || *p2 ){ - int iCol1; /* The current column index in pp1 */ - int iCol2; /* The current column index in pp2 */ - - if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); - else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; - else iCol1 = 0; - - if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); - else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; - else iCol2 = 0; - - if( iCol1==iCol2 ){ - sqlite3_int64 i1 = 0; /* Last position from pp1 */ - sqlite3_int64 i2 = 0; /* Last position from pp2 */ - sqlite3_int64 iPrev = 0; - int n = fts3PutColNumber(&p, iCol1); - p1 += n; - p2 += n; - - /* At this point, both p1 and p2 point to the start of column-lists - ** for the same column (the column with index iCol1 and iCol2). - ** A column-list is a list of non-negative delta-encoded varints, each - ** incremented by 2 before being stored. Each list is terminated by a - ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists - ** and writes the results to buffer p. p is left pointing to the byte - ** after the list written. No terminator (POS_END or POS_COLUMN) is - ** written to the output. - */ - fts3GetDeltaVarint(&p1, &i1); - fts3GetDeltaVarint(&p2, &i2); - do { - fts3PutDeltaVarint(&p, &iPrev, (i1pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. -** when the *pp1 token appears before the *pp2 token, but not more than nToken -** slots before it. -** -** e.g. nToken==1 searches for adjacent positions. -*/ -static int fts3PoslistPhraseMerge( - char **pp, /* IN/OUT: Preallocated output buffer */ - int nToken, /* Maximum difference in token positions */ - int isSaveLeft, /* Save the left position */ - int isExact, /* If *pp1 is exactly nTokens before *pp2 */ - char **pp1, /* IN/OUT: Left input list */ - char **pp2 /* IN/OUT: Right input list */ -){ - char *p = *pp; - char *p1 = *pp1; - char *p2 = *pp2; - int iCol1 = 0; - int iCol2 = 0; - - /* Never set both isSaveLeft and isExact for the same invocation. */ - assert( isSaveLeft==0 || isExact==0 ); - - assert( p!=0 && *p1!=0 && *p2!=0 ); - if( *p1==POS_COLUMN ){ - p1++; - p1 += fts3GetVarint32(p1, &iCol1); - } - if( *p2==POS_COLUMN ){ - p2++; - p2 += fts3GetVarint32(p2, &iCol2); - } - - while( 1 ){ - if( iCol1==iCol2 ){ - char *pSave = p; - sqlite3_int64 iPrev = 0; - sqlite3_int64 iPos1 = 0; - sqlite3_int64 iPos2 = 0; - - if( iCol1 ){ - *p++ = POS_COLUMN; - p += sqlite3Fts3PutVarint(p, iCol1); - } - - assert( *p1!=POS_END && *p1!=POS_COLUMN ); - assert( *p2!=POS_END && *p2!=POS_COLUMN ); - fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; - fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; - - while( 1 ){ - if( iPos2==iPos1+nToken - || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) - ){ - sqlite3_int64 iSave; - iSave = isSaveLeft ? iPos1 : iPos2; - fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; - pSave = 0; - assert( p ); - } - if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ - if( (*p2&0xFE)==0 ) break; - fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; - }else{ - if( (*p1&0xFE)==0 ) break; - fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; - } - } - - if( pSave ){ - assert( pp && p ); - p = pSave; - } - - fts3ColumnlistCopy(0, &p1); - fts3ColumnlistCopy(0, &p2); - assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); - if( 0==*p1 || 0==*p2 ) break; - - p1++; - p1 += fts3GetVarint32(p1, &iCol1); - p2++; - p2 += fts3GetVarint32(p2, &iCol2); - } - - /* Advance pointer p1 or p2 (whichever corresponds to the smaller of - ** iCol1 and iCol2) so that it points to either the 0x00 that marks the - ** end of the position list, or the 0x01 that precedes the next - ** column-number in the position list. - */ - else if( iCol1=pEnd ){ - *pp = 0; - }else{ - sqlite3_int64 iVal; - *pp += sqlite3Fts3GetVarint(*pp, &iVal); - if( bDescIdx ){ - *pVal -= iVal; - }else{ - *pVal += iVal; - } - } -} - -/* -** This function is used to write a single varint to a buffer. The varint -** is written to *pp. Before returning, *pp is set to point 1 byte past the -** end of the value written. -** -** If *pbFirst is zero when this function is called, the value written to -** the buffer is that of parameter iVal. -** -** If *pbFirst is non-zero when this function is called, then the value -** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) -** (if bDescIdx is non-zero). -** -** Before returning, this function always sets *pbFirst to 1 and *piPrev -** to the value of parameter iVal. -*/ -static void fts3PutDeltaVarint3( - char **pp, /* IN/OUT: Output pointer */ - int bDescIdx, /* True for descending docids */ - sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ - int *pbFirst, /* IN/OUT: True after first int written */ - sqlite3_int64 iVal /* Write this value to the list */ -){ - sqlite3_int64 iWrite; - if( bDescIdx==0 || *pbFirst==0 ){ - iWrite = iVal - *piPrev; - }else{ - iWrite = *piPrev - iVal; - } - assert( *pbFirst || *piPrev==0 ); - assert( *pbFirst==0 || iWrite>0 ); - *pp += sqlite3Fts3PutVarint(*pp, iWrite); - *piPrev = iVal; - *pbFirst = 1; -} - - -/* -** This macro is used by various functions that merge doclists. The two -** arguments are 64-bit docid values. If the value of the stack variable -** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). -** Otherwise, (i2-i1). -** -** Using this makes it easier to write code that can merge doclists that are -** sorted in either ascending or descending order. -*/ -#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) - -/* -** This function does an "OR" merge of two doclists (output contains all -** positions contained in either argument doclist). If the docids in the -** input doclists are sorted in ascending order, parameter bDescDoclist -** should be false. If they are sorted in ascending order, it should be -** passed a non-zero value. -** -** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer -** containing the output doclist and SQLITE_OK is returned. In this case -** *pnOut is set to the number of bytes in the output doclist. -** -** If an error occurs, an SQLite error code is returned. The output values -** are undefined in this case. -*/ -static int fts3DoclistOrMerge( - int bDescDoclist, /* True if arguments are desc */ - char *a1, int n1, /* First doclist */ - char *a2, int n2, /* Second doclist */ - char **paOut, int *pnOut /* OUT: Malloc'd doclist */ -){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - char *pEnd1 = &a1[n1]; - char *pEnd2 = &a2[n2]; - char *p1 = a1; - char *p2 = a2; - char *p; - char *aOut; - int bFirstOut = 0; - - *paOut = 0; - *pnOut = 0; - - /* Allocate space for the output. Both the input and output doclists - ** are delta encoded. If they are in ascending order (bDescDoclist==0), - ** then the first docid in each list is simply encoded as a varint. For - ** each subsequent docid, the varint stored is the difference between the - ** current and previous docid (a positive number - since the list is in - ** ascending order). - ** - ** The first docid written to the output is therefore encoded using the - ** same number of bytes as it is in whichever of the input lists it is - ** read from. And each subsequent docid read from the same input list - ** consumes either the same or less bytes as it did in the input (since - ** the difference between it and the previous value in the output must - ** be a positive value less than or equal to the delta value read from - ** the input list). The same argument applies to all but the first docid - ** read from the 'other' list. And to the contents of all position lists - ** that will be copied and merged from the input to the output. - ** - ** However, if the first docid copied to the output is a negative number, - ** then the encoding of the first docid from the 'other' input list may - ** be larger in the output than it was in the input (since the delta value - ** may be a larger positive integer than the actual docid). - ** - ** The space required to store the output is therefore the sum of the - ** sizes of the two inputs, plus enough space for exactly one of the input - ** docids to grow. - ** - ** A symetric argument may be made if the doclists are in descending - ** order. - */ - aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1); - if( !aOut ) return SQLITE_NOMEM; - - p = aOut; - fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); - while( p1 || p2 ){ - sqlite3_int64 iDiff = DOCID_CMP(i1, i2); - - if( p2 && p1 && iDiff==0 ){ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - }else if( !p2 || (p1 && iDiff<0) ){ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - fts3PoslistCopy(&p, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - }else{ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); - fts3PoslistCopy(&p, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - } - } - - *paOut = aOut; - *pnOut = (int)(p-aOut); - assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 ); - return SQLITE_OK; -} - -/* -** This function does a "phrase" merge of two doclists. In a phrase merge, -** the output contains a copy of each position from the right-hand input -** doclist for which there is a position in the left-hand input doclist -** exactly nDist tokens before it. -** -** If the docids in the input doclists are sorted in ascending order, -** parameter bDescDoclist should be false. If they are sorted in ascending -** order, it should be passed a non-zero value. -** -** The right-hand input doclist is overwritten by this function. -*/ -static int fts3DoclistPhraseMerge( - int bDescDoclist, /* True if arguments are desc */ - int nDist, /* Distance from left to right (1=adjacent) */ - char *aLeft, int nLeft, /* Left doclist */ - char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ -){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - char *aRight = *paRight; - char *pEnd1 = &aLeft[nLeft]; - char *pEnd2 = &aRight[*pnRight]; - char *p1 = aLeft; - char *p2 = aRight; - char *p; - int bFirstOut = 0; - char *aOut; - - assert( nDist>0 ); - if( bDescDoclist ){ - aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX); - if( aOut==0 ) return SQLITE_NOMEM; - }else{ - aOut = aRight; - } - p = aOut; - - fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); - - while( p1 && p2 ){ - sqlite3_int64 iDiff = DOCID_CMP(i1, i2); - if( iDiff==0 ){ - char *pSave = p; - sqlite3_int64 iPrevSave = iPrev; - int bFirstOutSave = bFirstOut; - - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ - p = pSave; - iPrev = iPrevSave; - bFirstOut = bFirstOutSave; - } - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - }else if( iDiff<0 ){ - fts3PoslistCopy(0, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - }else{ - fts3PoslistCopy(0, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - } - } - - *pnRight = (int)(p - aOut); - if( bDescDoclist ){ - sqlite3_free(aRight); - *paRight = aOut; - } - - return SQLITE_OK; -} - -/* -** Argument pList points to a position list nList bytes in size. This -** function checks to see if the position list contains any entries for -** a token in position 0 (of any column). If so, it writes argument iDelta -** to the output buffer pOut, followed by a position list consisting only -** of the entries from pList at position 0, and terminated by an 0x00 byte. -** The value returned is the number of bytes written to pOut (if any). -*/ -SQLITE_PRIVATE int sqlite3Fts3FirstFilter( - sqlite3_int64 iDelta, /* Varint that may be written to pOut */ - char *pList, /* Position list (no 0x00 term) */ - int nList, /* Size of pList in bytes */ - char *pOut /* Write output here */ -){ - int nOut = 0; - int bWritten = 0; /* True once iDelta has been written */ - char *p = pList; - char *pEnd = &pList[nList]; - - if( *p!=0x01 ){ - if( *p==0x02 ){ - nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); - pOut[nOut++] = 0x02; - bWritten = 1; - } - fts3ColumnlistCopy(0, &p); - } - - while( paaOutput); i++){ - if( pTS->aaOutput[i] ){ - if( !aOut ){ - aOut = pTS->aaOutput[i]; - nOut = pTS->anOutput[i]; - pTS->aaOutput[i] = 0; - }else{ - int nNew; - char *aNew; - - int rc = fts3DoclistOrMerge(p->bDescIdx, - pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aOut); - return rc; - } - - sqlite3_free(pTS->aaOutput[i]); - sqlite3_free(aOut); - pTS->aaOutput[i] = 0; - aOut = aNew; - nOut = nNew; - } - } - } - - pTS->aaOutput[0] = aOut; - pTS->anOutput[0] = nOut; - return SQLITE_OK; -} - -/* -** Merge the doclist aDoclist/nDoclist into the TermSelect object passed -** as the first argument. The merge is an "OR" merge (see function -** fts3DoclistOrMerge() for details). -** -** This function is called with the doclist for each term that matches -** a queried prefix. It merges all these doclists into one, the doclist -** for the specified prefix. Since there can be a very large number of -** doclists to merge, the merging is done pair-wise using the TermSelect -** object. -** -** This function returns SQLITE_OK if the merge is successful, or an -** SQLite error code (SQLITE_NOMEM) if an error occurs. -*/ -static int fts3TermSelectMerge( - Fts3Table *p, /* FTS table handle */ - TermSelect *pTS, /* TermSelect object to merge into */ - char *aDoclist, /* Pointer to doclist */ - int nDoclist /* Size of aDoclist in bytes */ -){ - if( pTS->aaOutput[0]==0 ){ - /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). - ** - ** Add FTS3_VARINT_MAX bytes of unused space to the end of the - ** allocation. This is so as to ensure that the buffer is big enough - ** to hold the current doclist AND'd with any other doclist. If the - ** doclists are stored in order=ASC order, this padding would not be - ** required (since the size of [doclistA AND doclistB] is always less - ** than or equal to the size of [doclistA] in that case). But this is - ** not true for order=DESC. For example, a doclist containing (1, -1) - ** may be smaller than (-1), as in the first example the -1 may be stored - ** as a single-byte delta, whereas in the second it must be stored as a - ** FTS3_VARINT_MAX byte varint. - ** - ** Similar padding is added in the fts3DoclistOrMerge() function. - */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); - pTS->anOutput[0] = nDoclist; - if( pTS->aaOutput[0] ){ - memcpy(pTS->aaOutput[0], aDoclist, nDoclist); - }else{ - return SQLITE_NOMEM; - } - }else{ - char *aMerge = aDoclist; - int nMerge = nDoclist; - int iOut; - - for(iOut=0; iOutaaOutput); iOut++){ - if( pTS->aaOutput[iOut]==0 ){ - assert( iOut>0 ); - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; - break; - }else{ - char *aNew; - int nNew; - - int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, - pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew - ); - if( rc!=SQLITE_OK ){ - if( aMerge!=aDoclist ) sqlite3_free(aMerge); - return rc; - } - - if( aMerge!=aDoclist ) sqlite3_free(aMerge); - sqlite3_free(pTS->aaOutput[iOut]); - pTS->aaOutput[iOut] = 0; - - aMerge = aNew; - nMerge = nNew; - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; - } - } - } - } - return SQLITE_OK; -} - -/* -** Append SegReader object pNew to the end of the pCsr->apSegment[] array. -*/ -static int fts3SegReaderCursorAppend( - Fts3MultiSegReader *pCsr, - Fts3SegReader *pNew -){ - if( (pCsr->nSegment%16)==0 ){ - Fts3SegReader **apNew; - int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); - apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); - if( !apNew ){ - sqlite3Fts3SegReaderFree(pNew); - return SQLITE_NOMEM; - } - pCsr->apSegment = apNew; - } - pCsr->apSegment[pCsr->nSegment++] = pNew; - return SQLITE_OK; -} - -/* -** Add seg-reader objects to the Fts3MultiSegReader object passed as the -** 8th argument. -** -** This function returns SQLITE_OK if successful, or an SQLite error code -** otherwise. -*/ -static int fts3SegReaderCursor( - Fts3Table *p, /* FTS3 table handle */ - int iLangid, /* Language id */ - int iIndex, /* Index to search (from 0 to p->nIndex-1) */ - int iLevel, /* Level of segments to scan */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ -){ - int rc = SQLITE_OK; /* Error code */ - sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ - int rc2; /* Result of sqlite3_reset() */ - - /* If iLevel is less than 0 and this is not a scan, include a seg-reader - ** for the pending-terms. If this is a scan, then this call must be being - ** made by an fts4aux module, not an FTS table. In this case calling - ** Fts3SegReaderPending might segfault, as the data structures used by - ** fts4aux are not completely populated. So it's easiest to filter these - ** calls out here. */ - if( iLevel<0 && p->aIndex ){ - Fts3SegReader *pSeg = 0; - rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); - if( rc==SQLITE_OK && pSeg ){ - rc = fts3SegReaderCursorAppend(pCsr, pSeg); - } - } - - if( iLevel!=FTS3_SEGCURSOR_PENDING ){ - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); - } - - while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ - Fts3SegReader *pSeg = 0; - - /* Read the values returned by the SELECT into local variables. */ - sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); - sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); - sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); - int nRoot = sqlite3_column_bytes(pStmt, 4); - char const *zRoot = sqlite3_column_blob(pStmt, 4); - - /* If zTerm is not NULL, and this segment is not stored entirely on its - ** root node, the range of leaves scanned can be reduced. Do this. */ - if( iStartBlock && zTerm ){ - sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); - rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); - if( rc!=SQLITE_OK ) goto finished; - if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; - } - - rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, - (isPrefix==0 && isScan==0), - iStartBlock, iLeavesEndBlock, - iEndBlock, zRoot, nRoot, &pSeg - ); - if( rc!=SQLITE_OK ) goto finished; - rc = fts3SegReaderCursorAppend(pCsr, pSeg); - } - } - - finished: - rc2 = sqlite3_reset(pStmt); - if( rc==SQLITE_DONE ) rc = rc2; - - return rc; -} - -/* -** Set up a cursor object for iterating through a full-text index or a -** single level therein. -*/ -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( - Fts3Table *p, /* FTS3 table handle */ - int iLangid, /* Language-id to search */ - int iIndex, /* Index to search (from 0 to p->nIndex-1) */ - int iLevel, /* Level of segments to scan */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ -){ - assert( iIndex>=0 && iIndexnIndex ); - assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel>=0 - ); - assert( iLevelbase.pVtab; - - if( isPrefix ){ - for(i=1; bFound==0 && inIndex; i++){ - if( p->aIndex[i].nPrefix==nTerm ){ - bFound = 1; - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, - i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr - ); - pSegcsr->bLookup = 1; - } - } - - for(i=1; bFound==0 && inIndex; i++){ - if( p->aIndex[i].nPrefix==nTerm+1 ){ - bFound = 1; - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, - i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr - ); - if( rc==SQLITE_OK ){ - rc = fts3SegReaderCursorAddZero( - p, pCsr->iLangid, zTerm, nTerm, pSegcsr - ); - } - } - } - } - - if( bFound==0 ){ - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, - 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr - ); - pSegcsr->bLookup = !isPrefix; - } - } - - *ppSegcsr = pSegcsr; - return rc; -} - -/* -** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). -*/ -static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ - sqlite3Fts3SegReaderFinish(pSegcsr); - sqlite3_free(pSegcsr); -} - -/* -** This function retrieves the doclist for the specified term (or term -** prefix) from the database. -*/ -static int fts3TermSelect( - Fts3Table *p, /* Virtual table handle */ - Fts3PhraseToken *pTok, /* Token to query for */ - int iColumn, /* Column to query (or -ve for all columns) */ - int *pnOut, /* OUT: Size of buffer at *ppOut */ - char **ppOut /* OUT: Malloced result buffer */ -){ - int rc; /* Return code */ - Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ - TermSelect tsc; /* Object for pair-wise doclist merging */ - Fts3SegFilter filter; /* Segment term filter configuration */ - - pSegcsr = pTok->pSegcsr; - memset(&tsc, 0, sizeof(TermSelect)); - - filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS - | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) - | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) - | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); - filter.iCol = iColumn; - filter.zTerm = pTok->z; - filter.nTerm = pTok->n; - - rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); - while( SQLITE_OK==rc - && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) - ){ - rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); - } - - if( rc==SQLITE_OK ){ - rc = fts3TermSelectFinishMerge(p, &tsc); - } - if( rc==SQLITE_OK ){ - *ppOut = tsc.aaOutput[0]; - *pnOut = tsc.anOutput[0]; - }else{ - int i; - for(i=0; ipSegcsr = 0; - return rc; -} - -/* -** This function counts the total number of docids in the doclist stored -** in buffer aList[], size nList bytes. -** -** If the isPoslist argument is true, then it is assumed that the doclist -** contains a position-list following each docid. Otherwise, it is assumed -** that the doclist is simply a list of docids stored as delta encoded -** varints. -*/ -static int fts3DoclistCountDocids(char *aList, int nList){ - int nDoc = 0; /* Return value */ - if( aList ){ - char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ - char *p = aList; /* Cursor */ - while( peSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - }else{ - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); - rc = SQLITE_OK; - } - }else{ - rc = fts3EvalNext((Fts3Cursor *)pCursor); - } - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - return rc; -} - -/* -** The following are copied from sqliteInt.h. -** -** Constants for the largest and smallest possible 64-bit signed integers. -** These macros are designed to work correctly on both 32-bit and 64-bit -** compilers. -*/ -#ifndef SQLITE_AMALGAMATION -# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) -#endif - -/* -** If the numeric type of argument pVal is "integer", then return it -** converted to a 64-bit signed integer. Otherwise, return a copy of -** the second parameter, iDefault. -*/ -static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ - if( pVal ){ - int eType = sqlite3_value_numeric_type(pVal); - if( eType==SQLITE_INTEGER ){ - return sqlite3_value_int64(pVal); - } - } - return iDefault; -} - -/* -** This is the xFilter interface for the virtual table. See -** the virtual table xFilter method documentation for additional -** information. -** -** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against -** the %_content table. -** -** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry -** in the %_content table. -** -** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The -** column on the left-hand side of the MATCH operator is column -** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand -** side of the MATCH operator. -*/ -static int fts3FilterMethod( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, /* Strategy index */ - const char *idxStr, /* Unused */ - int nVal, /* Number of elements in apVal */ - sqlite3_value **apVal /* Arguments for the indexing scheme */ -){ - int rc = SQLITE_OK; - char *zSql; /* SQL statement used to access %_content */ - int eSearch; - Fts3Table *p = (Fts3Table *)pCursor->pVtab; - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ - sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ - sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ - sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ - int iIdx; - - UNUSED_PARAMETER(idxStr); - UNUSED_PARAMETER(nVal); - - eSearch = (idxNum & 0x0000FFFF); - assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); - assert( p->pSegments==0 ); - - /* Collect arguments into local variables */ - iIdx = 0; - if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; - assert( iIdx==nVal ); - - /* In case the cursor has been used before, clear it now. */ - sqlite3_finalize(pCsr->pStmt); - sqlite3_free(pCsr->aDoclist); - sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); - sqlite3Fts3ExprFree(pCsr->pExpr); - memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); - - /* Set the lower and upper bounds on docids to return */ - pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); - pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); - - if( idxStr ){ - pCsr->bDesc = (idxStr[0]=='D'); - }else{ - pCsr->bDesc = p->bDescIdx; - } - pCsr->eSearch = (i16)eSearch; - - if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ - int iCol = eSearch-FTS3_FULLTEXT_SEARCH; - const char *zQuery = (const char *)sqlite3_value_text(pCons); - - if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ - return SQLITE_NOMEM; - } - - pCsr->iLangid = 0; - if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); - - assert( p->base.zErrMsg==0 ); - rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, - p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, - &p->base.zErrMsg - ); - if( rc!=SQLITE_OK ){ - return rc; - } - - rc = fts3EvalStart(pCsr); - sqlite3Fts3SegmentsClose(p); - if( rc!=SQLITE_OK ) return rc; - pCsr->pNextId = pCsr->aDoclist; - pCsr->iPrevId = 0; - } - - /* Compile a SELECT statement for this cursor. For a full-table-scan, the - ** statement loops through all rows of the %_content table. For a - ** full-text query or docid lookup, the statement retrieves a single - ** row by docid. - */ - if( eSearch==FTS3_FULLSCAN_SEARCH ){ - if( pDocidGe || pDocidLe ){ - zSql = sqlite3_mprintf( - "SELECT %s WHERE rowid BETWEEN %lld AND %lld ORDER BY rowid %s", - p->zReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid, - (pCsr->bDesc ? "DESC" : "ASC") - ); - }else{ - zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", - p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") - ); - } - if( zSql ){ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); - }else{ - rc = SQLITE_NOMEM; - } - }else if( eSearch==FTS3_DOCID_SEARCH ){ - rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); - if( rc==SQLITE_OK ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); - } - } - if( rc!=SQLITE_OK ) return rc; - - return fts3NextMethod(pCursor); -} - -/* -** This is the xEof method of the virtual table. SQLite calls this -** routine to find out if it has reached the end of a result set. -*/ -static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ - return ((Fts3Cursor *)pCursor)->isEof; -} - -/* -** This is the xRowid method. The SQLite core calls this routine to -** retrieve the rowid for the current row of the result set. fts3 -** exposes %_content.docid as the rowid for the virtual table. The -** rowid should be written to *pRowid. -*/ -static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - *pRowid = pCsr->iPrevId; - return SQLITE_OK; -} - -/* -** This is the xColumn method, called by SQLite to request a value from -** the row that the supplied cursor currently points to. -** -** If: -** -** (iCol < p->nColumn) -> The value of the iCol'th user column. -** (iCol == p->nColumn) -> Magic column with the same name as the table. -** (iCol == p->nColumn+1) -> Docid column -** (iCol == p->nColumn+2) -> Langid column -*/ -static int fts3ColumnMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ - int iCol /* Index of column to read value from */ -){ - int rc = SQLITE_OK; /* Return Code */ - Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - Fts3Table *p = (Fts3Table *)pCursor->pVtab; - - /* The column value supplied by SQLite must be in range. */ - assert( iCol>=0 && iCol<=p->nColumn+2 ); - - if( iCol==p->nColumn+1 ){ - /* This call is a request for the "docid" column. Since "docid" is an - ** alias for "rowid", use the xRowid() method to obtain the value. - */ - sqlite3_result_int64(pCtx, pCsr->iPrevId); - }else if( iCol==p->nColumn ){ - /* The extra column whose name is the same as the table. - ** Return a blob which is a pointer to the cursor. */ - sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ - sqlite3_result_int64(pCtx, pCsr->iLangid); - }else{ - /* The requested column is either a user column (one that contains - ** indexed data), or the language-id column. */ - rc = fts3CursorSeek(0, pCsr); - - if( rc==SQLITE_OK ){ - if( iCol==p->nColumn+2 ){ - int iLangid = 0; - if( p->zLanguageid ){ - iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); - } - sqlite3_result_int(pCtx, iLangid); - }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); - } - } - } - - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - return rc; -} - -/* -** This function is the implementation of the xUpdate callback used by -** FTS3 virtual tables. It is invoked by SQLite each time a row is to be -** inserted, updated or deleted. -*/ -static int fts3UpdateMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - int nArg, /* Size of argument array */ - sqlite3_value **apVal, /* Array of arguments */ - sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ -){ - return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); -} - -/* -** Implementation of xSync() method. Flush the contents of the pending-terms -** hash-table to the database. -*/ -static int fts3SyncMethod(sqlite3_vtab *pVtab){ - - /* Following an incremental-merge operation, assuming that the input - ** segments are not completely consumed (the usual case), they are updated - ** in place to remove the entries that have already been merged. This - ** involves updating the leaf block that contains the smallest unmerged - ** entry and each block (if any) between the leaf and the root node. So - ** if the height of the input segment b-trees is N, and input segments - ** are merged eight at a time, updating the input segments at the end - ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually - ** small - often between 0 and 2. So the overhead of the incremental - ** merge is somewhere between 8 and 24 blocks. To avoid this overhead - ** dwarfing the actual productive work accomplished, the incremental merge - ** is only attempted if it will write at least 64 leaf blocks. Hence - ** nMinMerge. - ** - ** Of course, updating the input segments also involves deleting a bunch - ** of blocks from the segments table. But this is not considered overhead - ** as it would also be required by a crisis-merge that used the same input - ** segments. - */ - const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ - - Fts3Table *p = (Fts3Table*)pVtab; - int rc = sqlite3Fts3PendingTermsFlush(p); - - if( rc==SQLITE_OK - && p->nLeafAdd>(nMinMerge/16) - && p->nAutoincrmerge && p->nAutoincrmerge!=0xff - ){ - int mxLevel = 0; /* Maximum relative level value in db */ - int A; /* Incr-merge parameter A */ - - rc = sqlite3Fts3MaxLevel(p, &mxLevel); - assert( rc==SQLITE_OK || mxLevel==0 ); - A = p->nLeafAdd * mxLevel; - A += (A/2); - if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); - } - sqlite3Fts3SegmentsClose(p); - return rc; -} - -/* -** If it is currently unknown whether or not the FTS table has an %_stat -** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat -** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code -** if an error occurs. -*/ -static int fts3SetHasStat(Fts3Table *p){ - int rc = SQLITE_OK; - if( p->bHasStat==2 ){ - const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; - char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); - if( zSql ){ - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); - rc = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) p->bHasStat = bHasStat; - } - sqlite3_free(zSql); - }else{ - rc = SQLITE_NOMEM; - } - } - return rc; -} - -/* -** Implementation of xBegin() method. -*/ -static int fts3BeginMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table*)pVtab; - UNUSED_PARAMETER(pVtab); - assert( p->pSegments==0 ); - assert( p->nPendingData==0 ); - assert( p->inTransaction!=1 ); - TESTONLY( p->inTransaction = 1 ); - TESTONLY( p->mxSavepoint = -1; ); - p->nLeafAdd = 0; - return fts3SetHasStat(p); -} - -/* -** Implementation of xCommit() method. This is a no-op. The contents of -** the pending-terms hash-table have already been flushed into the database -** by fts3SyncMethod(). -*/ -static int fts3CommitMethod(sqlite3_vtab *pVtab){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); - UNUSED_PARAMETER(pVtab); - assert( p->nPendingData==0 ); - assert( p->inTransaction!=0 ); - assert( p->pSegments==0 ); - TESTONLY( p->inTransaction = 0 ); - TESTONLY( p->mxSavepoint = -1; ); - return SQLITE_OK; -} - -/* -** Implementation of xRollback(). Discard the contents of the pending-terms -** hash-table. Any changes made to the database are reverted by SQLite. -*/ -static int fts3RollbackMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table*)pVtab; - sqlite3Fts3PendingTermsClear(p); - assert( p->inTransaction!=0 ); - TESTONLY( p->inTransaction = 0 ); - TESTONLY( p->mxSavepoint = -1; ); - return SQLITE_OK; -} - -/* -** When called, *ppPoslist must point to the byte immediately following the -** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function -** moves *ppPoslist so that it instead points to the first byte of the -** same position list. -*/ -static void fts3ReversePoslist(char *pStart, char **ppPoslist){ - char *p = &(*ppPoslist)[-2]; - char c = 0; - - /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ - while( p>pStart && (c=*p--)==0 ); - - /* Search backwards for a varint with value zero (the end of the previous - ** poslist). This is an 0x00 byte preceded by some byte that does not - ** have the 0x80 bit set. */ - while( p>pStart && (*p & 0x80) | c ){ - c = *p--; - } - assert( p==pStart || c==0 ); - - /* At this point p points to that preceding byte without the 0x80 bit - ** set. So to find the start of the poslist, skip forward 2 bytes then - ** over a varint. - ** - ** Normally. The other case is that p==pStart and the poslist to return - ** is the first in the doclist. In this case do not skip forward 2 bytes. - ** The second part of the if condition (c==0 && *ppPoslist>&p[2]) - ** is required for cases where the first byte of a doclist and the - ** doclist is empty. For example, if the first docid is 10, a doclist - ** that begins with: - ** - ** 0x0A 0x00 - */ - if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; } - while( *p++&0x80 ); - *ppPoslist = p; -} - -/* -** Helper function used by the implementation of the overloaded snippet(), -** offsets() and optimize() SQL functions. -** -** If the value passed as the third argument is a blob of size -** sizeof(Fts3Cursor*), then the blob contents are copied to the -** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error -** message is written to context pContext and SQLITE_ERROR returned. The -** string passed via zFunc is used as part of the error message. -*/ -static int fts3FunctionArg( - sqlite3_context *pContext, /* SQL function call context */ - const char *zFunc, /* Function name */ - sqlite3_value *pVal, /* argv[0] passed to function */ - Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ -){ - Fts3Cursor *pRet; - if( sqlite3_value_type(pVal)!=SQLITE_BLOB - || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) - ){ - char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); - sqlite3_result_error(pContext, zErr, -1); - sqlite3_free(zErr); - return SQLITE_ERROR; - } - memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); - *ppCsr = pRet; - return SQLITE_OK; -} - -/* -** Implementation of the snippet() function for FTS3 -*/ -static void fts3SnippetFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of apVal[] array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - const char *zStart = ""; - const char *zEnd = ""; - const char *zEllipsis = "..."; - int iCol = -1; - int nToken = 15; /* Default number of tokens in snippet */ - - /* There must be at least one argument passed to this function (otherwise - ** the non-overloaded version would have been called instead of this one). - */ - assert( nVal>=1 ); - - if( nVal>6 ){ - sqlite3_result_error(pContext, - "wrong number of arguments to function snippet()", -1); - return; - } - if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; - - switch( nVal ){ - case 6: nToken = sqlite3_value_int(apVal[5]); - case 5: iCol = sqlite3_value_int(apVal[4]); - case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); - case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); - case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); - } - if( !zEllipsis || !zEnd || !zStart ){ - sqlite3_result_error_nomem(pContext); - }else if( nToken==0 ){ - sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); - }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ - sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); - } -} - -/* -** Implementation of the offsets() function for FTS3 -*/ -static void fts3OffsetsFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - - UNUSED_PARAMETER(nVal); - - assert( nVal==1 ); - if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; - assert( pCsr ); - if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ - sqlite3Fts3Offsets(pContext, pCsr); - } -} - -/* -** Implementation of the special optimize() function for FTS3. This -** function merges all segments in the database to a single segment. -** Example usage is: -** -** SELECT optimize(t) FROM t LIMIT 1; -** -** where 't' is the name of an FTS3 table. -*/ -static void fts3OptimizeFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - int rc; /* Return code */ - Fts3Table *p; /* Virtual table handle */ - Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ - - UNUSED_PARAMETER(nVal); - - assert( nVal==1 ); - if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; - p = (Fts3Table *)pCursor->base.pVtab; - assert( p ); - - rc = sqlite3Fts3Optimize(p); - - switch( rc ){ - case SQLITE_OK: - sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); - break; - case SQLITE_DONE: - sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); - break; - default: - sqlite3_result_error_code(pContext, rc); - break; - } -} - -/* -** Implementation of the matchinfo() function for FTS3 -*/ -static void fts3MatchinfoFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - assert( nVal==1 || nVal==2 ); - if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ - const char *zArg = 0; - if( nVal>1 ){ - zArg = (const char *)sqlite3_value_text(apVal[1]); - } - sqlite3Fts3Matchinfo(pContext, pCsr, zArg); - } -} - -/* -** This routine implements the xFindFunction method for the FTS3 -** virtual table. -*/ -static int fts3FindFunctionMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - int nArg, /* Number of SQL function arguments */ - const char *zName, /* Name of SQL function */ - void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ - void **ppArg /* Unused */ -){ - struct Overloaded { - const char *zName; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } aOverload[] = { - { "snippet", fts3SnippetFunc }, - { "offsets", fts3OffsetsFunc }, - { "optimize", fts3OptimizeFunc }, - { "matchinfo", fts3MatchinfoFunc }, - }; - int i; /* Iterator variable */ - - UNUSED_PARAMETER(pVtab); - UNUSED_PARAMETER(nArg); - UNUSED_PARAMETER(ppArg); - - for(i=0; idb; /* Database connection */ - int rc; /* Return Code */ - - /* At this point it must be known if the %_stat table exists or not. - ** So bHasStat may not be 2. */ - rc = fts3SetHasStat(p); - - /* As it happens, the pending terms table is always empty here. This is - ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction - ** always opens a savepoint transaction. And the xSavepoint() method - ** flushes the pending terms table. But leave the (no-op) call to - ** PendingTermsFlush() in in case that changes. - */ - assert( p->nPendingData==0 ); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3PendingTermsFlush(p); - } - - if( p->zContentTbl==0 ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", - p->zDb, p->zName, zName - ); - } - - if( p->bHasDocsize ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", - p->zDb, p->zName, zName - ); - } - if( p->bHasStat ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", - p->zDb, p->zName, zName - ); - } - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", - p->zDb, p->zName, zName - ); - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", - p->zDb, p->zName, zName - ); - return rc; -} - -/* -** The xSavepoint() method. -** -** Flush the contents of the pending-terms table to disk. -*/ -static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ - int rc = SQLITE_OK; - UNUSED_PARAMETER(iSavepoint); - assert( ((Fts3Table *)pVtab)->inTransaction ); - assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); - TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ - rc = fts3SyncMethod(pVtab); - } - return rc; -} - -/* -** The xRelease() method. -** -** This is a no-op. -*/ -static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); - UNUSED_PARAMETER(iSavepoint); - UNUSED_PARAMETER(pVtab); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint-1 ); - return SQLITE_OK; -} - -/* -** The xRollbackTo() method. -** -** Discard the contents of the pending terms table. -*/ -static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts3Table *p = (Fts3Table*)pVtab; - UNUSED_PARAMETER(iSavepoint); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint ); - sqlite3Fts3PendingTermsClear(p); - return SQLITE_OK; -} - -static const sqlite3_module fts3Module = { - /* iVersion */ 2, - /* xCreate */ fts3CreateMethod, - /* xConnect */ fts3ConnectMethod, - /* xBestIndex */ fts3BestIndexMethod, - /* xDisconnect */ fts3DisconnectMethod, - /* xDestroy */ fts3DestroyMethod, - /* xOpen */ fts3OpenMethod, - /* xClose */ fts3CloseMethod, - /* xFilter */ fts3FilterMethod, - /* xNext */ fts3NextMethod, - /* xEof */ fts3EofMethod, - /* xColumn */ fts3ColumnMethod, - /* xRowid */ fts3RowidMethod, - /* xUpdate */ fts3UpdateMethod, - /* xBegin */ fts3BeginMethod, - /* xSync */ fts3SyncMethod, - /* xCommit */ fts3CommitMethod, - /* xRollback */ fts3RollbackMethod, - /* xFindFunction */ fts3FindFunctionMethod, - /* xRename */ fts3RenameMethod, - /* xSavepoint */ fts3SavepointMethod, - /* xRelease */ fts3ReleaseMethod, - /* xRollbackTo */ fts3RollbackToMethod, -}; - -/* -** This function is registered as the module destructor (called when an -** FTS3 enabled database connection is closed). It frees the memory -** allocated for the tokenizer hash table. -*/ -static void hashDestroy(void *p){ - Fts3Hash *pHash = (Fts3Hash *)p; - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); -} - -/* -** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are -** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c -** respectively. The following three forward declarations are for functions -** declared in these files used to retrieve the respective implementations. -** -** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed -** to by the argument to point to the "simple" tokenizer implementation. -** And so on. -*/ -SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); -SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#ifndef SQLITE_DISABLE_FTS3_UNICODE -SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); -#endif -#ifdef SQLITE_ENABLE_ICU -SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#endif - -/* -** Initialize the fts3 extension. If this extension is built as part -** of the sqlite library, then this function is called directly by -** SQLite. If fts3 is built as a dynamically loadable extension, this -** function is called by the sqlite3_extension_init() entry point. -*/ -SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ - int rc = SQLITE_OK; - Fts3Hash *pHash = 0; - const sqlite3_tokenizer_module *pSimple = 0; - const sqlite3_tokenizer_module *pPorter = 0; -#ifndef SQLITE_DISABLE_FTS3_UNICODE - const sqlite3_tokenizer_module *pUnicode = 0; -#endif - -#ifdef SQLITE_ENABLE_ICU - const sqlite3_tokenizer_module *pIcu = 0; - sqlite3Fts3IcuTokenizerModule(&pIcu); -#endif - -#ifndef SQLITE_DISABLE_FTS3_UNICODE - sqlite3Fts3UnicodeTokenizer(&pUnicode); -#endif - -#ifdef SQLITE_TEST - rc = sqlite3Fts3InitTerm(db); - if( rc!=SQLITE_OK ) return rc; -#endif - - rc = sqlite3Fts3InitAux(db); - if( rc!=SQLITE_OK ) return rc; - - sqlite3Fts3SimpleTokenizerModule(&pSimple); - sqlite3Fts3PorterTokenizerModule(&pPorter); - - /* Allocate and initialize the hash-table used to store tokenizers. */ - pHash = sqlite3_malloc(sizeof(Fts3Hash)); - if( !pHash ){ - rc = SQLITE_NOMEM; - }else{ - sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); - } - - /* Load the built-in tokenizers into the hash table */ - if( rc==SQLITE_OK ){ - if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) - || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) - -#ifndef SQLITE_DISABLE_FTS3_UNICODE - || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) -#endif -#ifdef SQLITE_ENABLE_ICU - || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) -#endif - ){ - rc = SQLITE_NOMEM; - } - } - -#ifdef SQLITE_TEST - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ExprInitTestInterface(db); - } -#endif - - /* Create the virtual table wrapper around the hash-table and overload - ** the two scalar functions. If this is successful, register the - ** module with sqlite. - */ - if( SQLITE_OK==rc - && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) - ){ - rc = sqlite3_create_module_v2( - db, "fts3", &fts3Module, (void *)pHash, hashDestroy - ); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_module_v2( - db, "fts4", &fts3Module, (void *)pHash, 0 - ); - } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3InitTok(db, (void *)pHash); - } - return rc; - } - - - /* An error has occurred. Delete the hash table and return the error code. */ - assert( rc!=SQLITE_OK ); - if( pHash ){ - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); - } - return rc; -} - -/* -** Allocate an Fts3MultiSegReader for each token in the expression headed -** by pExpr. -** -** An Fts3SegReader object is a cursor that can seek or scan a range of -** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple -** Fts3SegReader objects internally to provide an interface to seek or scan -** within the union of all segments of a b-tree. Hence the name. -** -** If the allocated Fts3MultiSegReader just seeks to a single entry in a -** segment b-tree (if the term is not a prefix or it is a prefix for which -** there exists prefix b-tree of the right length) then it may be traversed -** and merged incrementally. Otherwise, it has to be merged into an in-memory -** doclist and then traversed. -*/ -static void fts3EvalAllocateReaders( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Allocate readers for this expression */ - int *pnToken, /* OUT: Total number of tokens in phrase. */ - int *pnOr, /* OUT: Total number of OR nodes in expr. */ - int *pRc /* IN/OUT: Error code */ -){ - if( pExpr && SQLITE_OK==*pRc ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - int i; - int nToken = pExpr->pPhrase->nToken; - *pnToken += nToken; - for(i=0; ipPhrase->aToken[i]; - int rc = fts3TermSegReaderCursor(pCsr, - pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr - ); - if( rc!=SQLITE_OK ){ - *pRc = rc; - return; - } - } - assert( pExpr->pPhrase->iDoclistToken==0 ); - pExpr->pPhrase->iDoclistToken = -1; - }else{ - *pnOr += (pExpr->eType==FTSQUERY_OR); - fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); - fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); - } - } -} - -/* -** Arguments pList/nList contain the doclist for token iToken of phrase p. -** It is merged into the main doclist stored in p->doclist.aAll/nAll. -** -** This function assumes that pList points to a buffer allocated using -** sqlite3_malloc(). This function takes responsibility for eventually -** freeing the buffer. -** -** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. -*/ -static int fts3EvalPhraseMergeToken( - Fts3Table *pTab, /* FTS Table pointer */ - Fts3Phrase *p, /* Phrase to merge pList/nList into */ - int iToken, /* Token pList/nList corresponds to */ - char *pList, /* Pointer to doclist */ - int nList /* Number of bytes in pList */ -){ - int rc = SQLITE_OK; - assert( iToken!=p->iDoclistToken ); - - if( pList==0 ){ - sqlite3_free(p->doclist.aAll); - p->doclist.aAll = 0; - p->doclist.nAll = 0; - } - - else if( p->iDoclistToken<0 ){ - p->doclist.aAll = pList; - p->doclist.nAll = nList; - } - - else if( p->doclist.aAll==0 ){ - sqlite3_free(pList); - } - - else { - char *pLeft; - char *pRight; - int nLeft; - int nRight; - int nDiff; - - if( p->iDoclistTokendoclist.aAll; - nLeft = p->doclist.nAll; - pRight = pList; - nRight = nList; - nDiff = iToken - p->iDoclistToken; - }else{ - pRight = p->doclist.aAll; - nRight = p->doclist.nAll; - pLeft = pList; - nLeft = nList; - nDiff = p->iDoclistToken - iToken; - } - - rc = fts3DoclistPhraseMerge( - pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight - ); - sqlite3_free(pLeft); - p->doclist.aAll = pRight; - p->doclist.nAll = nRight; - } - - if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; - return rc; -} - -/* -** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist -** does not take deferred tokens into account. -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. -*/ -static int fts3EvalPhraseLoad( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p /* Phrase object */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int iToken; - int rc = SQLITE_OK; - - for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ - Fts3PhraseToken *pToken = &p->aToken[iToken]; - assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); - - if( pToken->pSegcsr ){ - int nThis = 0; - char *pThis = 0; - rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); - if( rc==SQLITE_OK ){ - rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); - } - } - assert( pToken->pSegcsr==0 ); - } - - return rc; -} - -/* -** This function is called on each phrase after the position lists for -** any deferred tokens have been loaded into memory. It updates the phrases -** current position list to include only those positions that are really -** instances of the phrase (after considering deferred tokens). If this -** means that the phrase does not appear in the current row, doclist.pList -** and doclist.nList are both zeroed. -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. -*/ -static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ - int iToken; /* Used to iterate through phrase tokens */ - char *aPoslist = 0; /* Position list for deferred tokens */ - int nPoslist = 0; /* Number of bytes in aPoslist */ - int iPrev = -1; /* Token number of previous deferred token */ - - assert( pPhrase->doclist.bFreeList==0 ); - - for(iToken=0; iTokennToken; iToken++){ - Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; - Fts3DeferredToken *pDeferred = pToken->pDeferred; - - if( pDeferred ){ - char *pList; - int nList; - int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); - if( rc!=SQLITE_OK ) return rc; - - if( pList==0 ){ - sqlite3_free(aPoslist); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - return SQLITE_OK; - - }else if( aPoslist==0 ){ - aPoslist = pList; - nPoslist = nList; - - }else{ - char *aOut = pList; - char *p1 = aPoslist; - char *p2 = aOut; - - assert( iPrev>=0 ); - fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); - sqlite3_free(aPoslist); - aPoslist = pList; - nPoslist = (int)(aOut - aPoslist); - if( nPoslist==0 ){ - sqlite3_free(aPoslist); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - return SQLITE_OK; - } - } - iPrev = iToken; - } - } - - if( iPrev>=0 ){ - int nMaxUndeferred = pPhrase->iDoclistToken; - if( nMaxUndeferred<0 ){ - pPhrase->doclist.pList = aPoslist; - pPhrase->doclist.nList = nPoslist; - pPhrase->doclist.iDocid = pCsr->iPrevId; - pPhrase->doclist.bFreeList = 1; - }else{ - int nDistance; - char *p1; - char *p2; - char *aOut; - - if( nMaxUndeferred>iPrev ){ - p1 = aPoslist; - p2 = pPhrase->doclist.pList; - nDistance = nMaxUndeferred - iPrev; - }else{ - p1 = pPhrase->doclist.pList; - p2 = aPoslist; - nDistance = iPrev - nMaxUndeferred; - } - - aOut = (char *)sqlite3_malloc(nPoslist+8); - if( !aOut ){ - sqlite3_free(aPoslist); - return SQLITE_NOMEM; - } - - pPhrase->doclist.pList = aOut; - if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ - pPhrase->doclist.bFreeList = 1; - pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); - }else{ - sqlite3_free(aOut); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - } - sqlite3_free(aPoslist); - } - } - - return SQLITE_OK; -} - -/* -** Maximum number of tokens a phrase may have to be considered for the -** incremental doclists strategy. -*/ -#define MAX_INCR_PHRASE_TOKENS 4 - -/* -** This function is called for each Fts3Phrase in a full-text query -** expression to initialize the mechanism for returning rows. Once this -** function has been called successfully on an Fts3Phrase, it may be -** used with fts3EvalPhraseNext() to iterate through the matching docids. -** -** If parameter bOptOk is true, then the phrase may (or may not) use the -** incremental loading strategy. Otherwise, the entire doclist is loaded into -** memory within this call. -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. -*/ -static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; /* Error code */ - int i; - - /* Determine if doclists may be loaded from disk incrementally. This is - ** possible if the bOptOk argument is true, the FTS doclists will be - ** scanned in forward order, and the phrase consists of - ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" - ** tokens or prefix tokens that cannot use a prefix-index. */ - int bHaveIncr = 0; - int bIncrOk = (bOptOk - && pCsr->bDesc==pTab->bDescIdx - && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 -#ifdef SQLITE_TEST - && pTab->bNoIncrDoclist==0 -#endif - ); - for(i=0; bIncrOk==1 && inToken; i++){ - Fts3PhraseToken *pToken = &p->aToken[i]; - if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ - bIncrOk = 0; - } - if( pToken->pSegcsr ) bHaveIncr = 1; - } - - if( bIncrOk && bHaveIncr ){ - /* Use the incremental approach. */ - int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); - for(i=0; rc==SQLITE_OK && inToken; i++){ - Fts3PhraseToken *pToken = &p->aToken[i]; - Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; - if( pSegcsr ){ - rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); - } - } - p->bIncr = 1; - }else{ - /* Load the full doclist for the phrase into memory. */ - rc = fts3EvalPhraseLoad(pCsr, p); - p->bIncr = 0; - } - - assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); - return rc; -} - -/* -** This function is used to iterate backwards (from the end to start) -** through doclists. It is used by this module to iterate through phrase -** doclists in reverse and by the fts3_write.c module to iterate through -** pending-terms lists when writing to databases with "order=desc". -** -** The doclist may be sorted in ascending (parameter bDescIdx==0) or -** descending (parameter bDescIdx==1) order of docid. Regardless, this -** function iterates from the end of the doclist to the beginning. -*/ -SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( - int bDescIdx, /* True if the doclist is desc */ - char *aDoclist, /* Pointer to entire doclist */ - int nDoclist, /* Length of aDoclist in bytes */ - char **ppIter, /* IN/OUT: Iterator pointer */ - sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ - int *pnList, /* OUT: List length pointer */ - u8 *pbEof /* OUT: End-of-file flag */ -){ - char *p = *ppIter; - - assert( nDoclist>0 ); - assert( *pbEof==0 ); - assert( p || *piDocid==0 ); - assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); - - if( p==0 ){ - sqlite3_int64 iDocid = 0; - char *pNext = 0; - char *pDocid = aDoclist; - char *pEnd = &aDoclist[nDoclist]; - int iMul = 1; - - while( pDocid0 ); - assert( *pbEof==0 ); - assert( p || *piDocid==0 ); - assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); - - if( p==0 ){ - p = aDoclist; - p += sqlite3Fts3GetVarint(p, piDocid); - }else{ - fts3PoslistCopy(0, &p); - while( p<&aDoclist[nDoclist] && *p==0 ) p++; - if( p>=&aDoclist[nDoclist] ){ - *pbEof = 1; - }else{ - sqlite3_int64 iVar; - p += sqlite3Fts3GetVarint(p, &iVar); - *piDocid += ((bDescIdx ? -1 : 1) * iVar); - } - } - - *ppIter = p; -} - -/* -** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof -** to true if EOF is reached. -*/ -static void fts3EvalDlPhraseNext( - Fts3Table *pTab, - Fts3Doclist *pDL, - u8 *pbEof -){ - char *pIter; /* Used to iterate through aAll */ - char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ - - if( pDL->pNextDocid ){ - pIter = pDL->pNextDocid; - }else{ - pIter = pDL->aAll; - } - - if( pIter>=pEnd ){ - /* We have already reached the end of this doclist. EOF. */ - *pbEof = 1; - }else{ - sqlite3_int64 iDelta; - pIter += sqlite3Fts3GetVarint(pIter, &iDelta); - if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ - pDL->iDocid += iDelta; - }else{ - pDL->iDocid -= iDelta; - } - pDL->pList = pIter; - fts3PoslistCopy(0, &pIter); - pDL->nList = (int)(pIter - pDL->pList); - - /* pIter now points just past the 0x00 that terminates the position- - ** list for document pDL->iDocid. However, if this position-list was - ** edited in place by fts3EvalNearTrim(), then pIter may not actually - ** point to the start of the next docid value. The following line deals - ** with this case by advancing pIter past the zero-padding added by - ** fts3EvalNearTrim(). */ - while( pIterpNextDocid = pIter; - assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); - *pbEof = 0; - } -} - -/* -** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). -*/ -typedef struct TokenDoclist TokenDoclist; -struct TokenDoclist { - int bIgnore; - sqlite3_int64 iDocid; - char *pList; - int nList; -}; - -/* -** Token pToken is an incrementally loaded token that is part of a -** multi-token phrase. Advance it to the next matching document in the -** database and populate output variable *p with the details of the new -** entry. Or, if the iterator has reached EOF, set *pbEof to true. -** -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. -*/ -static int incrPhraseTokenNext( - Fts3Table *pTab, /* Virtual table handle */ - Fts3Phrase *pPhrase, /* Phrase to advance token of */ - int iToken, /* Specific token to advance */ - TokenDoclist *p, /* OUT: Docid and doclist for new entry */ - u8 *pbEof /* OUT: True if iterator is at EOF */ -){ - int rc = SQLITE_OK; - - if( pPhrase->iDoclistToken==iToken ){ - assert( p->bIgnore==0 ); - assert( pPhrase->aToken[iToken].pSegcsr==0 ); - fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); - p->pList = pPhrase->doclist.pList; - p->nList = pPhrase->doclist.nList; - p->iDocid = pPhrase->doclist.iDocid; - }else{ - Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; - assert( pToken->pDeferred==0 ); - assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); - if( pToken->pSegcsr ){ - assert( p->bIgnore==0 ); - rc = sqlite3Fts3MsrIncrNext( - pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList - ); - if( p->pList==0 ) *pbEof = 1; - }else{ - p->bIgnore = 1; - } - } - - return rc; -} - - -/* -** The phrase iterator passed as the second argument: -** -** * features at least one token that uses an incremental doclist, and -** -** * does not contain any deferred tokens. -** -** Advance it to the next matching documnent in the database and populate -** the Fts3Doclist.pList and nList fields. -** -** If there is no "next" entry and no error occurs, then *pbEof is set to -** 1 before returning. Otherwise, if no error occurs and the iterator is -** successfully advanced, *pbEof is set to 0. -** -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. -*/ -static int fts3EvalIncrPhraseNext( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p, /* Phrase object to advance to next docid */ - u8 *pbEof /* OUT: Set to 1 if EOF */ -){ - int rc = SQLITE_OK; - Fts3Doclist *pDL = &p->doclist; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - u8 bEof = 0; - - /* This is only called if it is guaranteed that the phrase has at least - ** one incremental token. In which case the bIncr flag is set. */ - assert( p->bIncr==1 ); - - if( p->nToken==1 && p->bIncr ){ - rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, - &pDL->iDocid, &pDL->pList, &pDL->nList - ); - if( pDL->pList==0 ) bEof = 1; - }else{ - int bDescDoclist = pCsr->bDesc; - struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; - - memset(a, 0, sizeof(a)); - assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); - assert( p->iDoclistTokennToken && bEof==0; i++){ - rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); - if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ - iMax = a[i].iDocid; - bMaxSet = 1; - } - } - assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); - assert( rc!=SQLITE_OK || bMaxSet ); - - /* Keep advancing iterators until they all point to the same document */ - for(i=0; inToken; i++){ - while( rc==SQLITE_OK && bEof==0 - && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 - ){ - rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); - if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ - iMax = a[i].iDocid; - i = 0; - } - } - } - - /* Check if the current entries really are a phrase match */ - if( bEof==0 ){ - int nList = 0; - int nByte = a[p->nToken-1].nList; - char *aDoclist = sqlite3_malloc(nByte+1); - if( !aDoclist ) return SQLITE_NOMEM; - memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); - - for(i=0; i<(p->nToken-1); i++){ - if( a[i].bIgnore==0 ){ - char *pL = a[i].pList; - char *pR = aDoclist; - char *pOut = aDoclist; - int nDist = p->nToken-1-i; - int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); - if( res==0 ) break; - nList = (int)(pOut - aDoclist); - } - } - if( i==(p->nToken-1) ){ - pDL->iDocid = iMax; - pDL->pList = aDoclist; - pDL->nList = nList; - pDL->bFreeList = 1; - break; - } - sqlite3_free(aDoclist); - } - } - } - - *pbEof = bEof; - return rc; -} - -/* -** Attempt to move the phrase iterator to point to the next matching docid. -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. -** -** If there is no "next" entry and no error occurs, then *pbEof is set to -** 1 before returning. Otherwise, if no error occurs and the iterator is -** successfully advanced, *pbEof is set to 0. -*/ -static int fts3EvalPhraseNext( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p, /* Phrase object to advance to next docid */ - u8 *pbEof /* OUT: Set to 1 if EOF */ -){ - int rc = SQLITE_OK; - Fts3Doclist *pDL = &p->doclist; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - - if( p->bIncr ){ - rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); - }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ - sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, - &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof - ); - pDL->pList = pDL->pNextDocid; - }else{ - fts3EvalDlPhraseNext(pTab, pDL, pbEof); - } - - return rc; -} - -/* -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, fts3EvalPhraseStart() is called on all phrases within the -** expression. Also the Fts3Expr.bDeferred variable is set to true for any -** expressions for which all descendent tokens are deferred. -** -** If parameter bOptOk is zero, then it is guaranteed that the -** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for -** each phrase in the expression (subject to deferred token processing). -** Or, if bOptOk is non-zero, then one or more tokens within the expression -** may be loaded incrementally, meaning doclist.aAll/nAll is not available. -** -** If an error occurs within this function, *pRc is set to an SQLite error -** code before returning. -*/ -static void fts3EvalStartReaders( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pExpr, /* Expression to initialize phrases in */ - int *pRc /* IN/OUT: Error code */ -){ - if( pExpr && SQLITE_OK==*pRc ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - int nToken = pExpr->pPhrase->nToken; - if( nToken ){ - int i; - for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; - } - pExpr->bDeferred = (i==nToken); - } - *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); - }else{ - fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); - fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); - pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); - } - } -} - -/* -** An array of the following structures is assembled as part of the process -** of selecting tokens to defer before the query starts executing (as part -** of the xFilter() method). There is one element in the array for each -** token in the FTS expression. -** -** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong -** to phrases that are connected only by AND and NEAR operators (not OR or -** NOT). When determining tokens to defer, each AND/NEAR cluster is considered -** separately. The root of a tokens AND/NEAR cluster is stored in -** Fts3TokenAndCost.pRoot. -*/ -typedef struct Fts3TokenAndCost Fts3TokenAndCost; -struct Fts3TokenAndCost { - Fts3Phrase *pPhrase; /* The phrase the token belongs to */ - int iToken; /* Position of token in phrase */ - Fts3PhraseToken *pToken; /* The token itself */ - Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ - int nOvfl; /* Number of overflow pages to load doclist */ - int iCol; /* The column the token must match */ -}; - -/* -** This function is used to populate an allocated Fts3TokenAndCost array. -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, if an error occurs during execution, *pRc is set to an -** SQLite error code. -*/ -static void fts3EvalTokenCosts( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ - Fts3Expr *pExpr, /* Expression to consider */ - Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ - Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==SQLITE_OK ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int i; - for(i=0; *pRc==SQLITE_OK && inToken; i++){ - Fts3TokenAndCost *pTC = (*ppTC)++; - pTC->pPhrase = pPhrase; - pTC->iToken = i; - pTC->pRoot = pRoot; - pTC->pToken = &pPhrase->aToken[i]; - pTC->iCol = pPhrase->iColumn; - *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); - } - }else if( pExpr->eType!=FTSQUERY_NOT ){ - assert( pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND - || pExpr->eType==FTSQUERY_NEAR - ); - assert( pExpr->pLeft && pExpr->pRight ); - if( pExpr->eType==FTSQUERY_OR ){ - pRoot = pExpr->pLeft; - **ppOr = pRoot; - (*ppOr)++; - } - fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); - if( pExpr->eType==FTSQUERY_OR ){ - pRoot = pExpr->pRight; - **ppOr = pRoot; - (*ppOr)++; - } - fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); - } - } -} - -/* -** Determine the average document (row) size in pages. If successful, -** write this value to *pnPage and return SQLITE_OK. Otherwise, return -** an SQLite error code. -** -** The average document size in pages is calculated by first calculating -** determining the average size in bytes, B. If B is less than the amount -** of data that will fit on a single leaf page of an intkey table in -** this database, then the average docsize is 1. Otherwise, it is 1 plus -** the number of overflow pages consumed by a record B bytes in size. -*/ -static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ - if( pCsr->nRowAvg==0 ){ - /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ - int rc; - Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - sqlite3_stmt *pStmt; - sqlite3_int64 nDoc = 0; - sqlite3_int64 nByte = 0; - const char *pEnd; - const char *a; - - rc = sqlite3Fts3SelectDoctotal(p, &pStmt); - if( rc!=SQLITE_OK ) return rc; - a = sqlite3_column_blob(pStmt, 0); - assert( a ); - - pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( anDoc = nDoc; - pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); - assert( pCsr->nRowAvg>0 ); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; - } - - *pnPage = pCsr->nRowAvg; - return SQLITE_OK; -} - -/* -** This function is called to select the tokens (if any) that will be -** deferred. The array aTC[] has already been populated when this is -** called. -** -** This function is called once for each AND/NEAR cluster in the -** expression. Each invocation determines which tokens to defer within -** the cluster with root node pRoot. See comments above the definition -** of struct Fts3TokenAndCost for more details. -** -** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() -** called on each token to defer. Otherwise, an SQLite error code is -** returned. -*/ -static int fts3EvalSelectDeferred( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pRoot, /* Consider tokens with this root node */ - Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ - int nTC /* Number of entries in aTC[] */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int nDocSize = 0; /* Number of pages per doc loaded */ - int rc = SQLITE_OK; /* Return code */ - int ii; /* Iterator variable for various purposes */ - int nOvfl = 0; /* Total overflow pages used by doclists */ - int nToken = 0; /* Total number of tokens in cluster */ - - int nMinEst = 0; /* The minimum count for any phrase so far. */ - int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ - - /* Tokens are never deferred for FTS tables created using the content=xxx - ** option. The reason being that it is not guaranteed that the content - ** table actually contains the same data as the index. To prevent this from - ** causing any problems, the deferred token optimization is completely - ** disabled for content=xxx tables. */ - if( pTab->zContentTbl ){ - return SQLITE_OK; - } - - /* Count the tokens in this AND/NEAR cluster. If none of the doclists - ** associated with the tokens spill onto overflow pages, or if there is - ** only 1 token, exit early. No tokens to defer in this case. */ - for(ii=0; ii0 ); - - - /* Iterate through all tokens in this AND/NEAR cluster, in ascending order - ** of the number of overflow pages that will be loaded by the pager layer - ** to retrieve the entire doclist for the token from the full-text index. - ** Load the doclists for tokens that are either: - ** - ** a. The cheapest token in the entire query (i.e. the one visited by the - ** first iteration of this loop), or - ** - ** b. Part of a multi-token phrase. - ** - ** After each token doclist is loaded, merge it with the others from the - ** same phrase and count the number of documents that the merged doclist - ** contains. Set variable "nMinEst" to the smallest number of documents in - ** any phrase doclist for which 1 or more token doclists have been loaded. - ** Let nOther be the number of other phrases for which it is certain that - ** one or more tokens will not be deferred. - ** - ** Then, for each token, defer it if loading the doclist would result in - ** loading N or more overflow pages into memory, where N is computed as: - ** - ** (nMinEst + 4^nOther - 1) / (4^nOther) - */ - for(ii=0; iinOvfl) - ){ - pTC = &aTC[iTC]; - } - } - assert( pTC ); - - if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ - /* The number of overflow pages to load for this (and therefore all - ** subsequent) tokens is greater than the estimated number of pages - ** that will be loaded if all subsequent tokens are deferred. - */ - Fts3PhraseToken *pToken = pTC->pToken; - rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); - fts3SegReaderCursorFree(pToken->pSegcsr); - pToken->pSegcsr = 0; - }else{ - /* Set nLoad4 to the value of (4^nOther) for the next iteration of the - ** for-loop. Except, limit the value to 2^24 to prevent it from - ** overflowing the 32-bit integer it is stored in. */ - if( ii<12 ) nLoad4 = nLoad4*4; - - if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ - /* Either this is the cheapest token in the entire query, or it is - ** part of a multi-token phrase. Either way, the entire doclist will - ** (eventually) be loaded into memory. It may as well be now. */ - Fts3PhraseToken *pToken = pTC->pToken; - int nList = 0; - char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); - assert( rc==SQLITE_OK || pList==0 ); - if( rc==SQLITE_OK ){ - rc = fts3EvalPhraseMergeToken( - pTab, pTC->pPhrase, pTC->iToken,pList,nList - ); - } - if( rc==SQLITE_OK ){ - int nCount; - nCount = fts3DoclistCountDocids( - pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll - ); - if( ii==0 || nCountpToken = 0; - } - - return rc; -} - -/* -** This function is called from within the xFilter method. It initializes -** the full-text query currently stored in pCsr->pExpr. To iterate through -** the results of a query, the caller does: -** -** fts3EvalStart(pCsr); -** while( 1 ){ -** fts3EvalNext(pCsr); -** if( pCsr->bEof ) break; -** ... return row pCsr->iPrevId to the caller ... -** } -*/ -static int fts3EvalStart(Fts3Cursor *pCsr){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; - int nToken = 0; - int nOr = 0; - - /* Allocate a MultiSegReader for each token in the expression. */ - fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); - - /* Determine which, if any, tokens in the expression should be deferred. */ -#ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ - Fts3TokenAndCost *aTC; - Fts3Expr **apOr; - aTC = (Fts3TokenAndCost *)sqlite3_malloc( - sizeof(Fts3TokenAndCost) * nToken - + sizeof(Fts3Expr *) * nOr * 2 - ); - apOr = (Fts3Expr **)&aTC[nToken]; - - if( !aTC ){ - rc = SQLITE_NOMEM; - }else{ - int ii; - Fts3TokenAndCost *pTC = aTC; - Fts3Expr **ppOr = apOr; - - fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); - nToken = (int)(pTC-aTC); - nOr = (int)(ppOr-apOr); - - if( rc==SQLITE_OK ){ - rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); - for(ii=0; rc==SQLITE_OK && iipExpr, &rc); - return rc; -} - -/* -** Invalidate the current position list for phrase pPhrase. -*/ -static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ - if( pPhrase->doclist.bFreeList ){ - sqlite3_free(pPhrase->doclist.pList); - } - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - pPhrase->doclist.bFreeList = 0; -} - -/* -** This function is called to edit the position list associated with -** the phrase object passed as the fifth argument according to a NEAR -** condition. For example: -** -** abc NEAR/5 "def ghi" -** -** Parameter nNear is passed the NEAR distance of the expression (5 in -** the example above). When this function is called, *paPoslist points to -** the position list, and *pnToken is the number of phrase tokens in, the -** phrase on the other side of the NEAR operator to pPhrase. For example, -** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to -** the position list associated with phrase "abc". -** -** All positions in the pPhrase position list that are not sufficiently -** close to a position in the *paPoslist position list are removed. If this -** leaves 0 positions, zero is returned. Otherwise, non-zero. -** -** Before returning, *paPoslist is set to point to the position lsit -** associated with pPhrase. And *pnToken is set to the number of tokens in -** pPhrase. -*/ -static int fts3EvalNearTrim( - int nNear, /* NEAR distance. As in "NEAR/nNear". */ - char *aTmp, /* Temporary space to use */ - char **paPoslist, /* IN/OUT: Position list */ - int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ - Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ -){ - int nParam1 = nNear + pPhrase->nToken; - int nParam2 = nNear + *pnToken; - int nNew; - char *p2; - char *pOut; - int res; - - assert( pPhrase->doclist.pList ); - - p2 = pOut = pPhrase->doclist.pList; - res = fts3PoslistNearMerge( - &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 - ); - if( res ){ - nNew = (int)(pOut - pPhrase->doclist.pList) - 1; - assert( pPhrase->doclist.pList[nNew]=='\0' ); - assert( nNew<=pPhrase->doclist.nList && nNew>0 ); - memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); - pPhrase->doclist.nList = nNew; - *paPoslist = pPhrase->doclist.pList; - *pnToken = pPhrase->nToken; - } - - return res; -} - -/* -** This function is a no-op if *pRc is other than SQLITE_OK when it is called. -** Otherwise, it advances the expression passed as the second argument to -** point to the next matching row in the database. Expressions iterate through -** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, -** or descending if it is non-zero. -** -** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if -** successful, the following variables in pExpr are set: -** -** Fts3Expr.bEof (non-zero if EOF - there is no next row) -** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) -** -** If the expression is of type FTSQUERY_PHRASE, and the expression is not -** at EOF, then the following variables are populated with the position list -** for the phrase for the visited row: -** -** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) -** FTs3Expr.pPhrase->doclist.pList (pointer to position list) -** -** It says above that this function advances the expression to the next -** matching row. This is usually true, but there are the following exceptions: -** -** 1. Deferred tokens are not taken into account. If a phrase consists -** entirely of deferred tokens, it is assumed to match every row in -** the db. In this case the position-list is not populated at all. -** -** Or, if a phrase contains one or more deferred tokens and one or -** more non-deferred tokens, then the expression is advanced to the -** next possible match, considering only non-deferred tokens. In other -** words, if the phrase is "A B C", and "B" is deferred, the expression -** is advanced to the next row that contains an instance of "A * C", -** where "*" may match any single token. The position list in this case -** is populated as for "A * C" before returning. -** -** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is -** advanced to point to the next row that matches "x AND y". -** -** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is -** really a match, taking into account deferred tokens and NEAR operators. -*/ -static void fts3EvalNextRow( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pExpr, /* Expr. to advance to next matching row */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==SQLITE_OK ){ - int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ - assert( pExpr->bEof==0 ); - pExpr->bStart = 1; - - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - assert( !pLeft->bDeferred || !pRight->bDeferred ); - - if( pLeft->bDeferred ){ - /* LHS is entirely deferred. So we assume it matches every row. - ** Advance the RHS iterator to find the next row visited. */ - fts3EvalNextRow(pCsr, pRight, pRc); - pExpr->iDocid = pRight->iDocid; - pExpr->bEof = pRight->bEof; - }else if( pRight->bDeferred ){ - /* RHS is entirely deferred. So we assume it matches every row. - ** Advance the LHS iterator to find the next row visited. */ - fts3EvalNextRow(pCsr, pLeft, pRc); - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - }else{ - /* Neither the RHS or LHS are deferred. */ - fts3EvalNextRow(pCsr, pLeft, pRc); - fts3EvalNextRow(pCsr, pRight, pRc); - while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( iDiff==0 ) break; - if( iDiff<0 ){ - fts3EvalNextRow(pCsr, pLeft, pRc); - }else{ - fts3EvalNextRow(pCsr, pRight, pRc); - } - } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = (pLeft->bEof || pRight->bEof); - if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ - if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){ - Fts3Doclist *pDl = &pRight->pPhrase->doclist; - while( *pRc==SQLITE_OK && pRight->bEof==0 ){ - memset(pDl->pList, 0, pDl->nList); - fts3EvalNextRow(pCsr, pRight, pRc); - } - } - if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ - Fts3Doclist *pDl = &pLeft->pPhrase->doclist; - while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ - memset(pDl->pList, 0, pDl->nList); - fts3EvalNextRow(pCsr, pLeft, pRc); - } - } - } - } - break; - } - - case FTSQUERY_OR: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - - assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); - assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - fts3EvalNextRow(pCsr, pLeft, pRc); - }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ - fts3EvalNextRow(pCsr, pRight, pRc); - }else{ - fts3EvalNextRow(pCsr, pLeft, pRc); - fts3EvalNextRow(pCsr, pRight, pRc); - } - - pExpr->bEof = (pLeft->bEof && pRight->bEof); - iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - pExpr->iDocid = pLeft->iDocid; - }else{ - pExpr->iDocid = pRight->iDocid; - } - - break; - } - - case FTSQUERY_NOT: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - - if( pRight->bStart==0 ){ - fts3EvalNextRow(pCsr, pRight, pRc); - assert( *pRc!=SQLITE_OK || pRight->bStart ); - } - - fts3EvalNextRow(pCsr, pLeft, pRc); - if( pLeft->bEof==0 ){ - while( !*pRc - && !pRight->bEof - && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 - ){ - fts3EvalNextRow(pCsr, pRight, pRc); - } - } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - break; - } - - default: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalInvalidatePoslist(pPhrase); - *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); - pExpr->iDocid = pPhrase->doclist.iDocid; - break; - } - } - } -} - -/* -** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR -** cluster, then this function returns 1 immediately. -** -** Otherwise, it checks if the current row really does match the NEAR -** expression, using the data currently stored in the position lists -** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. -** -** If the current row is a match, the position list associated with each -** phrase in the NEAR expression is edited in place to contain only those -** phrase instances sufficiently close to their peers to satisfy all NEAR -** constraints. In this case it returns 1. If the NEAR expression does not -** match the current row, 0 is returned. The position lists may or may not -** be edited if 0 is returned. -*/ -static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ - int res = 1; - - /* The following block runs if pExpr is the root of a NEAR query. - ** For example, the query: - ** - ** "w" NEAR "x" NEAR "y" NEAR "z" - ** - ** which is represented in tree form as: - ** - ** | - ** +--NEAR--+ <-- root of NEAR query - ** | | - ** +--NEAR--+ "z" - ** | | - ** +--NEAR--+ "y" - ** | | - ** "w" "x" - ** - ** The right-hand child of a NEAR node is always a phrase. The - ** left-hand child may be either a phrase or a NEAR node. There are - ** no exceptions to this - it's the way the parser in fts3_expr.c works. - */ - if( *pRc==SQLITE_OK - && pExpr->eType==FTSQUERY_NEAR - && pExpr->bEof==0 - && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) - ){ - Fts3Expr *p; - int nTmp = 0; /* Bytes of temp space */ - char *aTmp; /* Temp space for PoslistNearMerge() */ - - /* Allocate temporary working space. */ - for(p=pExpr; p->pLeft; p=p->pLeft){ - nTmp += p->pRight->pPhrase->doclist.nList; - } - nTmp += p->pPhrase->doclist.nList; - if( nTmp==0 ){ - res = 0; - }else{ - aTmp = sqlite3_malloc(nTmp*2); - if( !aTmp ){ - *pRc = SQLITE_NOMEM; - res = 0; - }else{ - char *aPoslist = p->pPhrase->doclist.pList; - int nToken = p->pPhrase->nToken; - - for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ - Fts3Phrase *pPhrase = p->pRight->pPhrase; - int nNear = p->nNear; - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } - - aPoslist = pExpr->pRight->pPhrase->doclist.pList; - nToken = pExpr->pRight->pPhrase->nToken; - for(p=pExpr->pLeft; p && res; p=p->pLeft){ - int nNear; - Fts3Phrase *pPhrase; - assert( p->pParent && p->pParent->pLeft==p ); - nNear = p->pParent->nNear; - pPhrase = ( - p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase - ); - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } - } - - sqlite3_free(aTmp); - } - } - - return res; -} - -/* -** This function is a helper function for sqlite3Fts3EvalTestDeferred(). -** Assuming no error occurs or has occurred, It returns non-zero if the -** expression passed as the second argument matches the row that pCsr -** currently points to, or zero if it does not. -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** If an error occurs during execution of this function, *pRc is set to -** the appropriate SQLite error code. In this case the returned value is -** undefined. -*/ -static int fts3EvalTestExpr( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ - int *pRc /* IN/OUT: Error code */ -){ - int bHit = 1; /* Return value */ - if( *pRc==SQLITE_OK ){ - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: - bHit = ( - fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) - && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) - && fts3EvalNearTest(pExpr, pRc) - ); - - /* If the NEAR expression does not match any rows, zero the doclist for - ** all phrases involved in the NEAR. This is because the snippet(), - ** offsets() and matchinfo() functions are not supposed to recognize - ** any instances of phrases that are part of unmatched NEAR queries. - ** For example if this expression: - ** - ** ... MATCH 'a OR (b NEAR c)' - ** - ** is matched against a row containing: - ** - ** 'a b d e' - ** - ** then any snippet() should ony highlight the "a" term, not the "b" - ** (as "b" is part of a non-matching NEAR clause). - */ - if( bHit==0 - && pExpr->eType==FTSQUERY_NEAR - && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) - ){ - Fts3Expr *p; - for(p=pExpr; p->pPhrase==0; p=p->pLeft){ - if( p->pRight->iDocid==pCsr->iPrevId ){ - fts3EvalInvalidatePoslist(p->pRight->pPhrase); - } - } - if( p->iDocid==pCsr->iPrevId ){ - fts3EvalInvalidatePoslist(p->pPhrase); - } - } - - break; - - case FTSQUERY_OR: { - int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); - int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); - bHit = bHit1 || bHit2; - break; - } - - case FTSQUERY_NOT: - bHit = ( - fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) - && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) - ); - break; - - default: { -#ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( pCsr->pDeferred - && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) - ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); - if( pExpr->bDeferred ){ - fts3EvalInvalidatePoslist(pPhrase); - } - *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); - bHit = (pPhrase->doclist.pList!=0); - pExpr->iDocid = pCsr->iPrevId; - }else -#endif - { - bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); - } - break; - } - } - } - return bHit; -} - -/* -** This function is called as the second part of each xNext operation when -** iterating through the results of a full-text query. At this point the -** cursor points to a row that matches the query expression, with the -** following caveats: -** -** * Up until this point, "NEAR" operators in the expression have been -** treated as "AND". -** -** * Deferred tokens have not yet been considered. -** -** If *pRc is not SQLITE_OK when this function is called, it immediately -** returns 0. Otherwise, it tests whether or not after considering NEAR -** operators and deferred tokens the current row is still a match for the -** expression. It returns 1 if both of the following are true: -** -** 1. *pRc is SQLITE_OK when this function returns, and -** -** 2. After scanning the current FTS table row for the deferred tokens, -** it is determined that the row does *not* match the query. -** -** Or, if no error occurs and it seems the current row does match the FTS -** query, return 0. -*/ -SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ - int rc = *pRc; - int bMiss = 0; - if( rc==SQLITE_OK ){ - - /* If there are one or more deferred tokens, load the current row into - ** memory and scan it to determine the position list for each deferred - ** token. Then, see if this row is really a match, considering deferred - ** tokens and NEAR operators (neither of which were taken into account - ** earlier, by fts3EvalNextRow()). - */ - if( pCsr->pDeferred ){ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); - } - } - bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); - - /* Free the position-lists accumulated for each deferred token above. */ - sqlite3Fts3FreeDeferredDoclists(pCsr); - *pRc = rc; - } - return (rc==SQLITE_OK && bMiss); -} - -/* -** Advance to the next document that matches the FTS expression in -** Fts3Cursor.pExpr. -*/ -static int fts3EvalNext(Fts3Cursor *pCsr){ - int rc = SQLITE_OK; /* Return Code */ - Fts3Expr *pExpr = pCsr->pExpr; - assert( pCsr->isEof==0 ); - if( pExpr==0 ){ - pCsr->isEof = 1; - }else{ - do { - if( pCsr->isRequireSeek==0 ){ - sqlite3_reset(pCsr->pStmt); - } - assert( sqlite3_data_count(pCsr->pStmt)==0 ); - fts3EvalNextRow(pCsr, pExpr, &rc); - pCsr->isEof = pExpr->bEof; - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - pCsr->iPrevId = pExpr->iDocid; - }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); - } - - /* Check if the cursor is past the end of the docid range specified - ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ - if( rc==SQLITE_OK && ( - (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) - || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) - )){ - pCsr->isEof = 1; - } - - return rc; -} - -/* -** Restart interation for expression pExpr so that the next call to -** fts3EvalNext() visits the first row. Do not allow incremental -** loading or merging of phrase doclists for this iteration. -** -** If *pRc is other than SQLITE_OK when this function is called, it is -** a no-op. If an error occurs within this function, *pRc is set to an -** SQLite error code before returning. -*/ -static void fts3EvalRestart( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int *pRc -){ - if( pExpr && *pRc==SQLITE_OK ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - - if( pPhrase ){ - fts3EvalInvalidatePoslist(pPhrase); - if( pPhrase->bIncr ){ - int i; - for(i=0; inToken; i++){ - Fts3PhraseToken *pToken = &pPhrase->aToken[i]; - assert( pToken->pDeferred==0 ); - if( pToken->pSegcsr ){ - sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); - } - } - *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); - } - pPhrase->doclist.pNextDocid = 0; - pPhrase->doclist.iDocid = 0; - pPhrase->pOrPoslist = 0; - } - - pExpr->iDocid = 0; - pExpr->bEof = 0; - pExpr->bStart = 0; - - fts3EvalRestart(pCsr, pExpr->pLeft, pRc); - fts3EvalRestart(pCsr, pExpr->pRight, pRc); - } -} - -/* -** After allocating the Fts3Expr.aMI[] array for each phrase in the -** expression rooted at pExpr, the cursor iterates through all rows matched -** by pExpr, calling this function for each row. This function increments -** the values in Fts3Expr.aMI[] according to the position-list currently -** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase -** expression nodes. -*/ -static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - if( pPhrase && pPhrase->doclist.pList ){ - int iCol = 0; - char *p = pPhrase->doclist.pList; - - assert( *p ); - while( 1 ){ - u8 c = 0; - int iCnt = 0; - while( 0xFE & (*p | c) ){ - if( (c&0x80)==0 ) iCnt++; - c = *p++ & 0x80; - } - - /* aMI[iCol*3 + 1] = Number of occurrences - ** aMI[iCol*3 + 2] = Number of rows containing at least one instance - */ - pExpr->aMI[iCol*3 + 1] += iCnt; - pExpr->aMI[iCol*3 + 2] += (iCnt>0); - if( *p==0x00 ) break; - p++; - p += fts3GetVarint32(p, &iCol); - } - } - - fts3EvalUpdateCounts(pExpr->pLeft); - fts3EvalUpdateCounts(pExpr->pRight); - } -} - -/* -** Expression pExpr must be of type FTSQUERY_PHRASE. -** -** If it is not already allocated and populated, this function allocates and -** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part -** of a NEAR expression, then it also allocates and populates the same array -** for all other phrases that are part of the NEAR expression. -** -** SQLITE_OK is returned if the aMI[] array is successfully allocated and -** populated. Otherwise, if an error occurs, an SQLite error code is returned. -*/ -static int fts3EvalGatherStats( - Fts3Cursor *pCsr, /* Cursor object */ - Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ -){ - int rc = SQLITE_OK; /* Return code */ - - assert( pExpr->eType==FTSQUERY_PHRASE ); - if( pExpr->aMI==0 ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - Fts3Expr *pRoot; /* Root of NEAR expression */ - Fts3Expr *p; /* Iterator used for several purposes */ - - sqlite3_int64 iPrevId = pCsr->iPrevId; - sqlite3_int64 iDocid; - u8 bEof; - - /* Find the root of the NEAR expression */ - pRoot = pExpr; - while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ - pRoot = pRoot->pParent; - } - iDocid = pRoot->iDocid; - bEof = pRoot->bEof; - assert( pRoot->bStart ); - - /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ - for(p=pRoot; p; p=p->pLeft){ - Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); - assert( pE->aMI==0 ); - pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); - if( !pE->aMI ) return SQLITE_NOMEM; - memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); - } - - fts3EvalRestart(pCsr, pRoot, &rc); - - while( pCsr->isEof==0 && rc==SQLITE_OK ){ - - do { - /* Ensure the %_content statement is reset. */ - if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); - assert( sqlite3_data_count(pCsr->pStmt)==0 ); - - /* Advance to the next document */ - fts3EvalNextRow(pCsr, pRoot, &rc); - pCsr->isEof = pRoot->bEof; - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - pCsr->iPrevId = pRoot->iDocid; - }while( pCsr->isEof==0 - && pRoot->eType==FTSQUERY_NEAR - && sqlite3Fts3EvalTestDeferred(pCsr, &rc) - ); - - if( rc==SQLITE_OK && pCsr->isEof==0 ){ - fts3EvalUpdateCounts(pRoot); - } - } - - pCsr->isEof = 0; - pCsr->iPrevId = iPrevId; - - if( bEof ){ - pRoot->bEof = bEof; - }else{ - /* Caution: pRoot may iterate through docids in ascending or descending - ** order. For this reason, even though it seems more defensive, the - ** do loop can not be written: - ** - ** do {...} while( pRoot->iDocidbEof==0 ); - }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); - } - } - return rc; -} - -/* -** This function is used by the matchinfo() module to query a phrase -** expression node for the following information: -** -** 1. The total number of occurrences of the phrase in each column of -** the FTS table (considering all rows), and -** -** 2. For each column, the number of rows in the table for which the -** column contains at least one instance of the phrase. -** -** If no error occurs, SQLITE_OK is returned and the values for each column -** written into the array aiOut as follows: -** -** aiOut[iCol*3 + 1] = Number of occurrences -** aiOut[iCol*3 + 2] = Number of rows containing at least one instance -** -** Caveats: -** -** * If a phrase consists entirely of deferred tokens, then all output -** values are set to the number of documents in the table. In other -** words we assume that very common tokens occur exactly once in each -** column of each row of the table. -** -** * If a phrase contains some deferred tokens (and some non-deferred -** tokens), count the potential occurrence identified by considering -** the non-deferred tokens instead of actual phrase occurrences. -** -** * If the phrase is part of a NEAR expression, then only phrase instances -** that meet the NEAR constraint are included in the counts. -*/ -SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Phrase expression */ - u32 *aiOut /* Array to write results into (see above) */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; - int iCol; - - if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ - assert( pCsr->nDoc>0 ); - for(iCol=0; iColnColumn; iCol++){ - aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; - aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; - } - }else{ - rc = fts3EvalGatherStats(pCsr, pExpr); - if( rc==SQLITE_OK ){ - assert( pExpr->aMI ); - for(iCol=0; iColnColumn; iCol++){ - aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; - aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; - } - } - } - - return rc; -} - -/* -** The expression pExpr passed as the second argument to this function -** must be of type FTSQUERY_PHRASE. -** -** The returned value is either NULL or a pointer to a buffer containing -** a position-list indicating the occurrences of the phrase in column iCol -** of the current row. -** -** More specifically, the returned buffer contains 1 varint for each -** occurrence of the phrase in the column, stored using the normal (delta+2) -** compression and is terminated by either an 0x01 or 0x00 byte. For example, -** if the requested column contains "a b X c d X X" and the position-list -** for 'X' is requested, the buffer returned may contain: -** -** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 -** -** This function works regardless of whether or not the phrase is deferred, -** incremental, or neither. -*/ -SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( - Fts3Cursor *pCsr, /* FTS3 cursor object */ - Fts3Expr *pExpr, /* Phrase to return doclist for */ - int iCol, /* Column to return position list for */ - char **ppOut /* OUT: Pointer to position list */ -){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - char *pIter; - int iThis; - sqlite3_int64 iDocid; - - /* If this phrase is applies specifically to some column other than - ** column iCol, return a NULL pointer. */ - *ppOut = 0; - assert( iCol>=0 && iColnColumn ); - if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ - return SQLITE_OK; - } - - iDocid = pExpr->iDocid; - pIter = pPhrase->doclist.pList; - if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ - int rc = SQLITE_OK; - int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ - int bOr = 0; - u8 bTreeEof = 0; - Fts3Expr *p; /* Used to iterate from pExpr to root */ - Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ - int bMatch; - - /* Check if this phrase descends from an OR expression node. If not, - ** return NULL. Otherwise, the entry that corresponds to docid - ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the - ** tree that the node is part of has been marked as EOF, but the node - ** itself is not EOF, then it may point to an earlier entry. */ - pNear = pExpr; - for(p=pExpr->pParent; p; p=p->pParent){ - if( p->eType==FTSQUERY_OR ) bOr = 1; - if( p->eType==FTSQUERY_NEAR ) pNear = p; - if( p->bEof ) bTreeEof = 1; - } - if( bOr==0 ) return SQLITE_OK; - - /* This is the descendent of an OR node. In this case we cannot use - ** an incremental phrase. Load the entire doclist for the phrase - ** into memory in this case. */ - if( pPhrase->bIncr ){ - int bEofSave = pNear->bEof; - fts3EvalRestart(pCsr, pNear, &rc); - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); - if( bEofSave==0 && pNear->iDocid==iDocid ) break; - } - assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); - } - if( bTreeEof ){ - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); - } - } - if( rc!=SQLITE_OK ) return rc; - - bMatch = 1; - for(p=pNear; p; p=p->pLeft){ - u8 bEof = 0; - Fts3Expr *pTest = p; - Fts3Phrase *pPh; - assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE ); - if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight; - assert( pTest->eType==FTSQUERY_PHRASE ); - pPh = pTest->pPhrase; - - pIter = pPh->pOrPoslist; - iDocid = pPh->iOrDocid; - if( pCsr->bDesc==bDescDoclist ){ - bEof = !pPh->doclist.nAll || - (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll)); - while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ - sqlite3Fts3DoclistNext( - bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, - &pIter, &iDocid, &bEof - ); - } - }else{ - bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll); - while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ - int dummy; - sqlite3Fts3DoclistPrev( - bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, - &pIter, &iDocid, &dummy, &bEof - ); - } - } - pPh->pOrPoslist = pIter; - pPh->iOrDocid = iDocid; - if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0; - } - - if( bMatch ){ - pIter = pPhrase->pOrPoslist; - }else{ - pIter = 0; - } - } - if( pIter==0 ) return SQLITE_OK; - - if( *pIter==0x01 ){ - pIter++; - pIter += fts3GetVarint32(pIter, &iThis); - }else{ - iThis = 0; - } - while( iThisdoclist, and -** * any Fts3MultiSegReader objects held by phrase tokens. -*/ -SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ - if( pPhrase ){ - int i; - sqlite3_free(pPhrase->doclist.aAll); - fts3EvalInvalidatePoslist(pPhrase); - memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); - for(i=0; inToken; i++){ - fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); - pPhrase->aToken[i].pSegcsr = 0; - } - } -} - - -/* -** Return SQLITE_CORRUPT_VTAB. -*/ -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ - return SQLITE_CORRUPT_VTAB; -} -#endif - -#if !SQLITE_CORE -/* -** Initialize API pointer table, if required. -*/ -#ifdef _WIN32 -__declspec(dllexport) -#endif -SQLITE_API int sqlite3_fts3_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi) - return sqlite3Fts3Init(db); -} -#endif - -#endif - -/************** End of fts3.c ************************************************/ -/************** Begin file fts3_aux.c ****************************************/ -/* -** 2011 Jan 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -*/ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ - -typedef struct Fts3auxTable Fts3auxTable; -typedef struct Fts3auxCursor Fts3auxCursor; - -struct Fts3auxTable { - sqlite3_vtab base; /* Base class used by SQLite core */ - Fts3Table *pFts3Tab; -}; - -struct Fts3auxCursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3MultiSegReader csr; /* Must be right after "base" */ - Fts3SegFilter filter; - char *zStop; - int nStop; /* Byte-length of string zStop */ - int iLangid; /* Language id to query */ - int isEof; /* True if cursor is at EOF */ - sqlite3_int64 iRowid; /* Current rowid */ - - int iCol; /* Current value of 'col' column */ - int nStat; /* Size of aStat[] array */ - struct Fts3auxColstats { - sqlite3_int64 nDoc; /* 'documents' values for current csr row */ - sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ - } *aStat; -}; - -/* -** Schema of the terms table. -*/ -#define FTS3_AUX_SCHEMA \ - "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" - -/* -** This function does all the work for both the xConnect and xCreate methods. -** These tables have no persistent representation of their own, so xConnect -** and xCreate are identical operations. -*/ -static int fts3auxConnectMethod( - sqlite3 *db, /* Database connection */ - void *pUnused, /* Unused */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ -){ - char const *zDb; /* Name of database (e.g. "main") */ - char const *zFts3; /* Name of fts3 table */ - int nDb; /* Result of strlen(zDb) */ - int nFts3; /* Result of strlen(zFts3) */ - int nByte; /* Bytes of space to allocate here */ - int rc; /* value returned by declare_vtab() */ - Fts3auxTable *p; /* Virtual table object to return */ - - UNUSED_PARAMETER(pUnused); - - /* The user should invoke this in one of two forms: - ** - ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); - ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); - */ - if( argc!=4 && argc!=5 ) goto bad_args; - - zDb = argv[1]; - nDb = (int)strlen(zDb); - if( argc==5 ){ - if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ - zDb = argv[3]; - nDb = (int)strlen(zDb); - zFts3 = argv[4]; - }else{ - goto bad_args; - } - }else{ - zFts3 = argv[3]; - } - nFts3 = (int)strlen(zFts3); - - rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); - if( rc!=SQLITE_OK ) return rc; - - nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; - p = (Fts3auxTable *)sqlite3_malloc(nByte); - if( !p ) return SQLITE_NOMEM; - memset(p, 0, nByte); - - p->pFts3Tab = (Fts3Table *)&p[1]; - p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; - p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; - p->pFts3Tab->db = db; - p->pFts3Tab->nIndex = 1; - - memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); - memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); - sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); - - *ppVtab = (sqlite3_vtab *)p; - return SQLITE_OK; - - bad_args: - sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); - return SQLITE_ERROR; -} - -/* -** This function does the work for both the xDisconnect and xDestroy methods. -** These tables have no persistent representation of their own, so xDisconnect -** and xDestroy are identical operations. -*/ -static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ - Fts3auxTable *p = (Fts3auxTable *)pVtab; - Fts3Table *pFts3 = p->pFts3Tab; - int i; - - /* Free any prepared statements held */ - for(i=0; iaStmt); i++){ - sqlite3_finalize(pFts3->aStmt[i]); - } - sqlite3_free(pFts3->zSegmentsTbl); - sqlite3_free(p); - return SQLITE_OK; -} - -#define FTS4AUX_EQ_CONSTRAINT 1 -#define FTS4AUX_GE_CONSTRAINT 2 -#define FTS4AUX_LE_CONSTRAINT 4 - -/* -** xBestIndex - Analyze a WHERE and ORDER BY clause. -*/ -static int fts3auxBestIndexMethod( - sqlite3_vtab *pVTab, - sqlite3_index_info *pInfo -){ - int i; - int iEq = -1; - int iGe = -1; - int iLe = -1; - int iLangid = -1; - int iNext = 1; /* Next free argvIndex value */ - - UNUSED_PARAMETER(pVTab); - - /* This vtab delivers always results in "ORDER BY term ASC" order. */ - if( pInfo->nOrderBy==1 - && pInfo->aOrderBy[0].iColumn==0 - && pInfo->aOrderBy[0].desc==0 - ){ - pInfo->orderByConsumed = 1; - } - - /* Search for equality and range constraints on the "term" column. - ** And equality constraints on the hidden "languageid" column. */ - for(i=0; inConstraint; i++){ - if( pInfo->aConstraint[i].usable ){ - int op = pInfo->aConstraint[i].op; - int iCol = pInfo->aConstraint[i].iColumn; - - if( iCol==0 ){ - if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; - if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; - } - if( iCol==4 ){ - if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; - } - } - } - - if( iEq>=0 ){ - pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; - pInfo->aConstraintUsage[iEq].argvIndex = iNext++; - pInfo->estimatedCost = 5; - }else{ - pInfo->idxNum = 0; - pInfo->estimatedCost = 20000; - if( iGe>=0 ){ - pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; - pInfo->aConstraintUsage[iGe].argvIndex = iNext++; - pInfo->estimatedCost /= 2; - } - if( iLe>=0 ){ - pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; - pInfo->aConstraintUsage[iLe].argvIndex = iNext++; - pInfo->estimatedCost /= 2; - } - } - if( iLangid>=0 ){ - pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; - pInfo->estimatedCost--; - } - - return SQLITE_OK; -} - -/* -** xOpen - Open a cursor. -*/ -static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ - - UNUSED_PARAMETER(pVTab); - - pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); - if( !pCsr ) return SQLITE_NOMEM; - memset(pCsr, 0, sizeof(Fts3auxCursor)); - - *ppCsr = (sqlite3_vtab_cursor *)pCsr; - return SQLITE_OK; -} - -/* -** xClose - Close a cursor. -*/ -static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - - sqlite3Fts3SegmentsClose(pFts3); - sqlite3Fts3SegReaderFinish(&pCsr->csr); - sqlite3_free((void *)pCsr->filter.zTerm); - sqlite3_free(pCsr->zStop); - sqlite3_free(pCsr->aStat); - sqlite3_free(pCsr); - return SQLITE_OK; -} - -static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ - if( nSize>pCsr->nStat ){ - struct Fts3auxColstats *aNew; - aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, - sizeof(struct Fts3auxColstats) * nSize - ); - if( aNew==0 ) return SQLITE_NOMEM; - memset(&aNew[pCsr->nStat], 0, - sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) - ); - pCsr->aStat = aNew; - pCsr->nStat = nSize; - } - return SQLITE_OK; -} - -/* -** xNext - Advance the cursor to the next row, if any. -*/ -static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; - int rc; - - /* Increment our pretend rowid value. */ - pCsr->iRowid++; - - for(pCsr->iCol++; pCsr->iColnStat; pCsr->iCol++){ - if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; - } - - rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); - if( rc==SQLITE_ROW ){ - int i = 0; - int nDoclist = pCsr->csr.nDoclist; - char *aDoclist = pCsr->csr.aDoclist; - int iCol; - - int eState = 0; - - if( pCsr->zStop ){ - int n = (pCsr->nStopcsr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; - int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); - if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ - pCsr->isEof = 1; - return SQLITE_OK; - } - } - - if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; - memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); - iCol = 0; - - while( iaStat[0].nDoc++; - eState = 1; - iCol = 0; - break; - - /* State 1. In this state we are expecting either a 1, indicating - ** that the following integer will be a column number, or the - ** start of a position list for column 0. - ** - ** The only difference between state 1 and state 2 is that if the - ** integer encountered in state 1 is not 0 or 1, then we need to - ** increment the column 0 "nDoc" count for this term. - */ - case 1: - assert( iCol==0 ); - if( v>1 ){ - pCsr->aStat[1].nDoc++; - } - eState = 2; - /* fall through */ - - case 2: - if( v==0 ){ /* 0x00. Next integer will be a docid. */ - eState = 0; - }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ - eState = 3; - }else{ /* 2 or greater. A position. */ - pCsr->aStat[iCol+1].nOcc++; - pCsr->aStat[0].nOcc++; - } - break; - - /* State 3. The integer just read is a column number. */ - default: assert( eState==3 ); - iCol = (int)v; - if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; - pCsr->aStat[iCol+1].nDoc++; - eState = 2; - break; - } - } - - pCsr->iCol = 0; - rc = SQLITE_OK; - }else{ - pCsr->isEof = 1; - } - return rc; -} - -/* -** xFilter - Initialize a cursor to point at the start of its data. -*/ -static int fts3auxFilterMethod( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, /* Strategy index */ - const char *idxStr, /* Unused */ - int nVal, /* Number of elements in apVal */ - sqlite3_value **apVal /* Arguments for the indexing scheme */ -){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; - int rc; - int isScan = 0; - int iLangVal = 0; /* Language id to query */ - - int iEq = -1; /* Index of term=? value in apVal */ - int iGe = -1; /* Index of term>=? value in apVal */ - int iLe = -1; /* Index of term<=? value in apVal */ - int iLangid = -1; /* Index of languageid=? value in apVal */ - int iNext = 0; - - UNUSED_PARAMETER(nVal); - UNUSED_PARAMETER(idxStr); - - assert( idxStr==0 ); - assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 - || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT - || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) - ); - - if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ - iEq = iNext++; - }else{ - isScan = 1; - if( idxNum & FTS4AUX_GE_CONSTRAINT ){ - iGe = iNext++; - } - if( idxNum & FTS4AUX_LE_CONSTRAINT ){ - iLe = iNext++; - } - } - if( iNextfilter.zTerm); - sqlite3Fts3SegReaderFinish(&pCsr->csr); - sqlite3_free((void *)pCsr->filter.zTerm); - sqlite3_free(pCsr->aStat); - memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); - - pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; - if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - - if( iEq>=0 || iGe>=0 ){ - const unsigned char *zStr = sqlite3_value_text(apVal[0]); - assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); - if( zStr ){ - pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); - pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]); - if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; - } - } - - if( iLe>=0 ){ - pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); - pCsr->nStop = sqlite3_value_bytes(apVal[iLe]); - if( pCsr->zStop==0 ) return SQLITE_NOMEM; - } - - if( iLangid>=0 ){ - iLangVal = sqlite3_value_int(apVal[iLangid]); - - /* If the user specified a negative value for the languageid, use zero - ** instead. This works, as the "languageid=?" constraint will also - ** be tested by the VDBE layer. The test will always be false (since - ** this module will not return a row with a negative languageid), and - ** so the overall query will return zero rows. */ - if( iLangVal<0 ) iLangVal = 0; - } - pCsr->iLangid = iLangVal; - - rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, - pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr - ); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); - } - - if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); - return rc; -} - -/* -** xEof - Return true if the cursor is at EOF, or false otherwise. -*/ -static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - return pCsr->isEof; -} - -/* -** xColumn - Return a column value. -*/ -static int fts3auxColumnMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ - int iCol /* Index of column to read value from */ -){ - Fts3auxCursor *p = (Fts3auxCursor *)pCursor; - - assert( p->isEof==0 ); - switch( iCol ){ - case 0: /* term */ - sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); - break; - - case 1: /* col */ - if( p->iCol ){ - sqlite3_result_int(pCtx, p->iCol-1); - }else{ - sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); - } - break; - - case 2: /* documents */ - sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); - break; - - case 3: /* occurrences */ - sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); - break; - - default: /* languageid */ - assert( iCol==4 ); - sqlite3_result_int(pCtx, p->iLangid); - break; - } - - return SQLITE_OK; -} - -/* -** xRowid - Return the current rowid for the cursor. -*/ -static int fts3auxRowidMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite_int64 *pRowid /* OUT: Rowid value */ -){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - *pRowid = pCsr->iRowid; - return SQLITE_OK; -} - -/* -** Register the fts3aux module with database connection db. Return SQLITE_OK -** if successful or an error code if sqlite3_create_module() fails. -*/ -SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ - static const sqlite3_module fts3aux_module = { - 0, /* iVersion */ - fts3auxConnectMethod, /* xCreate */ - fts3auxConnectMethod, /* xConnect */ - fts3auxBestIndexMethod, /* xBestIndex */ - fts3auxDisconnectMethod, /* xDisconnect */ - fts3auxDisconnectMethod, /* xDestroy */ - fts3auxOpenMethod, /* xOpen */ - fts3auxCloseMethod, /* xClose */ - fts3auxFilterMethod, /* xFilter */ - fts3auxNextMethod, /* xNext */ - fts3auxEofMethod, /* xEof */ - fts3auxColumnMethod, /* xColumn */ - fts3auxRowidMethod, /* xRowid */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindFunction */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0 /* xRollbackTo */ - }; - int rc; /* Return code */ - - rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); - return rc; -} - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_aux.c ********************************************/ -/************** Begin file fts3_expr.c ***************************************/ -/* -** 2008 Nov 28 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This module contains code that implements a parser for fts3 query strings -** (the right-hand argument to the MATCH operator). Because the supported -** syntax is relatively simple, the whole tokenizer/parser system is -** hand-coded. -*/ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* -** By default, this module parses the legacy syntax that has been -** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS -** is defined, then it uses the new syntax. The differences between -** the new and the old syntaxes are: -** -** a) The new syntax supports parenthesis. The old does not. -** -** b) The new syntax supports the AND and NOT operators. The old does not. -** -** c) The old syntax supports the "-" token qualifier. This is not -** supported by the new syntax (it is replaced by the NOT operator). -** -** d) When using the old syntax, the OR operator has a greater precedence -** than an implicit AND. When using the new, both implicity and explicit -** AND operators have a higher precedence than OR. -** -** If compiled with SQLITE_TEST defined, then this module exports the -** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable -** to zero causes the module to use the old syntax. If it is set to -** non-zero the new syntax is activated. This is so both syntaxes can -** be tested using a single build of testfixture. -** -** The following describes the syntax supported by the fts3 MATCH -** operator in a similar format to that used by the lemon parser -** generator. This module does not use actually lemon, it uses a -** custom parser. -** -** query ::= andexpr (OR andexpr)*. -** -** andexpr ::= notexpr (AND? notexpr)*. -** -** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. -** notexpr ::= LP query RP. -** -** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. -** -** distance_opt ::= . -** distance_opt ::= / INTEGER. -** -** phrase ::= TOKEN. -** phrase ::= COLUMN:TOKEN. -** phrase ::= "TOKEN TOKEN TOKEN...". -*/ - -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_fts3_enable_parentheses = 0; -#else -# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS -# define sqlite3_fts3_enable_parentheses 1 -# else -# define sqlite3_fts3_enable_parentheses 0 -# endif -#endif - -/* -** Default span for NEAR operators. -*/ -#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 - -/* #include */ -/* #include */ - -/* -** isNot: -** This variable is used by function getNextNode(). When getNextNode() is -** called, it sets ParseContext.isNot to true if the 'next node' is a -** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the -** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to -** zero. -*/ -typedef struct ParseContext ParseContext; -struct ParseContext { - sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ - int iLangid; /* Language id used with tokenizer */ - const char **azCol; /* Array of column names for fts3 table */ - int bFts4; /* True to allow FTS4-only syntax */ - int nCol; /* Number of entries in azCol[] */ - int iDefaultCol; /* Default column to query */ - int isNot; /* True if getNextNode() sees a unary - */ - sqlite3_context *pCtx; /* Write error message here */ - int nNest; /* Number of nested brackets */ -}; - -/* -** This function is equivalent to the standard isspace() function. -** -** The standard isspace() can be awkward to use safely, because although it -** is defined to accept an argument of type int, its behavior when passed -** an integer that falls outside of the range of the unsigned char type -** is undefined (and sometimes, "undefined" means segfault). This wrapper -** is defined to accept an argument of type char, and always returns 0 for -** any values that fall outside of the range of the unsigned char type (i.e. -** negative values). -*/ -static int fts3isspace(char c){ - return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; -} - -/* -** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, -** zero the memory before returning a pointer to it. If unsuccessful, -** return NULL. -*/ -static void *fts3MallocZero(int nByte){ - void *pRet = sqlite3_malloc(nByte); - if( pRet ) memset(pRet, 0, nByte); - return pRet; -} - -SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer( - sqlite3_tokenizer *pTokenizer, - int iLangid, - const char *z, - int n, - sqlite3_tokenizer_cursor **ppCsr -){ - sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; - sqlite3_tokenizer_cursor *pCsr = 0; - int rc; - - rc = pModule->xOpen(pTokenizer, z, n, &pCsr); - assert( rc==SQLITE_OK || pCsr==0 ); - if( rc==SQLITE_OK ){ - pCsr->pTokenizer = pTokenizer; - if( pModule->iVersion>=1 ){ - rc = pModule->xLanguageid(pCsr, iLangid); - if( rc!=SQLITE_OK ){ - pModule->xClose(pCsr); - pCsr = 0; - } - } - } - *ppCsr = pCsr; - return rc; -} - -/* -** Function getNextNode(), which is called by fts3ExprParse(), may itself -** call fts3ExprParse(). So this forward declaration is required. -*/ -static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); - -/* -** Extract the next token from buffer z (length n) using the tokenizer -** and other information (column names etc.) in pParse. Create an Fts3Expr -** structure of type FTSQUERY_PHRASE containing a phrase consisting of this -** single token and set *ppExpr to point to it. If the end of the buffer is -** reached before a token is found, set *ppExpr to zero. It is the -** responsibility of the caller to eventually deallocate the allocated -** Fts3Expr structure (if any) by passing it to sqlite3_free(). -** -** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation -** fails. -*/ -static int getNextToken( - ParseContext *pParse, /* fts3 query parse context */ - int iCol, /* Value for Fts3Phrase.iColumn */ - const char *z, int n, /* Input string */ - Fts3Expr **ppExpr, /* OUT: expression */ - int *pnConsumed /* OUT: Number of bytes consumed */ -){ - sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; - sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; - int rc; - sqlite3_tokenizer_cursor *pCursor; - Fts3Expr *pRet = 0; - int i = 0; - - /* Set variable i to the maximum number of bytes of input to tokenize. */ - for(i=0; iiLangid, z, i, &pCursor); - if( rc==SQLITE_OK ){ - const char *zToken; - int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; - int nByte; /* total space to allocate */ - - rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); - if( rc==SQLITE_OK ){ - nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; - pRet = (Fts3Expr *)fts3MallocZero(nByte); - if( !pRet ){ - rc = SQLITE_NOMEM; - }else{ - pRet->eType = FTSQUERY_PHRASE; - pRet->pPhrase = (Fts3Phrase *)&pRet[1]; - pRet->pPhrase->nToken = 1; - pRet->pPhrase->iColumn = iCol; - pRet->pPhrase->aToken[0].n = nToken; - pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; - memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); - - if( iEndpPhrase->aToken[0].isPrefix = 1; - iEnd++; - } - - while( 1 ){ - if( !sqlite3_fts3_enable_parentheses - && iStart>0 && z[iStart-1]=='-' - ){ - pParse->isNot = 1; - iStart--; - }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ - pRet->pPhrase->aToken[0].bFirst = 1; - iStart--; - }else{ - break; - } - } - - } - *pnConsumed = iEnd; - }else if( i && rc==SQLITE_DONE ){ - rc = SQLITE_OK; - } - - pModule->xClose(pCursor); - } - - *ppExpr = pRet; - return rc; -} - - -/* -** Enlarge a memory allocation. If an out-of-memory allocation occurs, -** then free the old allocation. -*/ -static void *fts3ReallocOrFree(void *pOrig, int nNew){ - void *pRet = sqlite3_realloc(pOrig, nNew); - if( !pRet ){ - sqlite3_free(pOrig); - } - return pRet; -} - -/* -** Buffer zInput, length nInput, contains the contents of a quoted string -** that appeared as part of an fts3 query expression. Neither quote character -** is included in the buffer. This function attempts to tokenize the entire -** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE -** containing the results. -** -** If successful, SQLITE_OK is returned and *ppExpr set to point at the -** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory -** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set -** to 0. -*/ -static int getNextString( - ParseContext *pParse, /* fts3 query parse context */ - const char *zInput, int nInput, /* Input string */ - Fts3Expr **ppExpr /* OUT: expression */ -){ - sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; - sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; - int rc; - Fts3Expr *p = 0; - sqlite3_tokenizer_cursor *pCursor = 0; - char *zTemp = 0; - int nTemp = 0; - - const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - int nToken = 0; - - /* The final Fts3Expr data structure, including the Fts3Phrase, - ** Fts3PhraseToken structures token buffers are all stored as a single - ** allocation so that the expression can be freed with a single call to - ** sqlite3_free(). Setting this up requires a two pass approach. - ** - ** The first pass, in the block below, uses a tokenizer cursor to iterate - ** through the tokens in the expression. This pass uses fts3ReallocOrFree() - ** to assemble data in two dynamic buffers: - ** - ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase - ** structure, followed by the array of Fts3PhraseToken - ** structures. This pass only populates the Fts3PhraseToken array. - ** - ** Buffer zTemp: Contains copies of all tokens. - ** - ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, - ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase - ** structures. - */ - rc = sqlite3Fts3OpenTokenizer( - pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); - if( rc==SQLITE_OK ){ - int ii; - for(ii=0; rc==SQLITE_OK; ii++){ - const char *zByte; - int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; - rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); - if( rc==SQLITE_OK ){ - Fts3PhraseToken *pToken; - - p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); - if( !p ) goto no_mem; - - zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); - if( !zTemp ) goto no_mem; - - assert( nToken==ii ); - pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; - memset(pToken, 0, sizeof(Fts3PhraseToken)); - - memcpy(&zTemp[nTemp], zByte, nByte); - nTemp += nByte; - - pToken->n = nByte; - pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); - nToken = ii+1; - } - } - - pModule->xClose(pCursor); - pCursor = 0; - } - - if( rc==SQLITE_DONE ){ - int jj; - char *zBuf = 0; - - p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); - if( !p ) goto no_mem; - memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); - p->eType = FTSQUERY_PHRASE; - p->pPhrase = (Fts3Phrase *)&p[1]; - p->pPhrase->iColumn = pParse->iDefaultCol; - p->pPhrase->nToken = nToken; - - zBuf = (char *)&p->pPhrase->aToken[nToken]; - if( zTemp ){ - memcpy(zBuf, zTemp, nTemp); - sqlite3_free(zTemp); - }else{ - assert( nTemp==0 ); - } - - for(jj=0; jjpPhrase->nToken; jj++){ - p->pPhrase->aToken[jj].z = zBuf; - zBuf += p->pPhrase->aToken[jj].n; - } - rc = SQLITE_OK; - } - - *ppExpr = p; - return rc; -no_mem: - - if( pCursor ){ - pModule->xClose(pCursor); - } - sqlite3_free(zTemp); - sqlite3_free(p); - *ppExpr = 0; - return SQLITE_NOMEM; -} - -/* -** The output variable *ppExpr is populated with an allocated Fts3Expr -** structure, or set to 0 if the end of the input buffer is reached. -** -** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM -** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. -** If SQLITE_ERROR is returned, pContext is populated with an error message. -*/ -static int getNextNode( - ParseContext *pParse, /* fts3 query parse context */ - const char *z, int n, /* Input string */ - Fts3Expr **ppExpr, /* OUT: expression */ - int *pnConsumed /* OUT: Number of bytes consumed */ -){ - static const struct Fts3Keyword { - char *z; /* Keyword text */ - unsigned char n; /* Length of the keyword */ - unsigned char parenOnly; /* Only valid in paren mode */ - unsigned char eType; /* Keyword code */ - } aKeyword[] = { - { "OR" , 2, 0, FTSQUERY_OR }, - { "AND", 3, 1, FTSQUERY_AND }, - { "NOT", 3, 1, FTSQUERY_NOT }, - { "NEAR", 4, 0, FTSQUERY_NEAR } - }; - int ii; - int iCol; - int iColLen; - int rc; - Fts3Expr *pRet = 0; - - const char *zInput = z; - int nInput = n; - - pParse->isNot = 0; - - /* Skip over any whitespace before checking for a keyword, an open or - ** close bracket, or a quoted string. - */ - while( nInput>0 && fts3isspace(*zInput) ){ - nInput--; - zInput++; - } - if( nInput==0 ){ - return SQLITE_DONE; - } - - /* See if we are dealing with a keyword. */ - for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ - const struct Fts3Keyword *pKey = &aKeyword[ii]; - - if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ - continue; - } - - if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ - int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; - int nKey = pKey->n; - char cNext; - - /* If this is a "NEAR" keyword, check for an explicit nearness. */ - if( pKey->eType==FTSQUERY_NEAR ){ - assert( nKey==4 ); - if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ - nNear = 0; - for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){ - nNear = nNear * 10 + (zInput[nKey] - '0'); - } - } - } - - /* At this point this is probably a keyword. But for that to be true, - ** the next byte must contain either whitespace, an open or close - ** parenthesis, a quote character, or EOF. - */ - cNext = zInput[nKey]; - if( fts3isspace(cNext) - || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 - ){ - pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); - if( !pRet ){ - return SQLITE_NOMEM; - } - pRet->eType = pKey->eType; - pRet->nNear = nNear; - *ppExpr = pRet; - *pnConsumed = (int)((zInput - z) + nKey); - return SQLITE_OK; - } - - /* Turns out that wasn't a keyword after all. This happens if the - ** user has supplied a token such as "ORacle". Continue. - */ - } - } - - /* See if we are dealing with a quoted phrase. If this is the case, then - ** search for the closing quote and pass the whole string to getNextString() - ** for processing. This is easy to do, as fts3 has no syntax for escaping - ** a quote character embedded in a string. - */ - if( *zInput=='"' ){ - for(ii=1; iinNest++; - rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); - if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } - *pnConsumed = (int)(zInput - z) + 1 + nConsumed; - return rc; - }else if( *zInput==')' ){ - pParse->nNest--; - *pnConsumed = (int)((zInput - z) + 1); - *ppExpr = 0; - return SQLITE_DONE; - } - } - - /* If control flows to this point, this must be a regular token, or - ** the end of the input. Read a regular token using the sqlite3_tokenizer - ** interface. Before doing so, figure out if there is an explicit - ** column specifier for the token. - ** - ** TODO: Strangely, it is not possible to associate a column specifier - ** with a quoted phrase, only with a single token. Not sure if this was - ** an implementation artifact or an intentional decision when fts3 was - ** first implemented. Whichever it was, this module duplicates the - ** limitation. - */ - iCol = pParse->iDefaultCol; - iColLen = 0; - for(ii=0; iinCol; ii++){ - const char *zStr = pParse->azCol[ii]; - int nStr = (int)strlen(zStr); - if( nInput>nStr && zInput[nStr]==':' - && sqlite3_strnicmp(zStr, zInput, nStr)==0 - ){ - iCol = ii; - iColLen = (int)((zInput - z) + nStr + 1); - break; - } - } - rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); - *pnConsumed += iColLen; - return rc; -} - -/* -** The argument is an Fts3Expr structure for a binary operator (any type -** except an FTSQUERY_PHRASE). Return an integer value representing the -** precedence of the operator. Lower values have a higher precedence (i.e. -** group more tightly). For example, in the C language, the == operator -** groups more tightly than ||, and would therefore have a higher precedence. -** -** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS -** is defined), the order of the operators in precedence from highest to -** lowest is: -** -** NEAR -** NOT -** AND (including implicit ANDs) -** OR -** -** Note that when using the old query syntax, the OR operator has a higher -** precedence than the AND operator. -*/ -static int opPrecedence(Fts3Expr *p){ - assert( p->eType!=FTSQUERY_PHRASE ); - if( sqlite3_fts3_enable_parentheses ){ - return p->eType; - }else if( p->eType==FTSQUERY_NEAR ){ - return 1; - }else if( p->eType==FTSQUERY_OR ){ - return 2; - } - assert( p->eType==FTSQUERY_AND ); - return 3; -} - -/* -** Argument ppHead contains a pointer to the current head of a query -** expression tree being parsed. pPrev is the expression node most recently -** inserted into the tree. This function adds pNew, which is always a binary -** operator node, into the expression tree based on the relative precedence -** of pNew and the existing nodes of the tree. This may result in the head -** of the tree changing, in which case *ppHead is set to the new root node. -*/ -static void insertBinaryOperator( - Fts3Expr **ppHead, /* Pointer to the root node of a tree */ - Fts3Expr *pPrev, /* Node most recently inserted into the tree */ - Fts3Expr *pNew /* New binary node to insert into expression tree */ -){ - Fts3Expr *pSplit = pPrev; - while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ - pSplit = pSplit->pParent; - } - - if( pSplit->pParent ){ - assert( pSplit->pParent->pRight==pSplit ); - pSplit->pParent->pRight = pNew; - pNew->pParent = pSplit->pParent; - }else{ - *ppHead = pNew; - } - pNew->pLeft = pSplit; - pSplit->pParent = pNew; -} - -/* -** Parse the fts3 query expression found in buffer z, length n. This function -** returns either when the end of the buffer is reached or an unmatched -** closing bracket - ')' - is encountered. -** -** If successful, SQLITE_OK is returned, *ppExpr is set to point to the -** parsed form of the expression and *pnConsumed is set to the number of -** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM -** (out of memory error) or SQLITE_ERROR (parse error) is returned. -*/ -static int fts3ExprParse( - ParseContext *pParse, /* fts3 query parse context */ - const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr, /* OUT: Parsed query structure */ - int *pnConsumed /* OUT: Number of bytes consumed */ -){ - Fts3Expr *pRet = 0; - Fts3Expr *pPrev = 0; - Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ - int nIn = n; - const char *zIn = z; - int rc = SQLITE_OK; - int isRequirePhrase = 1; - - while( rc==SQLITE_OK ){ - Fts3Expr *p = 0; - int nByte = 0; - - rc = getNextNode(pParse, zIn, nIn, &p, &nByte); - assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); - if( rc==SQLITE_OK ){ - if( p ){ - int isPhrase; - - if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && pParse->isNot - ){ - /* Create an implicit NOT operator. */ - Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); - if( !pNot ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_NOMEM; - goto exprparse_out; - } - pNot->eType = FTSQUERY_NOT; - pNot->pRight = p; - p->pParent = pNot; - if( pNotBranch ){ - pNot->pLeft = pNotBranch; - pNotBranch->pParent = pNot; - } - pNotBranch = pNot; - p = pPrev; - }else{ - int eType = p->eType; - isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); - - /* The isRequirePhrase variable is set to true if a phrase or - ** an expression contained in parenthesis is required. If a - ** binary operator (AND, OR, NOT or NEAR) is encounted when - ** isRequirePhrase is set, this is a syntax error. - */ - if( !isPhrase && isRequirePhrase ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase && !isRequirePhrase ){ - /* Insert an implicit AND operator. */ - Fts3Expr *pAnd; - assert( pRet && pPrev ); - pAnd = fts3MallocZero(sizeof(Fts3Expr)); - if( !pAnd ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_NOMEM; - goto exprparse_out; - } - pAnd->eType = FTSQUERY_AND; - insertBinaryOperator(&pRet, pPrev, pAnd); - pPrev = pAnd; - } - - /* This test catches attempts to make either operand of a NEAR - ** operator something other than a phrase. For example, either of - ** the following: - ** - ** (bracketed expression) NEAR phrase - ** phrase NEAR (bracketed expression) - ** - ** Return an error in either case. - */ - if( pPrev && ( - (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) - || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) - )){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase ){ - if( pRet ){ - assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); - pPrev->pRight = p; - p->pParent = pPrev; - }else{ - pRet = p; - } - }else{ - insertBinaryOperator(&pRet, pPrev, p); - } - isRequirePhrase = !isPhrase; - } - pPrev = p; - } - assert( nByte>0 ); - } - assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); - nIn -= nByte; - zIn += nByte; - } - - if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ - rc = SQLITE_ERROR; - } - - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ - if( !pRet ){ - rc = SQLITE_ERROR; - }else{ - Fts3Expr *pIter = pNotBranch; - while( pIter->pLeft ){ - pIter = pIter->pLeft; - } - pIter->pLeft = pRet; - pRet->pParent = pIter; - pRet = pNotBranch; - } - } - } - *pnConsumed = n - nIn; - -exprparse_out: - if( rc!=SQLITE_OK ){ - sqlite3Fts3ExprFree(pRet); - sqlite3Fts3ExprFree(pNotBranch); - pRet = 0; - } - *ppExpr = pRet; - return rc; -} - -/* -** Return SQLITE_ERROR if the maximum depth of the expression tree passed -** as the only argument is more than nMaxDepth. -*/ -static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ - int rc = SQLITE_OK; - if( p ){ - if( nMaxDepth<0 ){ - rc = SQLITE_TOOBIG; - }else{ - rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); - if( rc==SQLITE_OK ){ - rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); - } - } - } - return rc; -} - -/* -** This function attempts to transform the expression tree at (*pp) to -** an equivalent but more balanced form. The tree is modified in place. -** If successful, SQLITE_OK is returned and (*pp) set to point to the -** new root expression node. -** -** nMaxDepth is the maximum allowable depth of the balanced sub-tree. -** -** Otherwise, if an error occurs, an SQLite error code is returned and -** expression (*pp) freed. -*/ -static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ - int rc = SQLITE_OK; /* Return code */ - Fts3Expr *pRoot = *pp; /* Initial root node */ - Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ - int eType = pRoot->eType; /* Type of node in this tree */ - - if( nMaxDepth==0 ){ - rc = SQLITE_ERROR; - } - - if( rc==SQLITE_OK ){ - if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ - Fts3Expr **apLeaf; - apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); - if( 0==apLeaf ){ - rc = SQLITE_NOMEM; - }else{ - memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); - } - - if( rc==SQLITE_OK ){ - int i; - Fts3Expr *p; - - /* Set $p to point to the left-most leaf in the tree of eType nodes. */ - for(p=pRoot; p->eType==eType; p=p->pLeft){ - assert( p->pParent==0 || p->pParent->pLeft==p ); - assert( p->pLeft && p->pRight ); - } - - /* This loop runs once for each leaf in the tree of eType nodes. */ - while( 1 ){ - int iLvl; - Fts3Expr *pParent = p->pParent; /* Current parent of p */ - - assert( pParent==0 || pParent->pLeft==p ); - p->pParent = 0; - if( pParent ){ - pParent->pLeft = 0; - }else{ - pRoot = 0; - } - rc = fts3ExprBalance(&p, nMaxDepth-1); - if( rc!=SQLITE_OK ) break; - - for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; - pFree->pRight = p; - pFree->pLeft->pParent = pFree; - pFree->pRight->pParent = pFree; - - p = pFree; - pFree = pFree->pParent; - p->pParent = 0; - apLeaf[iLvl] = 0; - } - } - if( p ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_TOOBIG; - break; - } - - /* If that was the last leaf node, break out of the loop */ - if( pParent==0 ) break; - - /* Set $p to point to the next leaf in the tree of eType nodes */ - for(p=pParent->pRight; p->eType==eType; p=p->pLeft); - - /* Remove pParent from the original tree. */ - assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); - pParent->pRight->pParent = pParent->pParent; - if( pParent->pParent ){ - pParent->pParent->pLeft = pParent->pRight; - }else{ - assert( pParent==pRoot ); - pRoot = pParent->pRight; - } - - /* Link pParent into the free node list. It will be used as an - ** internal node of the new tree. */ - pParent->pParent = pFree; - pFree = pParent; - } - - if( rc==SQLITE_OK ){ - p = 0; - for(i=0; ipParent = 0; - }else{ - assert( pFree!=0 ); - pFree->pRight = p; - pFree->pLeft = apLeaf[i]; - pFree->pLeft->pParent = pFree; - pFree->pRight->pParent = pFree; - - p = pFree; - pFree = pFree->pParent; - p->pParent = 0; - } - } - } - pRoot = p; - }else{ - /* An error occurred. Delete the contents of the apLeaf[] array - ** and pFree list. Everything else is cleaned up by the call to - ** sqlite3Fts3ExprFree(pRoot) below. */ - Fts3Expr *pDel; - for(i=0; ipParent; - sqlite3_free(pDel); - } - } - - assert( pFree==0 ); - sqlite3_free( apLeaf ); - } - }else if( eType==FTSQUERY_NOT ){ - Fts3Expr *pLeft = pRoot->pLeft; - Fts3Expr *pRight = pRoot->pRight; - - pRoot->pLeft = 0; - pRoot->pRight = 0; - pLeft->pParent = 0; - pRight->pParent = 0; - - rc = fts3ExprBalance(&pLeft, nMaxDepth-1); - if( rc==SQLITE_OK ){ - rc = fts3ExprBalance(&pRight, nMaxDepth-1); - } - - if( rc!=SQLITE_OK ){ - sqlite3Fts3ExprFree(pRight); - sqlite3Fts3ExprFree(pLeft); - }else{ - assert( pLeft && pRight ); - pRoot->pLeft = pLeft; - pLeft->pParent = pRoot; - pRoot->pRight = pRight; - pRight->pParent = pRoot; - } - } - } - - if( rc!=SQLITE_OK ){ - sqlite3Fts3ExprFree(pRoot); - pRoot = 0; - } - *pp = pRoot; - return rc; -} - -/* -** This function is similar to sqlite3Fts3ExprParse(), with the following -** differences: -** -** 1. It does not do expression rebalancing. -** 2. It does not check that the expression does not exceed the -** maximum allowable depth. -** 3. Even if it fails, *ppExpr may still be set to point to an -** expression tree. It should be deleted using sqlite3Fts3ExprFree() -** in this case. -*/ -static int fts3ExprParseUnbalanced( - sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ - int iLangid, /* Language id for tokenizer */ - char **azCol, /* Array of column names for fts3 table */ - int bFts4, /* True to allow FTS4-only syntax */ - int nCol, /* Number of entries in azCol[] */ - int iDefaultCol, /* Default column to query */ - const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr /* OUT: Parsed query structure */ -){ - int nParsed; - int rc; - ParseContext sParse; - - memset(&sParse, 0, sizeof(ParseContext)); - sParse.pTokenizer = pTokenizer; - sParse.iLangid = iLangid; - sParse.azCol = (const char **)azCol; - sParse.nCol = nCol; - sParse.iDefaultCol = iDefaultCol; - sParse.bFts4 = bFts4; - if( z==0 ){ - *ppExpr = 0; - return SQLITE_OK; - } - if( n<0 ){ - n = (int)strlen(z); - } - rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); - assert( rc==SQLITE_OK || *ppExpr==0 ); - - /* Check for mismatched parenthesis */ - if( rc==SQLITE_OK && sParse.nNest ){ - rc = SQLITE_ERROR; - } - - return rc; -} - -/* -** Parameters z and n contain a pointer to and length of a buffer containing -** an fts3 query expression, respectively. This function attempts to parse the -** query expression and create a tree of Fts3Expr structures representing the -** parsed expression. If successful, *ppExpr is set to point to the head -** of the parsed expression tree and SQLITE_OK is returned. If an error -** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse -** error) is returned and *ppExpr is set to 0. -** -** If parameter n is a negative number, then z is assumed to point to a -** nul-terminated string and the length is determined using strlen(). -** -** The first parameter, pTokenizer, is passed the fts3 tokenizer module to -** use to normalize query tokens while parsing the expression. The azCol[] -** array, which is assumed to contain nCol entries, should contain the names -** of each column in the target fts3 table, in order from left to right. -** Column names must be nul-terminated strings. -** -** The iDefaultCol parameter should be passed the index of the table column -** that appears on the left-hand-side of the MATCH operator (the default -** column to match against for tokens for which a column name is not explicitly -** specified as part of the query string), or -1 if tokens may by default -** match any table column. -*/ -SQLITE_PRIVATE int sqlite3Fts3ExprParse( - sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ - int iLangid, /* Language id for tokenizer */ - char **azCol, /* Array of column names for fts3 table */ - int bFts4, /* True to allow FTS4-only syntax */ - int nCol, /* Number of entries in azCol[] */ - int iDefaultCol, /* Default column to query */ - const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr, /* OUT: Parsed query structure */ - char **pzErr /* OUT: Error message (sqlite3_malloc) */ -){ - int rc = fts3ExprParseUnbalanced( - pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr - ); - - /* Rebalance the expression. And check that its depth does not exceed - ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ - if( rc==SQLITE_OK && *ppExpr ){ - rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); - if( rc==SQLITE_OK ){ - rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); - } - } - - if( rc!=SQLITE_OK ){ - sqlite3Fts3ExprFree(*ppExpr); - *ppExpr = 0; - if( rc==SQLITE_TOOBIG ){ - sqlite3Fts3ErrMsg(pzErr, - "FTS expression tree is too large (maximum depth %d)", - SQLITE_FTS3_MAX_EXPR_DEPTH - ); - rc = SQLITE_ERROR; - }else if( rc==SQLITE_ERROR ){ - sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); - } - } - - return rc; -} - -/* -** Free a single node of an expression tree. -*/ -static void fts3FreeExprNode(Fts3Expr *p){ - assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); - sqlite3Fts3EvalPhraseCleanup(p->pPhrase); - sqlite3_free(p->aMI); - sqlite3_free(p); -} - -/* -** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). -** -** This function would be simpler if it recursively called itself. But -** that would mean passing a sufficiently large expression to ExprParse() -** could cause a stack overflow. -*/ -SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ - Fts3Expr *p; - assert( pDel==0 || pDel->pParent==0 ); - for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ - assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); - } - while( p ){ - Fts3Expr *pParent = p->pParent; - fts3FreeExprNode(p); - if( pParent && p==pParent->pLeft && pParent->pRight ){ - p = pParent->pRight; - while( p && (p->pLeft || p->pRight) ){ - assert( p==p->pParent->pRight || p==p->pParent->pLeft ); - p = (p->pLeft ? p->pLeft : p->pRight); - } - }else{ - p = pParent; - } - } -} - -/**************************************************************************** -***************************************************************************** -** Everything after this point is just test code. -*/ - -#ifdef SQLITE_TEST - -/* #include */ - -/* -** Function to query the hash-table of tokenizers (see README.tokenizers). -*/ -static int queryTestTokenizer( - sqlite3 *db, - const char *zName, - const sqlite3_tokenizer_module **pp -){ - int rc; - sqlite3_stmt *pStmt; - const char zSql[] = "SELECT fts3_tokenizer(?)"; - - *pp = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - - sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ - memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); - } - } - - return sqlite3_finalize(pStmt); -} - -/* -** Return a pointer to a buffer containing a text representation of the -** expression passed as the first argument. The buffer is obtained from -** sqlite3_malloc(). It is the responsibility of the caller to use -** sqlite3_free() to release the memory. If an OOM condition is encountered, -** NULL is returned. -** -** If the second argument is not NULL, then its contents are prepended to -** the returned expression text and then freed using sqlite3_free(). -*/ -static char *exprToString(Fts3Expr *pExpr, char *zBuf){ - if( pExpr==0 ){ - return sqlite3_mprintf(""); - } - switch( pExpr->eType ){ - case FTSQUERY_PHRASE: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - int i; - zBuf = sqlite3_mprintf( - "%zPHRASE %d 0", zBuf, pPhrase->iColumn); - for(i=0; zBuf && inToken; i++){ - zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, - pPhrase->aToken[i].n, pPhrase->aToken[i].z, - (pPhrase->aToken[i].isPrefix?"+":"") - ); - } - return zBuf; - } - - case FTSQUERY_NEAR: - zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); - break; - case FTSQUERY_NOT: - zBuf = sqlite3_mprintf("%zNOT ", zBuf); - break; - case FTSQUERY_AND: - zBuf = sqlite3_mprintf("%zAND ", zBuf); - break; - case FTSQUERY_OR: - zBuf = sqlite3_mprintf("%zOR ", zBuf); - break; - } - - if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); - if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); - if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); - - if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); - if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); - - return zBuf; -} - -/* -** This is the implementation of a scalar SQL function used to test the -** expression parser. It should be called as follows: -** -** fts3_exprtest(, , , ...); -** -** The first argument, , is the name of the fts3 tokenizer used -** to parse the query expression (see README.tokenizers). The second argument -** is the query expression to parse. Each subsequent argument is the name -** of a column of the fts3 table that the query expression may refer to. -** For example: -** -** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); -*/ -static void fts3ExprTest( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - sqlite3_tokenizer_module const *pModule = 0; - sqlite3_tokenizer *pTokenizer = 0; - int rc; - char **azCol = 0; - const char *zExpr; - int nExpr; - int nCol; - int ii; - Fts3Expr *pExpr; - char *zBuf = 0; - sqlite3 *db = sqlite3_context_db_handle(context); - - if( argc<3 ){ - sqlite3_result_error(context, - "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 - ); - return; - } - - rc = queryTestTokenizer(db, - (const char *)sqlite3_value_text(argv[0]), &pModule); - if( rc==SQLITE_NOMEM ){ - sqlite3_result_error_nomem(context); - goto exprtest_out; - }else if( !pModule ){ - sqlite3_result_error(context, "No such tokenizer module", -1); - goto exprtest_out; - } - - rc = pModule->xCreate(0, 0, &pTokenizer); - assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); - if( rc==SQLITE_NOMEM ){ - sqlite3_result_error_nomem(context); - goto exprtest_out; - } - pTokenizer->pModule = pModule; - - zExpr = (const char *)sqlite3_value_text(argv[1]); - nExpr = sqlite3_value_bytes(argv[1]); - nCol = argc-2; - azCol = (char **)sqlite3_malloc(nCol*sizeof(char *)); - if( !azCol ){ - sqlite3_result_error_nomem(context); - goto exprtest_out; - } - for(ii=0; iixDestroy(pTokenizer); - } - sqlite3_free(azCol); -} - -/* -** Register the query expression parser test function fts3_exprtest() -** with database connection db. -*/ -SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ - int rc = sqlite3_create_function( - db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 - ); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", - -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0 - ); - } - return rc; -} - -#endif -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_expr.c *******************************************/ -/************** Begin file fts3_hash.c ***************************************/ -/* -** 2001 September 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This is the implementation of generic hash-tables used in SQLite. -** We've modified it slightly to serve as a standalone hash table -** implementation for the full-text indexing module. -*/ - -/* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or -** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). -*/ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ -/* #include */ - -/* #include "fts3_hash.h" */ - -/* -** Malloc and Free functions -*/ -static void *fts3HashMalloc(int n){ - void *p = sqlite3_malloc(n); - if( p ){ - memset(p, 0, n); - } - return p; -} -static void fts3HashFree(void *p){ - sqlite3_free(p); -} - -/* Turn bulk memory into a hash table object by initializing the -** fields of the Hash structure. -** -** "pNew" is a pointer to the hash table that is to be initialized. -** keyClass is one of the constants -** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass -** determines what kind of key the hash table will use. "copyKey" is -** true if the hash table should make its own private copy of keys and -** false if it should just use the supplied pointer. -*/ -SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ - assert( pNew!=0 ); - assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); - pNew->keyClass = keyClass; - pNew->copyKey = copyKey; - pNew->first = 0; - pNew->count = 0; - pNew->htsize = 0; - pNew->ht = 0; -} - -/* Remove all entries from a hash table. Reclaim all memory. -** Call this routine to delete a hash table or to reset a hash table -** to the empty state. -*/ -SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){ - Fts3HashElem *elem; /* For looping over all elements of the table */ - - assert( pH!=0 ); - elem = pH->first; - pH->first = 0; - fts3HashFree(pH->ht); - pH->ht = 0; - pH->htsize = 0; - while( elem ){ - Fts3HashElem *next_elem = elem->next; - if( pH->copyKey && elem->pKey ){ - fts3HashFree(elem->pKey); - } - fts3HashFree(elem); - elem = next_elem; - } - pH->count = 0; -} - -/* -** Hash and comparison functions when the mode is FTS3_HASH_STRING -*/ -static int fts3StrHash(const void *pKey, int nKey){ - const char *z = (const char *)pKey; - unsigned h = 0; - if( nKey<=0 ) nKey = (int) strlen(z); - while( nKey > 0 ){ - h = (h<<3) ^ h ^ *z++; - nKey--; - } - return (int)(h & 0x7fffffff); -} -static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ - if( n1!=n2 ) return 1; - return strncmp((const char*)pKey1,(const char*)pKey2,n1); -} - -/* -** Hash and comparison functions when the mode is FTS3_HASH_BINARY -*/ -static int fts3BinHash(const void *pKey, int nKey){ - int h = 0; - const char *z = (const char *)pKey; - while( nKey-- > 0 ){ - h = (h<<3) ^ h ^ *(z++); - } - return h & 0x7fffffff; -} -static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ - if( n1!=n2 ) return 1; - return memcmp(pKey1,pKey2,n1); -} - -/* -** Return a pointer to the appropriate hash function given the key class. -** -** The C syntax in this function definition may be unfamilar to some -** programmers, so we provide the following additional explanation: -** -** The name of the function is "ftsHashFunction". The function takes a -** single parameter "keyClass". The return value of ftsHashFunction() -** is a pointer to another function. Specifically, the return value -** of ftsHashFunction() is a pointer to a function that takes two parameters -** with types "const void*" and "int" and returns an "int". -*/ -static int (*ftsHashFunction(int keyClass))(const void*,int){ - if( keyClass==FTS3_HASH_STRING ){ - return &fts3StrHash; - }else{ - assert( keyClass==FTS3_HASH_BINARY ); - return &fts3BinHash; - } -} - -/* -** Return a pointer to the appropriate hash function given the key class. -** -** For help in interpreted the obscure C code in the function definition, -** see the header comment on the previous function. -*/ -static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ - if( keyClass==FTS3_HASH_STRING ){ - return &fts3StrCompare; - }else{ - assert( keyClass==FTS3_HASH_BINARY ); - return &fts3BinCompare; - } -} - -/* Link an element into the hash table -*/ -static void fts3HashInsertElement( - Fts3Hash *pH, /* The complete hash table */ - struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ - Fts3HashElem *pNew /* The element to be inserted */ -){ - Fts3HashElem *pHead; /* First element already in pEntry */ - pHead = pEntry->chain; - if( pHead ){ - pNew->next = pHead; - pNew->prev = pHead->prev; - if( pHead->prev ){ pHead->prev->next = pNew; } - else { pH->first = pNew; } - pHead->prev = pNew; - }else{ - pNew->next = pH->first; - if( pH->first ){ pH->first->prev = pNew; } - pNew->prev = 0; - pH->first = pNew; - } - pEntry->count++; - pEntry->chain = pNew; -} - - -/* Resize the hash table so that it cantains "new_size" buckets. -** "new_size" must be a power of 2. The hash table might fail -** to resize if sqliteMalloc() fails. -** -** Return non-zero if a memory allocation error occurs. -*/ -static int fts3Rehash(Fts3Hash *pH, int new_size){ - struct _fts3ht *new_ht; /* The new hash table */ - Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ - int (*xHash)(const void*,int); /* The hash function */ - - assert( (new_size & (new_size-1))==0 ); - new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); - if( new_ht==0 ) return 1; - fts3HashFree(pH->ht); - pH->ht = new_ht; - pH->htsize = new_size; - xHash = ftsHashFunction(pH->keyClass); - for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); - next_elem = elem->next; - fts3HashInsertElement(pH, &new_ht[h], elem); - } - return 0; -} - -/* This function (for internal use only) locates an element in an -** hash table that matches the given key. The hash for this key has -** already been computed and is passed as the 4th parameter. -*/ -static Fts3HashElem *fts3FindElementByHash( - const Fts3Hash *pH, /* The pH to be searched */ - const void *pKey, /* The key we are searching for */ - int nKey, - int h /* The hash for this key. */ -){ - Fts3HashElem *elem; /* Used to loop thru the element list */ - int count; /* Number of elements left to test */ - int (*xCompare)(const void*,int,const void*,int); /* comparison function */ - - if( pH->ht ){ - struct _fts3ht *pEntry = &pH->ht[h]; - elem = pEntry->chain; - count = pEntry->count; - xCompare = ftsCompareFunction(pH->keyClass); - while( count-- && elem ){ - if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ - return elem; - } - elem = elem->next; - } - } - return 0; -} - -/* Remove a single entry from the hash table given a pointer to that -** element and a hash on the element's key. -*/ -static void fts3RemoveElementByHash( - Fts3Hash *pH, /* The pH containing "elem" */ - Fts3HashElem* elem, /* The element to be removed from the pH */ - int h /* Hash value for the element */ -){ - struct _fts3ht *pEntry; - if( elem->prev ){ - elem->prev->next = elem->next; - }else{ - pH->first = elem->next; - } - if( elem->next ){ - elem->next->prev = elem->prev; - } - pEntry = &pH->ht[h]; - if( pEntry->chain==elem ){ - pEntry->chain = elem->next; - } - pEntry->count--; - if( pEntry->count<=0 ){ - pEntry->chain = 0; - } - if( pH->copyKey && elem->pKey ){ - fts3HashFree(elem->pKey); - } - fts3HashFree( elem ); - pH->count--; - if( pH->count<=0 ){ - assert( pH->first==0 ); - assert( pH->count==0 ); - fts3HashClear(pH); - } -} - -SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem( - const Fts3Hash *pH, - const void *pKey, - int nKey -){ - int h; /* A hash on key */ - int (*xHash)(const void*,int); /* The hash function */ - - if( pH==0 || pH->ht==0 ) return 0; - xHash = ftsHashFunction(pH->keyClass); - assert( xHash!=0 ); - h = (*xHash)(pKey,nKey); - assert( (pH->htsize & (pH->htsize-1))==0 ); - return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); -} - -/* -** Attempt to locate an element of the hash table pH with a key -** that matches pKey,nKey. Return the data for this element if it is -** found, or NULL if there is no match. -*/ -SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ - Fts3HashElem *pElem; /* The element that matches key (if any) */ - - pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); - return pElem ? pElem->data : 0; -} - -/* Insert an element into the hash table pH. The key is pKey,nKey -** and the data is "data". -** -** If no element exists with a matching key, then a new -** element is created. A copy of the key is made if the copyKey -** flag is set. NULL is returned. -** -** If another element already exists with the same key, then the -** new data replaces the old data and the old data is returned. -** The key is not copied in this instance. If a malloc fails, then -** the new data is returned and the hash table is unchanged. -** -** If the "data" parameter to this function is NULL, then the -** element corresponding to "key" is removed from the hash table. -*/ -SQLITE_PRIVATE void *sqlite3Fts3HashInsert( - Fts3Hash *pH, /* The hash table to insert into */ - const void *pKey, /* The key */ - int nKey, /* Number of bytes in the key */ - void *data /* The data */ -){ - int hraw; /* Raw hash value of the key */ - int h; /* the hash of the key modulo hash table size */ - Fts3HashElem *elem; /* Used to loop thru the element list */ - Fts3HashElem *new_elem; /* New element added to the pH */ - int (*xHash)(const void*,int); /* The hash function */ - - assert( pH!=0 ); - xHash = ftsHashFunction(pH->keyClass); - assert( xHash!=0 ); - hraw = (*xHash)(pKey, nKey); - assert( (pH->htsize & (pH->htsize-1))==0 ); - h = hraw & (pH->htsize-1); - elem = fts3FindElementByHash(pH,pKey,nKey,h); - if( elem ){ - void *old_data = elem->data; - if( data==0 ){ - fts3RemoveElementByHash(pH,elem,h); - }else{ - elem->data = data; - } - return old_data; - } - if( data==0 ) return 0; - if( (pH->htsize==0 && fts3Rehash(pH,8)) - || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) - ){ - pH->count = 0; - return data; - } - assert( pH->htsize>0 ); - new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); - if( new_elem==0 ) return data; - if( pH->copyKey && pKey!=0 ){ - new_elem->pKey = fts3HashMalloc( nKey ); - if( new_elem->pKey==0 ){ - fts3HashFree(new_elem); - return data; - } - memcpy((void*)new_elem->pKey, pKey, nKey); - }else{ - new_elem->pKey = (void*)pKey; - } - new_elem->nKey = nKey; - pH->count++; - assert( pH->htsize>0 ); - assert( (pH->htsize & (pH->htsize-1))==0 ); - h = hraw & (pH->htsize-1); - fts3HashInsertElement(pH, &pH->ht[h], new_elem); - new_elem->data = data; - return 0; -} - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_hash.c *******************************************/ -/************** Begin file fts3_porter.c *************************************/ -/* -** 2006 September 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Implementation of the full-text-search tokenizer that implements -** a Porter stemmer. -*/ - -/* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or -** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). -*/ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ -/* #include */ -/* #include */ - -/* #include "fts3_tokenizer.h" */ - -/* -** Class derived from sqlite3_tokenizer -*/ -typedef struct porter_tokenizer { - sqlite3_tokenizer base; /* Base class */ -} porter_tokenizer; - -/* -** Class derived from sqlite3_tokenizer_cursor -*/ -typedef struct porter_tokenizer_cursor { - sqlite3_tokenizer_cursor base; - const char *zInput; /* input we are tokenizing */ - int nInput; /* size of the input */ - int iOffset; /* current position in zInput */ - int iToken; /* index of next token to be returned */ - char *zToken; /* storage for current token */ - int nAllocated; /* space allocated to zToken buffer */ -} porter_tokenizer_cursor; - - -/* -** Create a new tokenizer instance. -*/ -static int porterCreate( - int argc, const char * const *argv, - sqlite3_tokenizer **ppTokenizer -){ - porter_tokenizer *t; - - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - - t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); - if( t==NULL ) return SQLITE_NOMEM; - memset(t, 0, sizeof(*t)); - *ppTokenizer = &t->base; - return SQLITE_OK; -} - -/* -** Destroy a tokenizer -*/ -static int porterDestroy(sqlite3_tokenizer *pTokenizer){ - sqlite3_free(pTokenizer); - return SQLITE_OK; -} - -/* -** Prepare to begin tokenizing a particular string. The input -** string to be tokenized is zInput[0..nInput-1]. A cursor -** used to incrementally tokenize this string is returned in -** *ppCursor. -*/ -static int porterOpen( - sqlite3_tokenizer *pTokenizer, /* The tokenizer */ - const char *zInput, int nInput, /* String to be tokenized */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ -){ - porter_tokenizer_cursor *c; - - UNUSED_PARAMETER(pTokenizer); - - c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); - if( c==NULL ) return SQLITE_NOMEM; - - c->zInput = zInput; - if( zInput==0 ){ - c->nInput = 0; - }else if( nInput<0 ){ - c->nInput = (int)strlen(zInput); - }else{ - c->nInput = nInput; - } - c->iOffset = 0; /* start tokenizing at the beginning */ - c->iToken = 0; - c->zToken = NULL; /* no space allocated, yet. */ - c->nAllocated = 0; - - *ppCursor = &c->base; - return SQLITE_OK; -} - -/* -** Close a tokenization cursor previously opened by a call to -** porterOpen() above. -*/ -static int porterClose(sqlite3_tokenizer_cursor *pCursor){ - porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; - sqlite3_free(c->zToken); - sqlite3_free(c); - return SQLITE_OK; -} -/* -** Vowel or consonant -*/ -static const char cType[] = { - 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 2, 1 -}; - -/* -** isConsonant() and isVowel() determine if their first character in -** the string they point to is a consonant or a vowel, according -** to Porter ruls. -** -** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. -** 'Y' is a consonant unless it follows another consonant, -** in which case it is a vowel. -** -** In these routine, the letters are in reverse order. So the 'y' rule -** is that 'y' is a consonant unless it is followed by another -** consonent. -*/ -static int isVowel(const char*); -static int isConsonant(const char *z){ - int j; - char x = *z; - if( x==0 ) return 0; - assert( x>='a' && x<='z' ); - j = cType[x-'a']; - if( j<2 ) return j; - return z[1]==0 || isVowel(z + 1); -} -static int isVowel(const char *z){ - int j; - char x = *z; - if( x==0 ) return 0; - assert( x>='a' && x<='z' ); - j = cType[x-'a']; - if( j<2 ) return 1-j; - return isConsonant(z + 1); -} - -/* -** Let any sequence of one or more vowels be represented by V and let -** C be sequence of one or more consonants. Then every word can be -** represented as: -** -** [C] (VC){m} [V] -** -** In prose: A word is an optional consonant followed by zero or -** vowel-consonant pairs followed by an optional vowel. "m" is the -** number of vowel consonant pairs. This routine computes the value -** of m for the first i bytes of a word. -** -** Return true if the m-value for z is 1 or more. In other words, -** return true if z contains at least one vowel that is followed -** by a consonant. -** -** In this routine z[] is in reverse order. So we are really looking -** for an instance of a consonant followed by a vowel. -*/ -static int m_gt_0(const char *z){ - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - return *z!=0; -} - -/* Like mgt0 above except we are looking for a value of m which is -** exactly 1 -*/ -static int m_eq_1(const char *z){ - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - if( *z==0 ) return 0; - while( isVowel(z) ){ z++; } - if( *z==0 ) return 1; - while( isConsonant(z) ){ z++; } - return *z==0; -} - -/* Like mgt0 above except we are looking for a value of m>1 instead -** or m>0 -*/ -static int m_gt_1(const char *z){ - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - if( *z==0 ) return 0; - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - return *z!=0; -} - -/* -** Return TRUE if there is a vowel anywhere within z[0..n-1] -*/ -static int hasVowel(const char *z){ - while( isConsonant(z) ){ z++; } - return *z!=0; -} - -/* -** Return TRUE if the word ends in a double consonant. -** -** The text is reversed here. So we are really looking at -** the first two characters of z[]. -*/ -static int doubleConsonant(const char *z){ - return isConsonant(z) && z[0]==z[1]; -} - -/* -** Return TRUE if the word ends with three letters which -** are consonant-vowel-consonent and where the final consonant -** is not 'w', 'x', or 'y'. -** -** The word is reversed here. So we are really checking the -** first three letters and the first one cannot be in [wxy]. -*/ -static int star_oh(const char *z){ - return - isConsonant(z) && - z[0]!='w' && z[0]!='x' && z[0]!='y' && - isVowel(z+1) && - isConsonant(z+2); -} - -/* -** If the word ends with zFrom and xCond() is true for the stem -** of the word that preceeds the zFrom ending, then change the -** ending to zTo. -** -** The input word *pz and zFrom are both in reverse order. zTo -** is in normal order. -** -** Return TRUE if zFrom matches. Return FALSE if zFrom does not -** match. Not that TRUE is returned even if xCond() fails and -** no substitution occurs. -*/ -static int stem( - char **pz, /* The word being stemmed (Reversed) */ - const char *zFrom, /* If the ending matches this... (Reversed) */ - const char *zTo, /* ... change the ending to this (not reversed) */ - int (*xCond)(const char*) /* Condition that must be true */ -){ - char *z = *pz; - while( *zFrom && *zFrom==*z ){ z++; zFrom++; } - if( *zFrom!=0 ) return 0; - if( xCond && !xCond(z) ) return 1; - while( *zTo ){ - *(--z) = *(zTo++); - } - *pz = z; - return 1; -} - -/* -** This is the fallback stemmer used when the porter stemmer is -** inappropriate. The input word is copied into the output with -** US-ASCII case folding. If the input word is too long (more -** than 20 bytes if it contains no digits or more than 6 bytes if -** it contains digits) then word is truncated to 20 or 6 bytes -** by taking 10 or 3 bytes from the beginning and end. -*/ -static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ - int i, mx, j; - int hasDigit = 0; - for(i=0; i='A' && c<='Z' ){ - zOut[i] = c - 'A' + 'a'; - }else{ - if( c>='0' && c<='9' ) hasDigit = 1; - zOut[i] = c; - } - } - mx = hasDigit ? 3 : 10; - if( nIn>mx*2 ){ - for(j=mx, i=nIn-mx; i=(int)sizeof(zReverse)-7 ){ - /* The word is too big or too small for the porter stemmer. - ** Fallback to the copy stemmer */ - copy_stemmer(zIn, nIn, zOut, pnOut); - return; - } - for(i=0, j=sizeof(zReverse)-6; i='A' && c<='Z' ){ - zReverse[j] = c + 'a' - 'A'; - }else if( c>='a' && c<='z' ){ - zReverse[j] = c; - }else{ - /* The use of a character not in [a-zA-Z] means that we fallback - ** to the copy stemmer */ - copy_stemmer(zIn, nIn, zOut, pnOut); - return; - } - } - memset(&zReverse[sizeof(zReverse)-5], 0, 5); - z = &zReverse[j+1]; - - - /* Step 1a */ - if( z[0]=='s' ){ - if( - !stem(&z, "sess", "ss", 0) && - !stem(&z, "sei", "i", 0) && - !stem(&z, "ss", "ss", 0) - ){ - z++; - } - } - - /* Step 1b */ - z2 = z; - if( stem(&z, "dee", "ee", m_gt_0) ){ - /* Do nothing. The work was all in the test */ - }else if( - (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) - && z!=z2 - ){ - if( stem(&z, "ta", "ate", 0) || - stem(&z, "lb", "ble", 0) || - stem(&z, "zi", "ize", 0) ){ - /* Do nothing. The work was all in the test */ - }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ - z++; - }else if( m_eq_1(z) && star_oh(z) ){ - *(--z) = 'e'; - } - } - - /* Step 1c */ - if( z[0]=='y' && hasVowel(z+1) ){ - z[0] = 'i'; - } - - /* Step 2 */ - switch( z[1] ){ - case 'a': - if( !stem(&z, "lanoita", "ate", m_gt_0) ){ - stem(&z, "lanoit", "tion", m_gt_0); - } - break; - case 'c': - if( !stem(&z, "icne", "ence", m_gt_0) ){ - stem(&z, "icna", "ance", m_gt_0); - } - break; - case 'e': - stem(&z, "rezi", "ize", m_gt_0); - break; - case 'g': - stem(&z, "igol", "log", m_gt_0); - break; - case 'l': - if( !stem(&z, "ilb", "ble", m_gt_0) - && !stem(&z, "illa", "al", m_gt_0) - && !stem(&z, "iltne", "ent", m_gt_0) - && !stem(&z, "ile", "e", m_gt_0) - ){ - stem(&z, "ilsuo", "ous", m_gt_0); - } - break; - case 'o': - if( !stem(&z, "noitazi", "ize", m_gt_0) - && !stem(&z, "noita", "ate", m_gt_0) - ){ - stem(&z, "rota", "ate", m_gt_0); - } - break; - case 's': - if( !stem(&z, "msila", "al", m_gt_0) - && !stem(&z, "ssenevi", "ive", m_gt_0) - && !stem(&z, "ssenluf", "ful", m_gt_0) - ){ - stem(&z, "ssensuo", "ous", m_gt_0); - } - break; - case 't': - if( !stem(&z, "itila", "al", m_gt_0) - && !stem(&z, "itivi", "ive", m_gt_0) - ){ - stem(&z, "itilib", "ble", m_gt_0); - } - break; - } - - /* Step 3 */ - switch( z[0] ){ - case 'e': - if( !stem(&z, "etaci", "ic", m_gt_0) - && !stem(&z, "evita", "", m_gt_0) - ){ - stem(&z, "ezila", "al", m_gt_0); - } - break; - case 'i': - stem(&z, "itici", "ic", m_gt_0); - break; - case 'l': - if( !stem(&z, "laci", "ic", m_gt_0) ){ - stem(&z, "luf", "", m_gt_0); - } - break; - case 's': - stem(&z, "ssen", "", m_gt_0); - break; - } - - /* Step 4 */ - switch( z[1] ){ - case 'a': - if( z[0]=='l' && m_gt_1(z+2) ){ - z += 2; - } - break; - case 'c': - if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ - z += 4; - } - break; - case 'e': - if( z[0]=='r' && m_gt_1(z+2) ){ - z += 2; - } - break; - case 'i': - if( z[0]=='c' && m_gt_1(z+2) ){ - z += 2; - } - break; - case 'l': - if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ - z += 4; - } - break; - case 'n': - if( z[0]=='t' ){ - if( z[2]=='a' ){ - if( m_gt_1(z+3) ){ - z += 3; - } - }else if( z[2]=='e' ){ - if( !stem(&z, "tneme", "", m_gt_1) - && !stem(&z, "tnem", "", m_gt_1) - ){ - stem(&z, "tne", "", m_gt_1); - } - } - } - break; - case 'o': - if( z[0]=='u' ){ - if( m_gt_1(z+2) ){ - z += 2; - } - }else if( z[3]=='s' || z[3]=='t' ){ - stem(&z, "noi", "", m_gt_1); - } - break; - case 's': - if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ - z += 3; - } - break; - case 't': - if( !stem(&z, "eta", "", m_gt_1) ){ - stem(&z, "iti", "", m_gt_1); - } - break; - case 'u': - if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ - z += 3; - } - break; - case 'v': - case 'z': - if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ - z += 3; - } - break; - } - - /* Step 5a */ - if( z[0]=='e' ){ - if( m_gt_1(z+1) ){ - z++; - }else if( m_eq_1(z+1) && !star_oh(z+1) ){ - z++; - } - } - - /* Step 5b */ - if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ - z++; - } - - /* z[] is now the stemmed word in reverse order. Flip it back - ** around into forward order and return. - */ - *pnOut = i = (int)strlen(z); - zOut[i] = 0; - while( *z ){ - zOut[--i] = *(z++); - } -} - -/* -** Characters that can be part of a token. We assume any character -** whose value is greater than 0x80 (any UTF character) can be -** part of a token. In other words, delimiters all must have -** values of 0x7f or lower. -*/ -static const char porterIdChar[] = { -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ -}; -#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30])) - -/* -** Extract the next token from a tokenization cursor. The cursor must -** have been opened by a prior call to porterOpen(). -*/ -static int porterNext( - sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ - const char **pzToken, /* OUT: *pzToken is the token text */ - int *pnBytes, /* OUT: Number of bytes in token */ - int *piStartOffset, /* OUT: Starting offset of token */ - int *piEndOffset, /* OUT: Ending offset of token */ - int *piPosition /* OUT: Position integer of token */ -){ - porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; - const char *z = c->zInput; - - while( c->iOffsetnInput ){ - int iStartOffset, ch; - - /* Scan past delimiter characters */ - while( c->iOffsetnInput && isDelim(z[c->iOffset]) ){ - c->iOffset++; - } - - /* Count non-delimiter characters. */ - iStartOffset = c->iOffset; - while( c->iOffsetnInput && !isDelim(z[c->iOffset]) ){ - c->iOffset++; - } - - if( c->iOffset>iStartOffset ){ - int n = c->iOffset-iStartOffset; - if( n>c->nAllocated ){ - char *pNew; - c->nAllocated = n+20; - pNew = sqlite3_realloc(c->zToken, c->nAllocated); - if( !pNew ) return SQLITE_NOMEM; - c->zToken = pNew; - } - porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); - *pzToken = c->zToken; - *piStartOffset = iStartOffset; - *piEndOffset = c->iOffset; - *piPosition = c->iToken++; - return SQLITE_OK; - } - } - return SQLITE_DONE; -} - -/* -** The set of routines that implement the porter-stemmer tokenizer -*/ -static const sqlite3_tokenizer_module porterTokenizerModule = { - 0, - porterCreate, - porterDestroy, - porterOpen, - porterClose, - porterNext, - 0 -}; - -/* -** Allocate a new porter tokenizer. Return a pointer to the new -** tokenizer in *ppModule -*/ -SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( - sqlite3_tokenizer_module const**ppModule -){ - *ppModule = &porterTokenizerModule; -} - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_porter.c *****************************************/ -/************** Begin file fts3_tokenizer.c **********************************/ -/* -** 2007 June 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This is part of an SQLite module implementing full-text search. -** This particular file implements the generic tokenizer interface. -*/ - -/* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or -** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). -*/ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ - -/* -** Return true if the two-argument version of fts3_tokenizer() -** has been activated via a prior call to sqlite3_db_config(db, -** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); -*/ -static int fts3TokenizerEnabled(sqlite3_context *context){ - sqlite3 *db = sqlite3_context_db_handle(context); - int isEnabled = 0; - sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled); - return isEnabled; -} - -/* -** Implementation of the SQL scalar function for accessing the underlying -** hash table. This function may be called as follows: -** -** SELECT (); -** SELECT (, ); -** -** where is the name passed as the second argument -** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). -** -** If the argument is specified, it must be a blob value -** containing a pointer to be stored as the hash data corresponding -** to the string . If is not specified, then -** the string must already exist in the has table. Otherwise, -** an error is returned. -** -** Whether or not the argument is specified, the value returned -** is a blob containing the pointer stored as the hash data corresponding -** to string (after the hash-table is updated, if applicable). -*/ -static void fts3TokenizerFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - Fts3Hash *pHash; - void *pPtr = 0; - const unsigned char *zName; - int nName; - - assert( argc==1 || argc==2 ); - - pHash = (Fts3Hash *)sqlite3_user_data(context); - - zName = sqlite3_value_text(argv[0]); - nName = sqlite3_value_bytes(argv[0])+1; - - if( argc==2 ){ - if( fts3TokenizerEnabled(context) ){ - void *pOld; - int n = sqlite3_value_bytes(argv[1]); - if( zName==0 || n!=sizeof(pPtr) ){ - sqlite3_result_error(context, "argument type mismatch", -1); - return; - } - pPtr = *(void **)sqlite3_value_blob(argv[1]); - pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); - if( pOld==pPtr ){ - sqlite3_result_error(context, "out of memory", -1); - } - }else{ - sqlite3_result_error(context, "fts3tokenize disabled", -1); - return; - } - }else{ - if( zName ){ - pPtr = sqlite3Fts3HashFind(pHash, zName, nName); - } - if( !pPtr ){ - char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); - return; - } - } - sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); -} - -SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ - static const char isFtsIdChar[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ - }; - return (c&0x80 || isFtsIdChar[(int)(c)]); -} - -SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ - const char *z1; - const char *z2 = 0; - - /* Find the start of the next token. */ - z1 = zStr; - while( z2==0 ){ - char c = *z1; - switch( c ){ - case '\0': return 0; /* No more tokens here */ - case '\'': - case '"': - case '`': { - z2 = z1; - while( *++z2 && (*z2!=c || *++z2==c) ); - break; - } - case '[': - z2 = &z1[1]; - while( *z2 && z2[0]!=']' ) z2++; - if( *z2 ) z2++; - break; - - default: - if( sqlite3Fts3IsIdChar(*z1) ){ - z2 = &z1[1]; - while( sqlite3Fts3IsIdChar(*z2) ) z2++; - }else{ - z1++; - } - } - } - - *pn = (int)(z2-z1); - return z1; -} - -SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( - Fts3Hash *pHash, /* Tokenizer hash table */ - const char *zArg, /* Tokenizer name */ - sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ - char **pzErr /* OUT: Set to malloced error message */ -){ - int rc; - char *z = (char *)zArg; - int n = 0; - char *zCopy; - char *zEnd; /* Pointer to nul-term of zCopy */ - sqlite3_tokenizer_module *m; - - zCopy = sqlite3_mprintf("%s", zArg); - if( !zCopy ) return SQLITE_NOMEM; - zEnd = &zCopy[strlen(zCopy)]; - - z = (char *)sqlite3Fts3NextToken(zCopy, &n); - if( z==0 ){ - assert( n==0 ); - z = zCopy; - } - z[n] = '\0'; - sqlite3Fts3Dequote(z); - - m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); - if( !m ){ - sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z); - rc = SQLITE_ERROR; - }else{ - char const **aArg = 0; - int iArg = 0; - z = &z[n+1]; - while( zxCreate(iArg, aArg, ppTok); - assert( rc!=SQLITE_OK || *ppTok ); - if( rc!=SQLITE_OK ){ - sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer"); - }else{ - (*ppTok)->pModule = m; - } - sqlite3_free((void *)aArg); - } - - sqlite3_free(zCopy); - return rc; -} - - -#ifdef SQLITE_TEST - -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif -/* #include */ - -/* -** Implementation of a special SQL scalar function for testing tokenizers -** designed to be used in concert with the Tcl testing framework. This -** function must be called with two or more arguments: -** -** SELECT (, ..., ); -** -** where is the name passed as the second argument -** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') -** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). -** -** The return value is a string that may be interpreted as a Tcl -** list. For each token in the , three elements are -** added to the returned list. The first is the token position, the -** second is the token text (folded, stemmed, etc.) and the third is the -** substring of associated with the token. For example, -** using the built-in "simple" tokenizer: -** -** SELECT fts_tokenizer_test('simple', 'I don't see how'); -** -** will return the string: -** -** "{0 i I 1 dont don't 2 see see 3 how how}" -** -*/ -static void testFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - Fts3Hash *pHash; - sqlite3_tokenizer_module *p; - sqlite3_tokenizer *pTokenizer = 0; - sqlite3_tokenizer_cursor *pCsr = 0; - - const char *zErr = 0; - - const char *zName; - int nName; - const char *zInput; - int nInput; - - const char *azArg[64]; - - const char *zToken; - int nToken = 0; - int iStart = 0; - int iEnd = 0; - int iPos = 0; - int i; - - Tcl_Obj *pRet; - - if( argc<2 ){ - sqlite3_result_error(context, "insufficient arguments", -1); - return; - } - - nName = sqlite3_value_bytes(argv[0]); - zName = (const char *)sqlite3_value_text(argv[0]); - nInput = sqlite3_value_bytes(argv[argc-1]); - zInput = (const char *)sqlite3_value_text(argv[argc-1]); - - pHash = (Fts3Hash *)sqlite3_user_data(context); - p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); - - if( !p ){ - char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName); - sqlite3_result_error(context, zErr2, -1); - sqlite3_free(zErr2); - return; - } - - pRet = Tcl_NewObj(); - Tcl_IncrRefCount(pRet); - - for(i=1; ixCreate(argc-2, azArg, &pTokenizer) ){ - zErr = "error in xCreate()"; - goto finish; - } - pTokenizer->pModule = p; - if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ - zErr = "error in xOpen()"; - goto finish; - } - - while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ - Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); - Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); - zToken = &zInput[iStart]; - nToken = iEnd-iStart; - Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); - } - - if( SQLITE_OK!=p->xClose(pCsr) ){ - zErr = "error in xClose()"; - goto finish; - } - if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ - zErr = "error in xDestroy()"; - goto finish; - } - -finish: - if( zErr ){ - sqlite3_result_error(context, zErr, -1); - }else{ - sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); - } - Tcl_DecrRefCount(pRet); -} - -static -int registerTokenizer( - sqlite3 *db, - char *zName, - const sqlite3_tokenizer_module *p -){ - int rc; - sqlite3_stmt *pStmt; - const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; - - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - - sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); - sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); - sqlite3_step(pStmt); - - return sqlite3_finalize(pStmt); -} - - -static -int queryTokenizer( - sqlite3 *db, - char *zName, - const sqlite3_tokenizer_module **pp -){ - int rc; - sqlite3_stmt *pStmt; - const char zSql[] = "SELECT fts3_tokenizer(?)"; - - *pp = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - - sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ - memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); - } - } - - return sqlite3_finalize(pStmt); -} - -SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); - -/* -** Implementation of the scalar function fts3_tokenizer_internal_test(). -** This function is used for testing only, it is not included in the -** build unless SQLITE_TEST is defined. -** -** The purpose of this is to test that the fts3_tokenizer() function -** can be used as designed by the C-code in the queryTokenizer and -** registerTokenizer() functions above. These two functions are repeated -** in the README.tokenizer file as an example, so it is important to -** test them. -** -** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar -** function with no arguments. An assert() will fail if a problem is -** detected. i.e.: -** -** SELECT fts3_tokenizer_internal_test(); -** -*/ -static void intTestFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int rc; - const sqlite3_tokenizer_module *p1; - const sqlite3_tokenizer_module *p2; - sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); - - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - - /* Test the query function */ - sqlite3Fts3SimpleTokenizerModule(&p1); - rc = queryTokenizer(db, "simple", &p2); - assert( rc==SQLITE_OK ); - assert( p1==p2 ); - rc = queryTokenizer(db, "nosuchtokenizer", &p2); - assert( rc==SQLITE_ERROR ); - assert( p2==0 ); - assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); - - /* Test the storage function */ - if( fts3TokenizerEnabled(context) ){ - rc = registerTokenizer(db, "nosuchtokenizer", p1); - assert( rc==SQLITE_OK ); - rc = queryTokenizer(db, "nosuchtokenizer", &p2); - assert( rc==SQLITE_OK ); - assert( p2==p1 ); - } - - sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); -} - -#endif - -/* -** Set up SQL objects in database db used to access the contents of -** the hash table pointed to by argument pHash. The hash table must -** been initialized to use string keys, and to take a private copy -** of the key when a value is inserted. i.e. by a call similar to: -** -** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); -** -** This function adds a scalar function (see header comment above -** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is -** defined at compilation time, a temporary virtual table (see header -** comment above struct HashTableVtab) to the database schema. Both -** provide read/write access to the contents of *pHash. -** -** The third argument to this function, zName, is used as the name -** of both the scalar and, if created, the virtual table. -*/ -SQLITE_PRIVATE int sqlite3Fts3InitHashTable( - sqlite3 *db, - Fts3Hash *pHash, - const char *zName -){ - int rc = SQLITE_OK; - void *p = (void *)pHash; - const int any = SQLITE_ANY; - -#ifdef SQLITE_TEST - char *zTest = 0; - char *zTest2 = 0; - void *pdb = (void *)db; - zTest = sqlite3_mprintf("%s_test", zName); - zTest2 = sqlite3_mprintf("%s_internal_test", zName); - if( !zTest || !zTest2 ){ - rc = SQLITE_NOMEM; - } -#endif - - if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0); - } - if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0); - } -#ifdef SQLITE_TEST - if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); - } - if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); - } -#endif - -#ifdef SQLITE_TEST - sqlite3_free(zTest); - sqlite3_free(zTest2); -#endif - - return rc; -} - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_tokenizer.c **************************************/ -/************** Begin file fts3_tokenizer1.c *********************************/ -/* -** 2006 Oct 10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** Implementation of the "simple" full-text-search tokenizer. -*/ - -/* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or -** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). -*/ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ -/* #include */ -/* #include */ - -/* #include "fts3_tokenizer.h" */ - -typedef struct simple_tokenizer { - sqlite3_tokenizer base; - char delim[128]; /* flag ASCII delimiters */ -} simple_tokenizer; - -typedef struct simple_tokenizer_cursor { - sqlite3_tokenizer_cursor base; - const char *pInput; /* input we are tokenizing */ - int nBytes; /* size of the input */ - int iOffset; /* current position in pInput */ - int iToken; /* index of next token to be returned */ - char *pToken; /* storage for current token */ - int nTokenAllocated; /* space allocated to zToken buffer */ -} simple_tokenizer_cursor; - - -static int simpleDelim(simple_tokenizer *t, unsigned char c){ - return c<0x80 && t->delim[c]; -} -static int fts3_isalnum(int x){ - return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); -} - -/* -** Create a new tokenizer instance. -*/ -static int simpleCreate( - int argc, const char * const *argv, - sqlite3_tokenizer **ppTokenizer -){ - simple_tokenizer *t; - - t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); - if( t==NULL ) return SQLITE_NOMEM; - memset(t, 0, sizeof(*t)); - - /* TODO(shess) Delimiters need to remain the same from run to run, - ** else we need to reindex. One solution would be a meta-table to - ** track such information in the database, then we'd only want this - ** information on the initial create. - */ - if( argc>1 ){ - int i, n = (int)strlen(argv[1]); - for(i=0; i=0x80 ){ - sqlite3_free(t); - return SQLITE_ERROR; - } - t->delim[ch] = 1; - } - } else { - /* Mark non-alphanumeric ASCII characters as delimiters */ - int i; - for(i=1; i<0x80; i++){ - t->delim[i] = !fts3_isalnum(i) ? -1 : 0; - } - } - - *ppTokenizer = &t->base; - return SQLITE_OK; -} - -/* -** Destroy a tokenizer -*/ -static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ - sqlite3_free(pTokenizer); - return SQLITE_OK; -} - -/* -** Prepare to begin tokenizing a particular string. The input -** string to be tokenized is pInput[0..nBytes-1]. A cursor -** used to incrementally tokenize this string is returned in -** *ppCursor. -*/ -static int simpleOpen( - sqlite3_tokenizer *pTokenizer, /* The tokenizer */ - const char *pInput, int nBytes, /* String to be tokenized */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ -){ - simple_tokenizer_cursor *c; - - UNUSED_PARAMETER(pTokenizer); - - c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); - if( c==NULL ) return SQLITE_NOMEM; - - c->pInput = pInput; - if( pInput==0 ){ - c->nBytes = 0; - }else if( nBytes<0 ){ - c->nBytes = (int)strlen(pInput); - }else{ - c->nBytes = nBytes; - } - c->iOffset = 0; /* start tokenizing at the beginning */ - c->iToken = 0; - c->pToken = NULL; /* no space allocated, yet. */ - c->nTokenAllocated = 0; - - *ppCursor = &c->base; - return SQLITE_OK; -} - -/* -** Close a tokenization cursor previously opened by a call to -** simpleOpen() above. -*/ -static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ - simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; - sqlite3_free(c->pToken); - sqlite3_free(c); - return SQLITE_OK; -} - -/* -** Extract the next token from a tokenization cursor. The cursor must -** have been opened by a prior call to simpleOpen(). -*/ -static int simpleNext( - sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ - const char **ppToken, /* OUT: *ppToken is the token text */ - int *pnBytes, /* OUT: Number of bytes in token */ - int *piStartOffset, /* OUT: Starting offset of token */ - int *piEndOffset, /* OUT: Ending offset of token */ - int *piPosition /* OUT: Position integer of token */ -){ - simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; - simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; - unsigned char *p = (unsigned char *)c->pInput; - - while( c->iOffsetnBytes ){ - int iStartOffset; - - /* Scan past delimiter characters */ - while( c->iOffsetnBytes && simpleDelim(t, p[c->iOffset]) ){ - c->iOffset++; - } - - /* Count non-delimiter characters. */ - iStartOffset = c->iOffset; - while( c->iOffsetnBytes && !simpleDelim(t, p[c->iOffset]) ){ - c->iOffset++; - } - - if( c->iOffset>iStartOffset ){ - int i, n = c->iOffset-iStartOffset; - if( n>c->nTokenAllocated ){ - char *pNew; - c->nTokenAllocated = n+20; - pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); - if( !pNew ) return SQLITE_NOMEM; - c->pToken = pNew; - } - for(i=0; ipToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); - } - *ppToken = c->pToken; - *pnBytes = n; - *piStartOffset = iStartOffset; - *piEndOffset = c->iOffset; - *piPosition = c->iToken++; - - return SQLITE_OK; - } - } - return SQLITE_DONE; -} - -/* -** The set of routines that implement the simple tokenizer -*/ -static const sqlite3_tokenizer_module simpleTokenizerModule = { - 0, - simpleCreate, - simpleDestroy, - simpleOpen, - simpleClose, - simpleNext, - 0, -}; - -/* -** Allocate a new simple tokenizer. Return a pointer to the new -** tokenizer in *ppModule -*/ -SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( - sqlite3_tokenizer_module const**ppModule -){ - *ppModule = &simpleTokenizerModule; -} - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_tokenizer1.c *************************************/ -/************** Begin file fts3_tokenize_vtab.c ******************************/ -/* -** 2013 Apr 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains code for the "fts3tokenize" virtual table module. -** An fts3tokenize virtual table is created as follows: -** -** CREATE VIRTUAL TABLE USING fts3tokenize( -** , , ... -** ); -** -** The table created has the following schema: -** -** CREATE TABLE (input, token, start, end, position) -** -** When queried, the query must include a WHERE clause of type: -** -** input = -** -** The virtual table module tokenizes this , using the FTS3 -** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE -** statement and returns one row for each token in the result. With -** fields set as follows: -** -** input: Always set to a copy of -** token: A token from the input. -** start: Byte offset of the token within the input . -** end: Byte offset of the byte immediately following the end of the -** token within the input string. -** pos: Token offset of token within input. -** -*/ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ - -typedef struct Fts3tokTable Fts3tokTable; -typedef struct Fts3tokCursor Fts3tokCursor; - -/* -** Virtual table structure. -*/ -struct Fts3tokTable { - sqlite3_vtab base; /* Base class used by SQLite core */ - const sqlite3_tokenizer_module *pMod; - sqlite3_tokenizer *pTok; -}; - -/* -** Virtual table cursor structure. -*/ -struct Fts3tokCursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - char *zInput; /* Input string */ - sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ - int iRowid; /* Current 'rowid' value */ - const char *zToken; /* Current 'token' value */ - int nToken; /* Size of zToken in bytes */ - int iStart; /* Current 'start' value */ - int iEnd; /* Current 'end' value */ - int iPos; /* Current 'pos' value */ -}; - -/* -** Query FTS for the tokenizer implementation named zName. -*/ -static int fts3tokQueryTokenizer( - Fts3Hash *pHash, - const char *zName, - const sqlite3_tokenizer_module **pp, - char **pzErr -){ - sqlite3_tokenizer_module *p; - int nName = (int)strlen(zName); - - p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); - if( !p ){ - sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName); - return SQLITE_ERROR; - } - - *pp = p; - return SQLITE_OK; -} - -/* -** The second argument, argv[], is an array of pointers to nul-terminated -** strings. This function makes a copy of the array and strings into a -** single block of memory. It then dequotes any of the strings that appear -** to be quoted. -** -** If successful, output parameter *pazDequote is set to point at the -** array of dequoted strings and SQLITE_OK is returned. The caller is -** responsible for eventually calling sqlite3_free() to free the array -** in this case. Or, if an error occurs, an SQLite error code is returned. -** The final value of *pazDequote is undefined in this case. -*/ -static int fts3tokDequoteArray( - int argc, /* Number of elements in argv[] */ - const char * const *argv, /* Input array */ - char ***pazDequote /* Output array */ -){ - int rc = SQLITE_OK; /* Return code */ - if( argc==0 ){ - *pazDequote = 0; - }else{ - int i; - int nByte = 0; - char **azDequote; - - for(i=0; ixCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); - } - - if( rc==SQLITE_OK ){ - pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); - if( pTab==0 ){ - rc = SQLITE_NOMEM; - } - } - - if( rc==SQLITE_OK ){ - memset(pTab, 0, sizeof(Fts3tokTable)); - pTab->pMod = pMod; - pTab->pTok = pTok; - *ppVtab = &pTab->base; - }else{ - if( pTok ){ - pMod->xDestroy(pTok); - } - } - - sqlite3_free(azDequote); - return rc; -} - -/* -** This function does the work for both the xDisconnect and xDestroy methods. -** These tables have no persistent representation of their own, so xDisconnect -** and xDestroy are identical operations. -*/ -static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ - Fts3tokTable *pTab = (Fts3tokTable *)pVtab; - - pTab->pMod->xDestroy(pTab->pTok); - sqlite3_free(pTab); - return SQLITE_OK; -} - -/* -** xBestIndex - Analyze a WHERE and ORDER BY clause. -*/ -static int fts3tokBestIndexMethod( - sqlite3_vtab *pVTab, - sqlite3_index_info *pInfo -){ - int i; - UNUSED_PARAMETER(pVTab); - - for(i=0; inConstraint; i++){ - if( pInfo->aConstraint[i].usable - && pInfo->aConstraint[i].iColumn==0 - && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ - ){ - pInfo->idxNum = 1; - pInfo->aConstraintUsage[i].argvIndex = 1; - pInfo->aConstraintUsage[i].omit = 1; - pInfo->estimatedCost = 1; - return SQLITE_OK; - } - } - - pInfo->idxNum = 0; - assert( pInfo->estimatedCost>1000000.0 ); - - return SQLITE_OK; -} - -/* -** xOpen - Open a cursor. -*/ -static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - Fts3tokCursor *pCsr; - UNUSED_PARAMETER(pVTab); - - pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); - if( pCsr==0 ){ - return SQLITE_NOMEM; - } - memset(pCsr, 0, sizeof(Fts3tokCursor)); - - *ppCsr = (sqlite3_vtab_cursor *)pCsr; - return SQLITE_OK; -} - -/* -** Reset the tokenizer cursor passed as the only argument. As if it had -** just been returned by fts3tokOpenMethod(). -*/ -static void fts3tokResetCursor(Fts3tokCursor *pCsr){ - if( pCsr->pCsr ){ - Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); - pTab->pMod->xClose(pCsr->pCsr); - pCsr->pCsr = 0; - } - sqlite3_free(pCsr->zInput); - pCsr->zInput = 0; - pCsr->zToken = 0; - pCsr->nToken = 0; - pCsr->iStart = 0; - pCsr->iEnd = 0; - pCsr->iPos = 0; - pCsr->iRowid = 0; -} - -/* -** xClose - Close a cursor. -*/ -static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; - - fts3tokResetCursor(pCsr); - sqlite3_free(pCsr); - return SQLITE_OK; -} - -/* -** xNext - Advance the cursor to the next row, if any. -*/ -static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; - Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); - int rc; /* Return code */ - - pCsr->iRowid++; - rc = pTab->pMod->xNext(pCsr->pCsr, - &pCsr->zToken, &pCsr->nToken, - &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos - ); - - if( rc!=SQLITE_OK ){ - fts3tokResetCursor(pCsr); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; - } - - return rc; -} - -/* -** xFilter - Initialize a cursor to point at the start of its data. -*/ -static int fts3tokFilterMethod( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, /* Strategy index */ - const char *idxStr, /* Unused */ - int nVal, /* Number of elements in apVal */ - sqlite3_value **apVal /* Arguments for the indexing scheme */ -){ - int rc = SQLITE_ERROR; - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; - Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); - UNUSED_PARAMETER(idxStr); - UNUSED_PARAMETER(nVal); - - fts3tokResetCursor(pCsr); - if( idxNum==1 ){ - const char *zByte = (const char *)sqlite3_value_text(apVal[0]); - int nByte = sqlite3_value_bytes(apVal[0]); - pCsr->zInput = sqlite3_malloc(nByte+1); - if( pCsr->zInput==0 ){ - rc = SQLITE_NOMEM; - }else{ - memcpy(pCsr->zInput, zByte, nByte); - pCsr->zInput[nByte] = 0; - rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); - if( rc==SQLITE_OK ){ - pCsr->pCsr->pTokenizer = pTab->pTok; - } - } - } - - if( rc!=SQLITE_OK ) return rc; - return fts3tokNextMethod(pCursor); -} - -/* -** xEof - Return true if the cursor is at EOF, or false otherwise. -*/ -static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; - return (pCsr->zToken==0); -} - -/* -** xColumn - Return a column value. -*/ -static int fts3tokColumnMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ - int iCol /* Index of column to read value from */ -){ - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; - - /* CREATE TABLE x(input, token, start, end, position) */ - switch( iCol ){ - case 0: - sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); - break; - case 1: - sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); - break; - case 2: - sqlite3_result_int(pCtx, pCsr->iStart); - break; - case 3: - sqlite3_result_int(pCtx, pCsr->iEnd); - break; - default: - assert( iCol==4 ); - sqlite3_result_int(pCtx, pCsr->iPos); - break; - } - return SQLITE_OK; -} - -/* -** xRowid - Return the current rowid for the cursor. -*/ -static int fts3tokRowidMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite_int64 *pRowid /* OUT: Rowid value */ -){ - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; - *pRowid = (sqlite3_int64)pCsr->iRowid; - return SQLITE_OK; -} - -/* -** Register the fts3tok module with database connection db. Return SQLITE_OK -** if successful or an error code if sqlite3_create_module() fails. -*/ -SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ - static const sqlite3_module fts3tok_module = { - 0, /* iVersion */ - fts3tokConnectMethod, /* xCreate */ - fts3tokConnectMethod, /* xConnect */ - fts3tokBestIndexMethod, /* xBestIndex */ - fts3tokDisconnectMethod, /* xDisconnect */ - fts3tokDisconnectMethod, /* xDestroy */ - fts3tokOpenMethod, /* xOpen */ - fts3tokCloseMethod, /* xClose */ - fts3tokFilterMethod, /* xFilter */ - fts3tokNextMethod, /* xNext */ - fts3tokEofMethod, /* xEof */ - fts3tokColumnMethod, /* xColumn */ - fts3tokRowidMethod, /* xRowid */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindFunction */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0 /* xRollbackTo */ - }; - int rc; /* Return code */ - - rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); - return rc; -} - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_tokenize_vtab.c **********************************/ -/************** Begin file fts3_write.c **************************************/ -/* -** 2009 Oct 23 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file is part of the SQLite FTS3 extension module. Specifically, -** this file contains code to insert, update and delete rows from FTS3 -** tables. It also contains code to merge FTS3 b-tree segments. Some -** of the sub-routines used to merge segments are also used by the query -** code in fts3.c. -*/ - -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ -/* #include */ - - -#define FTS_MAX_APPENDABLE_HEIGHT 16 - -/* -** When full-text index nodes are loaded from disk, the buffer that they -** are loaded into has the following number of bytes of padding at the end -** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer -** of 920 bytes is allocated for it. -** -** This means that if we have a pointer into a buffer containing node data, -** it is always safe to read up to two varints from it without risking an -** overread, even if the node data is corrupted. -*/ -#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) - -/* -** Under certain circumstances, b-tree nodes (doclists) can be loaded into -** memory incrementally instead of all at once. This can be a big performance -** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() -** method before retrieving all query results (as may happen, for example, -** if a query has a LIMIT clause). -** -** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD -** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. -** The code is written so that the hard lower-limit for each of these values -** is 1. Clearly such small values would be inefficient, but can be useful -** for testing purposes. -** -** If this module is built with SQLITE_TEST defined, these constants may -** be overridden at runtime for testing purposes. File fts3_test.c contains -** a Tcl interface to read and write the values. -*/ -#ifdef SQLITE_TEST -int test_fts3_node_chunksize = (4*1024); -int test_fts3_node_chunk_threshold = (4*1024)*4; -# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize -# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold -#else -# define FTS3_NODE_CHUNKSIZE (4*1024) -# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) -#endif - -/* -** The two values that may be meaningfully bound to the :1 parameter in -** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. -*/ -#define FTS_STAT_DOCTOTAL 0 -#define FTS_STAT_INCRMERGEHINT 1 -#define FTS_STAT_AUTOINCRMERGE 2 - -/* -** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic -** and incremental merge operation that takes place. This is used for -** debugging FTS only, it should not usually be turned on in production -** systems. -*/ -#ifdef FTS3_LOG_MERGES -static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ - sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); -} -#else -#define fts3LogMerge(x, y) -#endif - - -typedef struct PendingList PendingList; -typedef struct SegmentNode SegmentNode; -typedef struct SegmentWriter SegmentWriter; - -/* -** An instance of the following data structure is used to build doclists -** incrementally. See function fts3PendingListAppend() for details. -*/ -struct PendingList { - int nData; - char *aData; - int nSpace; - sqlite3_int64 iLastDocid; - sqlite3_int64 iLastCol; - sqlite3_int64 iLastPos; -}; - - -/* -** Each cursor has a (possibly empty) linked list of the following objects. -*/ -struct Fts3DeferredToken { - Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ - int iCol; /* Column token must occur in */ - Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ - PendingList *pList; /* Doclist is assembled here */ -}; - -/* -** An instance of this structure is used to iterate through the terms on -** a contiguous set of segment b-tree leaf nodes. Although the details of -** this structure are only manipulated by code in this file, opaque handles -** of type Fts3SegReader* are also used by code in fts3.c to iterate through -** terms when querying the full-text index. See functions: -** -** sqlite3Fts3SegReaderNew() -** sqlite3Fts3SegReaderFree() -** sqlite3Fts3SegReaderIterate() -** -** Methods used to manipulate Fts3SegReader structures: -** -** fts3SegReaderNext() -** fts3SegReaderFirstDocid() -** fts3SegReaderNextDocid() -*/ -struct Fts3SegReader { - int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ - u8 bLookup; /* True for a lookup only */ - u8 rootOnly; /* True for a root-only reader */ - - sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ - sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ - sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ - sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ - - char *aNode; /* Pointer to node data (or NULL) */ - int nNode; /* Size of buffer at aNode (or 0) */ - int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ - sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ - - Fts3HashElem **ppNextElem; - - /* Variables set by fts3SegReaderNext(). These may be read directly - ** by the caller. They are valid from the time SegmentReaderNew() returns - ** until SegmentReaderNext() returns something other than SQLITE_OK - ** (i.e. SQLITE_DONE). - */ - int nTerm; /* Number of bytes in current term */ - char *zTerm; /* Pointer to current term */ - int nTermAlloc; /* Allocated size of zTerm buffer */ - char *aDoclist; /* Pointer to doclist of current entry */ - int nDoclist; /* Size of doclist in current entry */ - - /* The following variables are used by fts3SegReaderNextDocid() to iterate - ** through the current doclist (aDoclist/nDoclist). - */ - char *pOffsetList; - int nOffsetList; /* For descending pending seg-readers only */ - sqlite3_int64 iDocid; -}; - -#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) -#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0) - -/* -** An instance of this structure is used to create a segment b-tree in the -** database. The internal details of this type are only accessed by the -** following functions: -** -** fts3SegWriterAdd() -** fts3SegWriterFlush() -** fts3SegWriterFree() -*/ -struct SegmentWriter { - SegmentNode *pTree; /* Pointer to interior tree structure */ - sqlite3_int64 iFirst; /* First slot in %_segments written */ - sqlite3_int64 iFree; /* Next free slot in %_segments */ - char *zTerm; /* Pointer to previous term buffer */ - int nTerm; /* Number of bytes in zTerm */ - int nMalloc; /* Size of malloc'd buffer at zMalloc */ - char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ - int nSize; /* Size of allocation at aData */ - int nData; /* Bytes of data in aData */ - char *aData; /* Pointer to block from malloc() */ - i64 nLeafData; /* Number of bytes of leaf data written */ -}; - -/* -** Type SegmentNode is used by the following three functions to create -** the interior part of the segment b+-tree structures (everything except -** the leaf nodes). These functions and type are only ever used by code -** within the fts3SegWriterXXX() family of functions described above. -** -** fts3NodeAddTerm() -** fts3NodeWrite() -** fts3NodeFree() -** -** When a b+tree is written to the database (either as a result of a merge -** or the pending-terms table being flushed), leaves are written into the -** database file as soon as they are completely populated. The interior of -** the tree is assembled in memory and written out only once all leaves have -** been populated and stored. This is Ok, as the b+-tree fanout is usually -** very large, meaning that the interior of the tree consumes relatively -** little memory. -*/ -struct SegmentNode { - SegmentNode *pParent; /* Parent node (or NULL for root node) */ - SegmentNode *pRight; /* Pointer to right-sibling */ - SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ - int nEntry; /* Number of terms written to node so far */ - char *zTerm; /* Pointer to previous term buffer */ - int nTerm; /* Number of bytes in zTerm */ - int nMalloc; /* Size of malloc'd buffer at zMalloc */ - char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ - int nData; /* Bytes of valid data so far */ - char *aData; /* Node data */ -}; - -/* -** Valid values for the second argument to fts3SqlStmt(). -*/ -#define SQL_DELETE_CONTENT 0 -#define SQL_IS_EMPTY 1 -#define SQL_DELETE_ALL_CONTENT 2 -#define SQL_DELETE_ALL_SEGMENTS 3 -#define SQL_DELETE_ALL_SEGDIR 4 -#define SQL_DELETE_ALL_DOCSIZE 5 -#define SQL_DELETE_ALL_STAT 6 -#define SQL_SELECT_CONTENT_BY_ROWID 7 -#define SQL_NEXT_SEGMENT_INDEX 8 -#define SQL_INSERT_SEGMENTS 9 -#define SQL_NEXT_SEGMENTS_ID 10 -#define SQL_INSERT_SEGDIR 11 -#define SQL_SELECT_LEVEL 12 -#define SQL_SELECT_LEVEL_RANGE 13 -#define SQL_SELECT_LEVEL_COUNT 14 -#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 -#define SQL_DELETE_SEGDIR_LEVEL 16 -#define SQL_DELETE_SEGMENTS_RANGE 17 -#define SQL_CONTENT_INSERT 18 -#define SQL_DELETE_DOCSIZE 19 -#define SQL_REPLACE_DOCSIZE 20 -#define SQL_SELECT_DOCSIZE 21 -#define SQL_SELECT_STAT 22 -#define SQL_REPLACE_STAT 23 - -#define SQL_SELECT_ALL_PREFIX_LEVEL 24 -#define SQL_DELETE_ALL_TERMS_SEGDIR 25 -#define SQL_DELETE_SEGDIR_RANGE 26 -#define SQL_SELECT_ALL_LANGID 27 -#define SQL_FIND_MERGE_LEVEL 28 -#define SQL_MAX_LEAF_NODE_ESTIMATE 29 -#define SQL_DELETE_SEGDIR_ENTRY 30 -#define SQL_SHIFT_SEGDIR_ENTRY 31 -#define SQL_SELECT_SEGDIR 32 -#define SQL_CHOMP_SEGDIR 33 -#define SQL_SEGMENT_IS_APPENDABLE 34 -#define SQL_SELECT_INDEXES 35 -#define SQL_SELECT_MXLEVEL 36 - -#define SQL_SELECT_LEVEL_RANGE2 37 -#define SQL_UPDATE_LEVEL_IDX 38 -#define SQL_UPDATE_LEVEL 39 - -/* -** This function is used to obtain an SQLite prepared statement handle -** for the statement identified by the second argument. If successful, -** *pp is set to the requested statement handle and SQLITE_OK returned. -** Otherwise, an SQLite error code is returned and *pp is set to 0. -** -** If argument apVal is not NULL, then it must point to an array with -** at least as many entries as the requested statement has bound -** parameters. The values are bound to the statements parameters before -** returning. -*/ -static int fts3SqlStmt( - Fts3Table *p, /* Virtual table handle */ - int eStmt, /* One of the SQL_XXX constants above */ - sqlite3_stmt **pp, /* OUT: Statement handle */ - sqlite3_value **apVal /* Values to bind to statement */ -){ - const char *azSql[] = { -/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", -/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", -/* 2 */ "DELETE FROM %Q.'%q_content'", -/* 3 */ "DELETE FROM %Q.'%q_segments'", -/* 4 */ "DELETE FROM %Q.'%q_segdir'", -/* 5 */ "DELETE FROM %Q.'%q_docsize'", -/* 6 */ "DELETE FROM %Q.'%q_stat'", -/* 7 */ "SELECT %s WHERE rowid=?", -/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", -/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", -/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", -/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", - - /* Return segments in order from oldest to newest.*/ -/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", -/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" - "ORDER BY level DESC, idx ASC", - -/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", -/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", - -/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", -/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", -/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", -/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", -/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", -/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", -/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", -/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", -/* 24 */ "", -/* 25 */ "", - -/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", -/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", - -/* This statement is used to determine which level to read the input from -** when performing an incremental merge. It returns the absolute level number -** of the oldest level in the db that contains at least ? segments. Or, -** if no level in the FTS index contains more than ? segments, the statement -** returns zero rows. */ -/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " - " GROUP BY level HAVING cnt>=?" - " ORDER BY (level %% 1024) ASC LIMIT 1", - -/* Estimate the upper limit on the number of leaf nodes in a new segment -** created by merging the oldest :2 segments from absolute level :1. See -** function sqlite3Fts3Incrmerge() for details. */ -/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " - " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", - -/* SQL_DELETE_SEGDIR_ENTRY -** Delete the %_segdir entry on absolute level :1 with index :2. */ -/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", - -/* SQL_SHIFT_SEGDIR_ENTRY -** Modify the idx value for the segment with idx=:3 on absolute level :2 -** to :1. */ -/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", - -/* SQL_SELECT_SEGDIR -** Read a single entry from the %_segdir table. The entry from absolute -** level :1 with index value :2. */ -/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", - -/* SQL_CHOMP_SEGDIR -** Update the start_block (:1) and root (:2) fields of the %_segdir -** entry located on absolute level :3 with index :4. */ -/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" - "WHERE level = ? AND idx = ?", - -/* SQL_SEGMENT_IS_APPENDABLE -** Return a single row if the segment with end_block=? is appendable. Or -** no rows otherwise. */ -/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", - -/* SQL_SELECT_INDEXES -** Return the list of valid segment indexes for absolute level ? */ -/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", - -/* SQL_SELECT_MXLEVEL -** Return the largest relative level in the FTS index or indexes. */ -/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", - - /* Return segments in order from oldest to newest.*/ -/* 37 */ "SELECT level, idx, end_block " - "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " - "ORDER BY level DESC, idx ASC", - - /* Update statements used while promoting segments */ -/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " - "WHERE level=? AND idx=?", -/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" - - }; - int rc = SQLITE_OK; - sqlite3_stmt *pStmt; - - assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); - assert( eStmt=0 ); - - pStmt = p->aStmt[eStmt]; - if( !pStmt ){ - char *zSql; - if( eStmt==SQL_CONTENT_INSERT ){ - zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); - }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ - zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); - }else{ - zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); - } - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); - sqlite3_free(zSql); - assert( rc==SQLITE_OK || pStmt==0 ); - p->aStmt[eStmt] = pStmt; - } - } - if( apVal ){ - int i; - int nParam = sqlite3_bind_parameter_count(pStmt); - for(i=0; rc==SQLITE_OK && inPendingData==0 ){ - sqlite3_stmt *pStmt; - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_null(pStmt, 1); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - } - } - - return rc; -} - -/* -** FTS maintains a separate indexes for each language-id (a 32-bit integer). -** Within each language id, a separate index is maintained to store the -** document terms, and each configured prefix size (configured the FTS -** "prefix=" option). And each index consists of multiple levels ("relative -** levels"). -** -** All three of these values (the language id, the specific index and the -** level within the index) are encoded in 64-bit integer values stored -** in the %_segdir table on disk. This function is used to convert three -** separate component values into the single 64-bit integer value that -** can be used to query the %_segdir table. -** -** Specifically, each language-id/index combination is allocated 1024 -** 64-bit integer level values ("absolute levels"). The main terms index -** for language-id 0 is allocate values 0-1023. The first prefix index -** (if any) for language-id 0 is allocated values 1024-2047. And so on. -** Language 1 indexes are allocated immediately following language 0. -** -** So, for a system with nPrefix prefix indexes configured, the block of -** absolute levels that corresponds to language-id iLangid and index -** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). -*/ -static sqlite3_int64 getAbsoluteLevel( - Fts3Table *p, /* FTS3 table handle */ - int iLangid, /* Language id */ - int iIndex, /* Index in p->aIndex[] */ - int iLevel /* Level of segments */ -){ - sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ - assert( iLangid>=0 ); - assert( p->nIndex>0 ); - assert( iIndex>=0 && iIndexnIndex ); - - iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; - return iBase + iLevel; -} - -/* -** Set *ppStmt to a statement handle that may be used to iterate through -** all rows in the %_segdir table, from oldest to newest. If successful, -** return SQLITE_OK. If an error occurs while preparing the statement, -** return an SQLite error code. -** -** There is only ever one instance of this SQL statement compiled for -** each FTS3 table. -** -** The statement returns the following columns from the %_segdir table: -** -** 0: idx -** 1: start_block -** 2: leaves_end_block -** 3: end_block -** 4: root -*/ -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( - Fts3Table *p, /* FTS3 table */ - int iLangid, /* Language being queried */ - int iIndex, /* Index for p->aIndex[] */ - int iLevel, /* Level to select (relative level) */ - sqlite3_stmt **ppStmt /* OUT: Compiled statement */ -){ - int rc; - sqlite3_stmt *pStmt = 0; - - assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); - assert( iLevel=0 && iIndexnIndex ); - - if( iLevel<0 ){ - /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int64(pStmt, 2, - getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) - ); - } - }else{ - /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); - } - } - *ppStmt = pStmt; - return rc; -} - - -/* -** Append a single varint to a PendingList buffer. SQLITE_OK is returned -** if successful, or an SQLite error code otherwise. -** -** This function also serves to allocate the PendingList structure itself. -** For example, to create a new PendingList structure containing two -** varints: -** -** PendingList *p = 0; -** fts3PendingListAppendVarint(&p, 1); -** fts3PendingListAppendVarint(&p, 2); -*/ -static int fts3PendingListAppendVarint( - PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ - sqlite3_int64 i /* Value to append to data */ -){ - PendingList *p = *pp; - - /* Allocate or grow the PendingList as required. */ - if( !p ){ - p = sqlite3_malloc(sizeof(*p) + 100); - if( !p ){ - return SQLITE_NOMEM; - } - p->nSpace = 100; - p->aData = (char *)&p[1]; - p->nData = 0; - } - else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ - int nNew = p->nSpace * 2; - p = sqlite3_realloc(p, sizeof(*p) + nNew); - if( !p ){ - sqlite3_free(*pp); - *pp = 0; - return SQLITE_NOMEM; - } - p->nSpace = nNew; - p->aData = (char *)&p[1]; - } - - /* Append the new serialized varint to the end of the list. */ - p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); - p->aData[p->nData] = '\0'; - *pp = p; - return SQLITE_OK; -} - -/* -** Add a docid/column/position entry to a PendingList structure. Non-zero -** is returned if the structure is sqlite3_realloced as part of adding -** the entry. Otherwise, zero. -** -** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. -** Zero is always returned in this case. Otherwise, if no OOM error occurs, -** it is set to SQLITE_OK. -*/ -static int fts3PendingListAppend( - PendingList **pp, /* IN/OUT: PendingList structure */ - sqlite3_int64 iDocid, /* Docid for entry to add */ - sqlite3_int64 iCol, /* Column for entry to add */ - sqlite3_int64 iPos, /* Position of term for entry to add */ - int *pRc /* OUT: Return code */ -){ - PendingList *p = *pp; - int rc = SQLITE_OK; - - assert( !p || p->iLastDocid<=iDocid ); - - if( !p || p->iLastDocid!=iDocid ){ - sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); - if( p ){ - assert( p->nDatanSpace ); - assert( p->aData[p->nData]==0 ); - p->nData++; - } - if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ - goto pendinglistappend_out; - } - p->iLastCol = -1; - p->iLastPos = 0; - p->iLastDocid = iDocid; - } - if( iCol>0 && p->iLastCol!=iCol ){ - if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) - || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) - ){ - goto pendinglistappend_out; - } - p->iLastCol = iCol; - p->iLastPos = 0; - } - if( iCol>=0 ){ - assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); - rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); - if( rc==SQLITE_OK ){ - p->iLastPos = iPos; - } - } - - pendinglistappend_out: - *pRc = rc; - if( p!=*pp ){ - *pp = p; - return 1; - } - return 0; -} - -/* -** Free a PendingList object allocated by fts3PendingListAppend(). -*/ -static void fts3PendingListDelete(PendingList *pList){ - sqlite3_free(pList); -} - -/* -** Add an entry to one of the pending-terms hash tables. -*/ -static int fts3PendingTermsAddOne( - Fts3Table *p, - int iCol, - int iPos, - Fts3Hash *pHash, /* Pending terms hash table to add entry to */ - const char *zToken, - int nToken -){ - PendingList *pList; - int rc = SQLITE_OK; - - pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); - if( pList ){ - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); - } - if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ - if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ - /* Malloc failed while inserting the new entry. This can only - ** happen if there was no previous entry for this token. - */ - assert( 0==fts3HashFind(pHash, zToken, nToken) ); - sqlite3_free(pList); - rc = SQLITE_NOMEM; - } - } - if( rc==SQLITE_OK ){ - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); - } - return rc; -} - -/* -** Tokenize the nul-terminated string zText and add all tokens to the -** pending-terms hash-table. The docid used is that currently stored in -** p->iPrevDocid, and the column is specified by argument iCol. -** -** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. -*/ -static int fts3PendingTermsAdd( - Fts3Table *p, /* Table into which text will be inserted */ - int iLangid, /* Language id to use */ - const char *zText, /* Text of document to be inserted */ - int iCol, /* Column into which text is being inserted */ - u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ -){ - int rc; - int iStart = 0; - int iEnd = 0; - int iPos = 0; - int nWord = 0; - - char const *zToken; - int nToken = 0; - - sqlite3_tokenizer *pTokenizer = p->pTokenizer; - sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; - sqlite3_tokenizer_cursor *pCsr; - int (*xNext)(sqlite3_tokenizer_cursor *pCursor, - const char**,int*,int*,int*,int*); - - assert( pTokenizer && pModule ); - - /* If the user has inserted a NULL value, this function may be called with - ** zText==0. In this case, add zero token entries to the hash table and - ** return early. */ - if( zText==0 ){ - *pnWord = 0; - return SQLITE_OK; - } - - rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); - if( rc!=SQLITE_OK ){ - return rc; - } - - xNext = pModule->xNext; - while( SQLITE_OK==rc - && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) - ){ - int i; - if( iPos>=nWord ) nWord = iPos+1; - - /* Positions cannot be negative; we use -1 as a terminator internally. - ** Tokens must have a non-zero length. - */ - if( iPos<0 || !zToken || nToken<=0 ){ - rc = SQLITE_ERROR; - break; - } - - /* Add the term to the terms index */ - rc = fts3PendingTermsAddOne( - p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken - ); - - /* Add the term to each of the prefix indexes that it is not too - ** short for. */ - for(i=1; rc==SQLITE_OK && inIndex; i++){ - struct Fts3Index *pIndex = &p->aIndex[i]; - if( nTokennPrefix ) continue; - rc = fts3PendingTermsAddOne( - p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix - ); - } - } - - pModule->xClose(pCsr); - *pnWord += nWord; - return (rc==SQLITE_DONE ? SQLITE_OK : rc); -} - -/* -** Calling this function indicates that subsequent calls to -** fts3PendingTermsAdd() are to add term/position-list pairs for the -** contents of the document with docid iDocid. -*/ -static int fts3PendingTermsDocid( - Fts3Table *p, /* Full-text table handle */ - int bDelete, /* True if this op is a delete */ - int iLangid, /* Language id of row being written */ - sqlite_int64 iDocid /* Docid of row being written */ -){ - assert( iLangid>=0 ); - assert( bDelete==1 || bDelete==0 ); - - /* TODO(shess) Explore whether partially flushing the buffer on - ** forced-flush would provide better performance. I suspect that if - ** we ordered the doclists by size and flushed the largest until the - ** buffer was half empty, that would let the less frequent terms - ** generate longer doclists. - */ - if( iDocidiPrevDocid - || (iDocid==p->iPrevDocid && p->bPrevDelete==0) - || p->iPrevLangid!=iLangid - || p->nPendingData>p->nMaxPendingData - ){ - int rc = sqlite3Fts3PendingTermsFlush(p); - if( rc!=SQLITE_OK ) return rc; - } - p->iPrevDocid = iDocid; - p->iPrevLangid = iLangid; - p->bPrevDelete = bDelete; - return SQLITE_OK; -} - -/* -** Discard the contents of the pending-terms hash tables. -*/ -SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ - int i; - for(i=0; inIndex; i++){ - Fts3HashElem *pElem; - Fts3Hash *pHash = &p->aIndex[i].hPending; - for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ - PendingList *pList = (PendingList *)fts3HashData(pElem); - fts3PendingListDelete(pList); - } - fts3HashClear(pHash); - } - p->nPendingData = 0; -} - -/* -** This function is called by the xUpdate() method as part of an INSERT -** operation. It adds entries for each term in the new record to the -** pendingTerms hash table. -** -** Argument apVal is the same as the similarly named argument passed to -** fts3InsertData(). Parameter iDocid is the docid of the new row. -*/ -static int fts3InsertTerms( - Fts3Table *p, - int iLangid, - sqlite3_value **apVal, - u32 *aSz -){ - int i; /* Iterator variable */ - for(i=2; inColumn+2; i++){ - int iCol = i-2; - if( p->abNotindexed[iCol]==0 ){ - const char *zText = (const char *)sqlite3_value_text(apVal[i]); - int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); - if( rc!=SQLITE_OK ){ - return rc; - } - aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); - } - } - return SQLITE_OK; -} - -/* -** This function is called by the xUpdate() method for an INSERT operation. -** The apVal parameter is passed a copy of the apVal argument passed by -** SQLite to the xUpdate() method. i.e: -** -** apVal[0] Not used for INSERT. -** apVal[1] rowid -** apVal[2] Left-most user-defined column -** ... -** apVal[p->nColumn+1] Right-most user-defined column -** apVal[p->nColumn+2] Hidden column with same name as table -** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) -** apVal[p->nColumn+4] Hidden languageid column -*/ -static int fts3InsertData( - Fts3Table *p, /* Full-text table */ - sqlite3_value **apVal, /* Array of values to insert */ - sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ -){ - int rc; /* Return code */ - sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ - - if( p->zContentTbl ){ - sqlite3_value *pRowid = apVal[p->nColumn+3]; - if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ - pRowid = apVal[1]; - } - if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ - return SQLITE_CONSTRAINT; - } - *piDocid = sqlite3_value_int64(pRowid); - return SQLITE_OK; - } - - /* Locate the statement handle used to insert data into the %_content - ** table. The SQL for this statement is: - ** - ** INSERT INTO %_content VALUES(?, ?, ?, ...) - ** - ** The statement features N '?' variables, where N is the number of user - ** defined columns in the FTS3 table, plus one for the docid field. - */ - rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); - if( rc==SQLITE_OK && p->zLanguageid ){ - rc = sqlite3_bind_int( - pContentInsert, p->nColumn+2, - sqlite3_value_int(apVal[p->nColumn+4]) - ); - } - if( rc!=SQLITE_OK ) return rc; - - /* There is a quirk here. The users INSERT statement may have specified - ** a value for the "rowid" field, for the "docid" field, or for both. - ** Which is a problem, since "rowid" and "docid" are aliases for the - ** same value. For example: - ** - ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); - ** - ** In FTS3, this is an error. It is an error to specify non-NULL values - ** for both docid and some other rowid alias. - */ - if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ - if( SQLITE_NULL==sqlite3_value_type(apVal[0]) - && SQLITE_NULL!=sqlite3_value_type(apVal[1]) - ){ - /* A rowid/docid conflict. */ - return SQLITE_ERROR; - } - rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); - if( rc!=SQLITE_OK ) return rc; - } - - /* Execute the statement to insert the record. Set *piDocid to the - ** new docid value. - */ - sqlite3_step(pContentInsert); - rc = sqlite3_reset(pContentInsert); - - *piDocid = sqlite3_last_insert_rowid(p->db); - return rc; -} - - - -/* -** Remove all data from the FTS3 table. Clear the hash table containing -** pending terms. -*/ -static int fts3DeleteAll(Fts3Table *p, int bContent){ - int rc = SQLITE_OK; /* Return code */ - - /* Discard the contents of the pending-terms hash table. */ - sqlite3Fts3PendingTermsClear(p); - - /* Delete everything from the shadow tables. Except, leave %_content as - ** is if bContent is false. */ - assert( p->zContentTbl==0 || bContent==0 ); - if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); - } - if( p->bHasStat ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); - } - return rc; -} - -/* -** -*/ -static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ - int iLangid = 0; - if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); - return iLangid; -} - -/* -** The first element in the apVal[] array is assumed to contain the docid -** (an integer) of a row about to be deleted. Remove all terms from the -** full-text index. -*/ -static void fts3DeleteTerms( - int *pRC, /* Result code */ - Fts3Table *p, /* The FTS table to delete from */ - sqlite3_value *pRowid, /* The docid to be deleted */ - u32 *aSz, /* Sizes of deleted document written here */ - int *pbFound /* OUT: Set to true if row really does exist */ -){ - int rc; - sqlite3_stmt *pSelect; - - assert( *pbFound==0 ); - if( *pRC ) return; - rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pSelect) ){ - int i; - int iLangid = langidFromSelect(p, pSelect); - i64 iDocid = sqlite3_column_int64(pSelect, 0); - rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); - for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ - int iCol = i-1; - if( p->abNotindexed[iCol]==0 ){ - const char *zText = (const char *)sqlite3_column_text(pSelect, i); - rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); - aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); - } - } - if( rc!=SQLITE_OK ){ - sqlite3_reset(pSelect); - *pRC = rc; - return; - } - *pbFound = 1; - } - rc = sqlite3_reset(pSelect); - }else{ - sqlite3_reset(pSelect); - } - *pRC = rc; -} - -/* -** Forward declaration to account for the circular dependency between -** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). -*/ -static int fts3SegmentMerge(Fts3Table *, int, int, int); - -/* -** This function allocates a new level iLevel index in the segdir table. -** Usually, indexes are allocated within a level sequentially starting -** with 0, so the allocated index is one greater than the value returned -** by: -** -** SELECT max(idx) FROM %_segdir WHERE level = :iLevel -** -** However, if there are already FTS3_MERGE_COUNT indexes at the requested -** level, they are merged into a single level (iLevel+1) segment and the -** allocated index is 0. -** -** If successful, *piIdx is set to the allocated index slot and SQLITE_OK -** returned. Otherwise, an SQLite error code is returned. -*/ -static int fts3AllocateSegdirIdx( - Fts3Table *p, - int iLangid, /* Language id */ - int iIndex, /* Index for p->aIndex */ - int iLevel, - int *piIdx -){ - int rc; /* Return Code */ - sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ - int iNext = 0; /* Result of query pNextIdx */ - - assert( iLangid>=0 ); - assert( p->nIndex>=1 ); - - /* Set variable iNext to the next available segdir index at level iLevel. */ - rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64( - pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) - ); - if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ - iNext = sqlite3_column_int(pNextIdx, 0); - } - rc = sqlite3_reset(pNextIdx); - } - - if( rc==SQLITE_OK ){ - /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already - ** full, merge all segments in level iLevel into a single iLevel+1 - ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, - ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. - */ - if( iNext>=FTS3_MERGE_COUNT ){ - fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); - rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); - *piIdx = 0; - }else{ - *piIdx = iNext; - } - } - - return rc; -} - -/* -** The %_segments table is declared as follows: -** -** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) -** -** This function reads data from a single row of the %_segments table. The -** specific row is identified by the iBlockid parameter. If paBlob is not -** NULL, then a buffer is allocated using sqlite3_malloc() and populated -** with the contents of the blob stored in the "block" column of the -** identified table row is. Whether or not paBlob is NULL, *pnBlob is set -** to the size of the blob in bytes before returning. -** -** If an error occurs, or the table does not contain the specified row, -** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If -** paBlob is non-NULL, then it is the responsibility of the caller to -** eventually free the returned buffer. -** -** This function may leave an open sqlite3_blob* handle in the -** Fts3Table.pSegments variable. This handle is reused by subsequent calls -** to this function. The handle may be closed by calling the -** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy -** performance improvement, but the blob handle should always be closed -** before control is returned to the user (to prevent a lock being held -** on the database file for longer than necessary). Thus, any virtual table -** method (xFilter etc.) that may directly or indirectly call this function -** must call sqlite3Fts3SegmentsClose() before returning. -*/ -SQLITE_PRIVATE int sqlite3Fts3ReadBlock( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ - char **paBlob, /* OUT: Blob data in malloc'd buffer */ - int *pnBlob, /* OUT: Size of blob data */ - int *pnLoad /* OUT: Bytes actually loaded */ -){ - int rc; /* Return code */ - - /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ - assert( pnBlob ); - - if( p->pSegments ){ - rc = sqlite3_blob_reopen(p->pSegments, iBlockid); - }else{ - if( 0==p->zSegmentsTbl ){ - p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); - if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; - } - rc = sqlite3_blob_open( - p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments - ); - } - - if( rc==SQLITE_OK ){ - int nByte = sqlite3_blob_bytes(p->pSegments); - *pnBlob = nByte; - if( paBlob ){ - char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); - if( !aByte ){ - rc = SQLITE_NOMEM; - }else{ - if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ - nByte = FTS3_NODE_CHUNKSIZE; - *pnLoad = nByte; - } - rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); - memset(&aByte[nByte], 0, FTS3_NODE_PADDING); - if( rc!=SQLITE_OK ){ - sqlite3_free(aByte); - aByte = 0; - } - } - *paBlob = aByte; - } - } - - return rc; -} - -/* -** Close the blob handle at p->pSegments, if it is open. See comments above -** the sqlite3Fts3ReadBlock() function for details. -*/ -SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ - sqlite3_blob_close(p->pSegments); - p->pSegments = 0; -} - -static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ - int nRead; /* Number of bytes to read */ - int rc; /* Return code */ - - nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); - rc = sqlite3_blob_read( - pReader->pBlob, - &pReader->aNode[pReader->nPopulate], - nRead, - pReader->nPopulate - ); - - if( rc==SQLITE_OK ){ - pReader->nPopulate += nRead; - memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); - if( pReader->nPopulate==pReader->nNode ){ - sqlite3_blob_close(pReader->pBlob); - pReader->pBlob = 0; - pReader->nPopulate = 0; - } - } - return rc; -} - -static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ - int rc = SQLITE_OK; - assert( !pReader->pBlob - || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) - ); - while( pReader->pBlob && rc==SQLITE_OK - && (pFrom - pReader->aNode + nByte)>pReader->nPopulate - ){ - rc = fts3SegReaderIncrRead(pReader); - } - return rc; -} - -/* -** Set an Fts3SegReader cursor to point at EOF. -*/ -static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ - if( !fts3SegReaderIsRootOnly(pSeg) ){ - sqlite3_free(pSeg->aNode); - sqlite3_blob_close(pSeg->pBlob); - pSeg->pBlob = 0; - } - pSeg->aNode = 0; -} - -/* -** Move the iterator passed as the first argument to the next term in the -** segment. If successful, SQLITE_OK is returned. If there is no next term, -** SQLITE_DONE. Otherwise, an SQLite error code. -*/ -static int fts3SegReaderNext( - Fts3Table *p, - Fts3SegReader *pReader, - int bIncr -){ - int rc; /* Return code of various sub-routines */ - char *pNext; /* Cursor variable */ - int nPrefix; /* Number of bytes in term prefix */ - int nSuffix; /* Number of bytes in term suffix */ - - if( !pReader->aDoclist ){ - pNext = pReader->aNode; - }else{ - pNext = &pReader->aDoclist[pReader->nDoclist]; - } - - if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ - - if( fts3SegReaderIsPending(pReader) ){ - Fts3HashElem *pElem = *(pReader->ppNextElem); - sqlite3_free(pReader->aNode); - pReader->aNode = 0; - if( pElem ){ - char *aCopy; - PendingList *pList = (PendingList *)fts3HashData(pElem); - int nCopy = pList->nData+1; - pReader->zTerm = (char *)fts3HashKey(pElem); - pReader->nTerm = fts3HashKeysize(pElem); - aCopy = (char*)sqlite3_malloc(nCopy); - if( !aCopy ) return SQLITE_NOMEM; - memcpy(aCopy, pList->aData, nCopy); - pReader->nNode = pReader->nDoclist = nCopy; - pReader->aNode = pReader->aDoclist = aCopy; - pReader->ppNextElem++; - assert( pReader->aNode ); - } - return SQLITE_OK; - } - - fts3SegReaderSetEof(pReader); - - /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf - ** blocks have already been traversed. */ - assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); - if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ - return SQLITE_OK; - } - - rc = sqlite3Fts3ReadBlock( - p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, - (bIncr ? &pReader->nPopulate : 0) - ); - if( rc!=SQLITE_OK ) return rc; - assert( pReader->pBlob==0 ); - if( bIncr && pReader->nPopulatenNode ){ - pReader->pBlob = p->pSegments; - p->pSegments = 0; - } - pNext = pReader->aNode; - } - - assert( !fts3SegReaderIsPending(pReader) ); - - rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); - if( rc!=SQLITE_OK ) return rc; - - /* Because of the FTS3_NODE_PADDING bytes of padding, the following is - ** safe (no risk of overread) even if the node data is corrupted. */ - pNext += fts3GetVarint32(pNext, &nPrefix); - pNext += fts3GetVarint32(pNext, &nSuffix); - if( nPrefix<0 || nSuffix<=0 - || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] - ){ - return FTS_CORRUPT_VTAB; - } - - if( nPrefix+nSuffix>pReader->nTermAlloc ){ - int nNew = (nPrefix+nSuffix)*2; - char *zNew = sqlite3_realloc(pReader->zTerm, nNew); - if( !zNew ){ - return SQLITE_NOMEM; - } - pReader->zTerm = zNew; - pReader->nTermAlloc = nNew; - } - - rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); - if( rc!=SQLITE_OK ) return rc; - - memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); - pReader->nTerm = nPrefix+nSuffix; - pNext += nSuffix; - pNext += fts3GetVarint32(pNext, &pReader->nDoclist); - pReader->aDoclist = pNext; - pReader->pOffsetList = 0; - - /* Check that the doclist does not appear to extend past the end of the - ** b-tree node. And that the final byte of the doclist is 0x00. If either - ** of these statements is untrue, then the data structure is corrupt. - */ - if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] - || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) - ){ - return FTS_CORRUPT_VTAB; - } - return SQLITE_OK; -} - -/* -** Set the SegReader to point to the first docid in the doclist associated -** with the current term. -*/ -static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ - int rc = SQLITE_OK; - assert( pReader->aDoclist ); - assert( !pReader->pOffsetList ); - if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ - u8 bEof = 0; - pReader->iDocid = 0; - pReader->nOffsetList = 0; - sqlite3Fts3DoclistPrev(0, - pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, - &pReader->iDocid, &pReader->nOffsetList, &bEof - ); - }else{ - rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); - if( rc==SQLITE_OK ){ - int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); - pReader->pOffsetList = &pReader->aDoclist[n]; - } - } - return rc; -} - -/* -** Advance the SegReader to point to the next docid in the doclist -** associated with the current term. -** -** If arguments ppOffsetList and pnOffsetList are not NULL, then -** *ppOffsetList is set to point to the first column-offset list -** in the doclist entry (i.e. immediately past the docid varint). -** *pnOffsetList is set to the length of the set of column-offset -** lists, not including the nul-terminator byte. For example: -*/ -static int fts3SegReaderNextDocid( - Fts3Table *pTab, - Fts3SegReader *pReader, /* Reader to advance to next docid */ - char **ppOffsetList, /* OUT: Pointer to current position-list */ - int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ -){ - int rc = SQLITE_OK; - char *p = pReader->pOffsetList; - char c = 0; - - assert( p ); - - if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ - /* A pending-terms seg-reader for an FTS4 table that uses order=desc. - ** Pending-terms doclists are always built up in ascending order, so - ** we have to iterate through them backwards here. */ - u8 bEof = 0; - if( ppOffsetList ){ - *ppOffsetList = pReader->pOffsetList; - *pnOffsetList = pReader->nOffsetList - 1; - } - sqlite3Fts3DoclistPrev(0, - pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, - &pReader->nOffsetList, &bEof - ); - if( bEof ){ - pReader->pOffsetList = 0; - }else{ - pReader->pOffsetList = p; - } - }else{ - char *pEnd = &pReader->aDoclist[pReader->nDoclist]; - - /* Pointer p currently points at the first byte of an offset list. The - ** following block advances it to point one byte past the end of - ** the same offset list. */ - while( 1 ){ - - /* The following line of code (and the "p++" below the while() loop) is - ** normally all that is required to move pointer p to the desired - ** position. The exception is if this node is being loaded from disk - ** incrementally and pointer "p" now points to the first byte past - ** the populated part of pReader->aNode[]. - */ - while( *p | c ) c = *p++ & 0x80; - assert( *p==0 ); - - if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; - rc = fts3SegReaderIncrRead(pReader); - if( rc!=SQLITE_OK ) return rc; - } - p++; - - /* If required, populate the output variables with a pointer to and the - ** size of the previous offset-list. - */ - if( ppOffsetList ){ - *ppOffsetList = pReader->pOffsetList; - *pnOffsetList = (int)(p - pReader->pOffsetList - 1); - } - - /* List may have been edited in place by fts3EvalNearTrim() */ - while( p=pEnd ){ - pReader->pOffsetList = 0; - }else{ - rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); - if( rc==SQLITE_OK ){ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); - if( pTab->bDescIdx ){ - pReader->iDocid -= iDelta; - }else{ - pReader->iDocid += iDelta; - } - } - } - } - - return SQLITE_OK; -} - - -SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( - Fts3Cursor *pCsr, - Fts3MultiSegReader *pMsr, - int *pnOvfl -){ - Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - int nOvfl = 0; - int ii; - int rc = SQLITE_OK; - int pgsz = p->nPgsz; - - assert( p->bFts4 ); - assert( pgsz>0 ); - - for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ - Fts3SegReader *pReader = pMsr->apSegment[ii]; - if( !fts3SegReaderIsPending(pReader) - && !fts3SegReaderIsRootOnly(pReader) - ){ - sqlite3_int64 jj; - for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ - int nBlob; - rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); - if( rc!=SQLITE_OK ) break; - if( (nBlob+35)>pgsz ){ - nOvfl += (nBlob + 34)/pgsz; - } - } - } - } - *pnOvfl = nOvfl; - return rc; -} - -/* -** Free all allocations associated with the iterator passed as the -** second argument. -*/ -SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ - if( pReader ){ - if( !fts3SegReaderIsPending(pReader) ){ - sqlite3_free(pReader->zTerm); - } - if( !fts3SegReaderIsRootOnly(pReader) ){ - sqlite3_free(pReader->aNode); - } - sqlite3_blob_close(pReader->pBlob); - } - sqlite3_free(pReader); -} - -/* -** Allocate a new SegReader object. -*/ -SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( - int iAge, /* Segment "age". */ - int bLookup, /* True for a lookup only */ - sqlite3_int64 iStartLeaf, /* First leaf to traverse */ - sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ - sqlite3_int64 iEndBlock, /* Final block of segment */ - const char *zRoot, /* Buffer containing root node */ - int nRoot, /* Size of buffer containing root node */ - Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ -){ - Fts3SegReader *pReader; /* Newly allocated SegReader object */ - int nExtra = 0; /* Bytes to allocate segment root node */ - - assert( iStartLeaf<=iEndLeaf ); - if( iStartLeaf==0 ){ - nExtra = nRoot + FTS3_NODE_PADDING; - } - - pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); - if( !pReader ){ - return SQLITE_NOMEM; - } - memset(pReader, 0, sizeof(Fts3SegReader)); - pReader->iIdx = iAge; - pReader->bLookup = bLookup!=0; - pReader->iStartBlock = iStartLeaf; - pReader->iLeafEndBlock = iEndLeaf; - pReader->iEndBlock = iEndBlock; - - if( nExtra ){ - /* The entire segment is stored in the root node. */ - pReader->aNode = (char *)&pReader[1]; - pReader->rootOnly = 1; - pReader->nNode = nRoot; - memcpy(pReader->aNode, zRoot, nRoot); - memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); - }else{ - pReader->iCurrentBlock = iStartLeaf-1; - } - *ppReader = pReader; - return SQLITE_OK; -} - -/* -** This is a comparison function used as a qsort() callback when sorting -** an array of pending terms by term. This occurs as part of flushing -** the contents of the pending-terms hash table to the database. -*/ -static int SQLITE_CDECL fts3CompareElemByTerm( - const void *lhs, - const void *rhs -){ - char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); - char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); - int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); - int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); - - int n = (n1aIndex */ - const char *zTerm, /* Term to search for */ - int nTerm, /* Size of buffer zTerm */ - int bPrefix, /* True for a prefix iterator */ - Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ -){ - Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ - Fts3HashElem *pE; /* Iterator variable */ - Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ - int nElem = 0; /* Size of array at aElem */ - int rc = SQLITE_OK; /* Return Code */ - Fts3Hash *pHash; - - pHash = &p->aIndex[iIndex].hPending; - if( bPrefix ){ - int nAlloc = 0; /* Size of allocated array at aElem */ - - for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ - char *zKey = (char *)fts3HashKey(pE); - int nKey = fts3HashKeysize(pE); - if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ - if( nElem==nAlloc ){ - Fts3HashElem **aElem2; - nAlloc += 16; - aElem2 = (Fts3HashElem **)sqlite3_realloc( - aElem, nAlloc*sizeof(Fts3HashElem *) - ); - if( !aElem2 ){ - rc = SQLITE_NOMEM; - nElem = 0; - break; - } - aElem = aElem2; - } - - aElem[nElem++] = pE; - } - } - - /* If more than one term matches the prefix, sort the Fts3HashElem - ** objects in term order using qsort(). This uses the same comparison - ** callback as is used when flushing terms to disk. - */ - if( nElem>1 ){ - qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); - } - - }else{ - /* The query is a simple term lookup that matches at most one term in - ** the index. All that is required is a straight hash-lookup. - ** - ** Because the stack address of pE may be accessed via the aElem pointer - ** below, the "Fts3HashElem *pE" must be declared so that it is valid - ** within this entire function, not just this "else{...}" block. - */ - pE = fts3HashFindElem(pHash, zTerm, nTerm); - if( pE ){ - aElem = &pE; - nElem = 1; - } - } - - if( nElem>0 ){ - int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); - pReader = (Fts3SegReader *)sqlite3_malloc(nByte); - if( !pReader ){ - rc = SQLITE_NOMEM; - }else{ - memset(pReader, 0, nByte); - pReader->iIdx = 0x7FFFFFFF; - pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; - memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); - } - } - - if( bPrefix ){ - sqlite3_free(aElem); - } - *ppReader = pReader; - return rc; -} - -/* -** Compare the entries pointed to by two Fts3SegReader structures. -** Comparison is as follows: -** -** 1) EOF is greater than not EOF. -** -** 2) The current terms (if any) are compared using memcmp(). If one -** term is a prefix of another, the longer term is considered the -** larger. -** -** 3) By segment age. An older segment is considered larger. -*/ -static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ - int rc; - if( pLhs->aNode && pRhs->aNode ){ - int rc2 = pLhs->nTerm - pRhs->nTerm; - if( rc2<0 ){ - rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); - }else{ - rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); - } - if( rc==0 ){ - rc = rc2; - } - }else{ - rc = (pLhs->aNode==0) - (pRhs->aNode==0); - } - if( rc==0 ){ - rc = pRhs->iIdx - pLhs->iIdx; - } - assert( rc!=0 ); - return rc; -} - -/* -** A different comparison function for SegReader structures. In this -** version, it is assumed that each SegReader points to an entry in -** a doclist for identical terms. Comparison is made as follows: -** -** 1) EOF (end of doclist in this case) is greater than not EOF. -** -** 2) By current docid. -** -** 3) By segment age. An older segment is considered larger. -*/ -static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ - int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); - if( rc==0 ){ - if( pLhs->iDocid==pRhs->iDocid ){ - rc = pRhs->iIdx - pLhs->iIdx; - }else{ - rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; - } - } - assert( pLhs->aNode && pRhs->aNode ); - return rc; -} -static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ - int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); - if( rc==0 ){ - if( pLhs->iDocid==pRhs->iDocid ){ - rc = pRhs->iIdx - pLhs->iIdx; - }else{ - rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; - } - } - assert( pLhs->aNode && pRhs->aNode ); - return rc; -} - -/* -** Compare the term that the Fts3SegReader object passed as the first argument -** points to with the term specified by arguments zTerm and nTerm. -** -** If the pSeg iterator is already at EOF, return 0. Otherwise, return -** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are -** equal, or +ve if the pSeg term is greater than zTerm/nTerm. -*/ -static int fts3SegReaderTermCmp( - Fts3SegReader *pSeg, /* Segment reader object */ - const char *zTerm, /* Term to compare to */ - int nTerm /* Size of term zTerm in bytes */ -){ - int res = 0; - if( pSeg->aNode ){ - if( pSeg->nTerm>nTerm ){ - res = memcmp(pSeg->zTerm, zTerm, nTerm); - }else{ - res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); - } - if( res==0 ){ - res = pSeg->nTerm-nTerm; - } - } - return res; -} - -/* -** Argument apSegment is an array of nSegment elements. It is known that -** the final (nSegment-nSuspect) members are already in sorted order -** (according to the comparison function provided). This function shuffles -** the array around until all entries are in sorted order. -*/ -static void fts3SegReaderSort( - Fts3SegReader **apSegment, /* Array to sort entries of */ - int nSegment, /* Size of apSegment array */ - int nSuspect, /* Unsorted entry count */ - int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ -){ - int i; /* Iterator variable */ - - assert( nSuspect<=nSegment ); - - if( nSuspect==nSegment ) nSuspect--; - for(i=nSuspect-1; i>=0; i--){ - int j; - for(j=i; j<(nSegment-1); j++){ - Fts3SegReader *pTmp; - if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; - pTmp = apSegment[j+1]; - apSegment[j+1] = apSegment[j]; - apSegment[j] = pTmp; - } - } - -#ifndef NDEBUG - /* Check that the list really is sorted now. */ - for(i=0; i<(nSuspect-1); i++){ - assert( xCmp(apSegment[i], apSegment[i+1])<0 ); - } -#endif -} - -/* -** Insert a record into the %_segments table. -*/ -static int fts3WriteSegment( - Fts3Table *p, /* Virtual table handle */ - sqlite3_int64 iBlock, /* Block id for new block */ - char *z, /* Pointer to buffer containing block data */ - int n /* Size of buffer z in bytes */ -){ - sqlite3_stmt *pStmt; - int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pStmt, 1, iBlock); - sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - } - return rc; -} - -/* -** Find the largest relative level number in the table. If successful, set -** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, -** set *pnMax to zero and return an SQLite error code. -*/ -SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ - int rc; - int mxLevel = 0; - sqlite3_stmt *pStmt = 0; - - rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - mxLevel = sqlite3_column_int(pStmt, 0); - } - rc = sqlite3_reset(pStmt); - } - *pnMax = mxLevel; - return rc; -} - -/* -** Insert a record into the %_segdir table. -*/ -static int fts3WriteSegdir( - Fts3Table *p, /* Virtual table handle */ - sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ - int iIdx, /* Value for "idx" field */ - sqlite3_int64 iStartBlock, /* Value for "start_block" field */ - sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ - sqlite3_int64 iEndBlock, /* Value for "end_block" field */ - sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ - char *zRoot, /* Blob value for "root" field */ - int nRoot /* Number of bytes in buffer zRoot */ -){ - sqlite3_stmt *pStmt; - int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pStmt, 1, iLevel); - sqlite3_bind_int(pStmt, 2, iIdx); - sqlite3_bind_int64(pStmt, 3, iStartBlock); - sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); - if( nLeafData==0 ){ - sqlite3_bind_int64(pStmt, 5, iEndBlock); - }else{ - char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); - if( !zEnd ) return SQLITE_NOMEM; - sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); - } - sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - } - return rc; -} - -/* -** Return the size of the common prefix (if any) shared by zPrev and -** zNext, in bytes. For example, -** -** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 -** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 -** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 -*/ -static int fts3PrefixCompress( - const char *zPrev, /* Buffer containing previous term */ - int nPrev, /* Size of buffer zPrev in bytes */ - const char *zNext, /* Buffer containing next term */ - int nNext /* Size of buffer zNext in bytes */ -){ - int n; - UNUSED_PARAMETER(nNext); - for(n=0; nnData; /* Current size of node in bytes */ - int nReq = nData; /* Required space after adding zTerm */ - int nPrefix; /* Number of bytes of prefix compression */ - int nSuffix; /* Suffix length */ - - nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); - nSuffix = nTerm-nPrefix; - - nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; - if( nReq<=p->nNodeSize || !pTree->zTerm ){ - - if( nReq>p->nNodeSize ){ - /* An unusual case: this is the first term to be added to the node - ** and the static node buffer (p->nNodeSize bytes) is not large - ** enough. Use a separately malloced buffer instead This wastes - ** p->nNodeSize bytes, but since this scenario only comes about when - ** the database contain two terms that share a prefix of almost 2KB, - ** this is not expected to be a serious problem. - */ - assert( pTree->aData==(char *)&pTree[1] ); - pTree->aData = (char *)sqlite3_malloc(nReq); - if( !pTree->aData ){ - return SQLITE_NOMEM; - } - } - - if( pTree->zTerm ){ - /* There is no prefix-length field for first term in a node */ - nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); - } - - nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); - memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); - pTree->nData = nData + nSuffix; - pTree->nEntry++; - - if( isCopyTerm ){ - if( pTree->nMalloczMalloc, nTerm*2); - if( !zNew ){ - return SQLITE_NOMEM; - } - pTree->nMalloc = nTerm*2; - pTree->zMalloc = zNew; - } - pTree->zTerm = pTree->zMalloc; - memcpy(pTree->zTerm, zTerm, nTerm); - pTree->nTerm = nTerm; - }else{ - pTree->zTerm = (char *)zTerm; - pTree->nTerm = nTerm; - } - return SQLITE_OK; - } - } - - /* If control flows to here, it was not possible to append zTerm to the - ** current node. Create a new node (a right-sibling of the current node). - ** If this is the first node in the tree, the term is added to it. - ** - ** Otherwise, the term is not added to the new node, it is left empty for - ** now. Instead, the term is inserted into the parent of pTree. If pTree - ** has no parent, one is created here. - */ - pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); - if( !pNew ){ - return SQLITE_NOMEM; - } - memset(pNew, 0, sizeof(SegmentNode)); - pNew->nData = 1 + FTS3_VARINT_MAX; - pNew->aData = (char *)&pNew[1]; - - if( pTree ){ - SegmentNode *pParent = pTree->pParent; - rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); - if( pTree->pParent==0 ){ - pTree->pParent = pParent; - } - pTree->pRight = pNew; - pNew->pLeftmost = pTree->pLeftmost; - pNew->pParent = pParent; - pNew->zMalloc = pTree->zMalloc; - pNew->nMalloc = pTree->nMalloc; - pTree->zMalloc = 0; - }else{ - pNew->pLeftmost = pNew; - rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); - } - - *ppTree = pNew; - return rc; -} - -/* -** Helper function for fts3NodeWrite(). -*/ -static int fts3TreeFinishNode( - SegmentNode *pTree, - int iHeight, - sqlite3_int64 iLeftChild -){ - int nStart; - assert( iHeight>=1 && iHeight<128 ); - nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); - pTree->aData[nStart] = (char)iHeight; - sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); - return nStart; -} - -/* -** Write the buffer for the segment node pTree and all of its peers to the -** database. Then call this function recursively to write the parent of -** pTree and its peers to the database. -** -** Except, if pTree is a root node, do not write it to the database. Instead, -** set output variables *paRoot and *pnRoot to contain the root node. -** -** If successful, SQLITE_OK is returned and output variable *piLast is -** set to the largest blockid written to the database (or zero if no -** blocks were written to the db). Otherwise, an SQLite error code is -** returned. -*/ -static int fts3NodeWrite( - Fts3Table *p, /* Virtual table handle */ - SegmentNode *pTree, /* SegmentNode handle */ - int iHeight, /* Height of this node in tree */ - sqlite3_int64 iLeaf, /* Block id of first leaf node */ - sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ - sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ - char **paRoot, /* OUT: Data for root node */ - int *pnRoot /* OUT: Size of root node in bytes */ -){ - int rc = SQLITE_OK; - - if( !pTree->pParent ){ - /* Root node of the tree. */ - int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); - *piLast = iFree-1; - *pnRoot = pTree->nData - nStart; - *paRoot = &pTree->aData[nStart]; - }else{ - SegmentNode *pIter; - sqlite3_int64 iNextFree = iFree; - sqlite3_int64 iNextLeaf = iLeaf; - for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ - int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); - int nWrite = pIter->nData - nStart; - - rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); - iNextFree++; - iNextLeaf += (pIter->nEntry+1); - } - if( rc==SQLITE_OK ){ - assert( iNextLeaf==iFree ); - rc = fts3NodeWrite( - p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot - ); - } - } - - return rc; -} - -/* -** Free all memory allocations associated with the tree pTree. -*/ -static void fts3NodeFree(SegmentNode *pTree){ - if( pTree ){ - SegmentNode *p = pTree->pLeftmost; - fts3NodeFree(p->pParent); - while( p ){ - SegmentNode *pRight = p->pRight; - if( p->aData!=(char *)&p[1] ){ - sqlite3_free(p->aData); - } - assert( pRight==0 || p->zMalloc==0 ); - sqlite3_free(p->zMalloc); - sqlite3_free(p); - p = pRight; - } - } -} - -/* -** Add a term to the segment being constructed by the SegmentWriter object -** *ppWriter. When adding the first term to a segment, *ppWriter should -** be passed NULL. This function will allocate a new SegmentWriter object -** and return it via the input/output variable *ppWriter in this case. -** -** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. -*/ -static int fts3SegWriterAdd( - Fts3Table *p, /* Virtual table handle */ - SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ - int isCopyTerm, /* True if buffer zTerm must be copied */ - const char *zTerm, /* Pointer to buffer containing term */ - int nTerm, /* Size of term in bytes */ - const char *aDoclist, /* Pointer to buffer containing doclist */ - int nDoclist /* Size of doclist in bytes */ -){ - int nPrefix; /* Size of term prefix in bytes */ - int nSuffix; /* Size of term suffix in bytes */ - int nReq; /* Number of bytes required on leaf page */ - int nData; - SegmentWriter *pWriter = *ppWriter; - - if( !pWriter ){ - int rc; - sqlite3_stmt *pStmt; - - /* Allocate the SegmentWriter structure */ - pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); - if( !pWriter ) return SQLITE_NOMEM; - memset(pWriter, 0, sizeof(SegmentWriter)); - *ppWriter = pWriter; - - /* Allocate a buffer in which to accumulate data */ - pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); - if( !pWriter->aData ) return SQLITE_NOMEM; - pWriter->nSize = p->nNodeSize; - - /* Find the next free blockid in the %_segments table */ - rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - pWriter->iFree = sqlite3_column_int64(pStmt, 0); - pWriter->iFirst = pWriter->iFree; - } - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; - } - nData = pWriter->nData; - - nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); - nSuffix = nTerm-nPrefix; - - /* Figure out how many bytes are required by this new entry */ - nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ - sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ - nSuffix + /* Term suffix */ - sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ - nDoclist; /* Doclist data */ - - if( nData>0 && nData+nReq>p->nNodeSize ){ - int rc; - - /* The current leaf node is full. Write it out to the database. */ - rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); - if( rc!=SQLITE_OK ) return rc; - p->nLeafAdd++; - - /* Add the current term to the interior node tree. The term added to - ** the interior tree must: - ** - ** a) be greater than the largest term on the leaf node just written - ** to the database (still available in pWriter->zTerm), and - ** - ** b) be less than or equal to the term about to be added to the new - ** leaf node (zTerm/nTerm). - ** - ** In other words, it must be the prefix of zTerm 1 byte longer than - ** the common prefix (if any) of zTerm and pWriter->zTerm. - */ - assert( nPrefixpTree, isCopyTerm, zTerm, nPrefix+1); - if( rc!=SQLITE_OK ) return rc; - - nData = 0; - pWriter->nTerm = 0; - - nPrefix = 0; - nSuffix = nTerm; - nReq = 1 + /* varint containing prefix size */ - sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ - nTerm + /* Term suffix */ - sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ - nDoclist; /* Doclist data */ - } - - /* Increase the total number of bytes written to account for the new entry. */ - pWriter->nLeafData += nReq; - - /* If the buffer currently allocated is too small for this entry, realloc - ** the buffer to make it large enough. - */ - if( nReq>pWriter->nSize ){ - char *aNew = sqlite3_realloc(pWriter->aData, nReq); - if( !aNew ) return SQLITE_NOMEM; - pWriter->aData = aNew; - pWriter->nSize = nReq; - } - assert( nData+nReq<=pWriter->nSize ); - - /* Append the prefix-compressed term and doclist to the buffer. */ - nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); - nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); - memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); - nData += nSuffix; - nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); - memcpy(&pWriter->aData[nData], aDoclist, nDoclist); - pWriter->nData = nData + nDoclist; - - /* Save the current term so that it can be used to prefix-compress the next. - ** If the isCopyTerm parameter is true, then the buffer pointed to by - ** zTerm is transient, so take a copy of the term data. Otherwise, just - ** store a copy of the pointer. - */ - if( isCopyTerm ){ - if( nTerm>pWriter->nMalloc ){ - char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); - if( !zNew ){ - return SQLITE_NOMEM; - } - pWriter->nMalloc = nTerm*2; - pWriter->zMalloc = zNew; - pWriter->zTerm = zNew; - } - assert( pWriter->zTerm==pWriter->zMalloc ); - memcpy(pWriter->zTerm, zTerm, nTerm); - }else{ - pWriter->zTerm = (char *)zTerm; - } - pWriter->nTerm = nTerm; - - return SQLITE_OK; -} - -/* -** Flush all data associated with the SegmentWriter object pWriter to the -** database. This function must be called after all terms have been added -** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is -** returned. Otherwise, an SQLite error code. -*/ -static int fts3SegWriterFlush( - Fts3Table *p, /* Virtual table handle */ - SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ - sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ - int iIdx /* Value for 'idx' column of %_segdir */ -){ - int rc; /* Return code */ - if( pWriter->pTree ){ - sqlite3_int64 iLast = 0; /* Largest block id written to database */ - sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ - char *zRoot = NULL; /* Pointer to buffer containing root node */ - int nRoot = 0; /* Size of buffer zRoot */ - - iLastLeaf = pWriter->iFree; - rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); - if( rc==SQLITE_OK ){ - rc = fts3NodeWrite(p, pWriter->pTree, 1, - pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); - } - if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir(p, iLevel, iIdx, - pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); - } - }else{ - /* The entire tree fits on the root node. Write it to the segdir table. */ - rc = fts3WriteSegdir(p, iLevel, iIdx, - 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); - } - p->nLeafAdd++; - return rc; -} - -/* -** Release all memory held by the SegmentWriter object passed as the -** first argument. -*/ -static void fts3SegWriterFree(SegmentWriter *pWriter){ - if( pWriter ){ - sqlite3_free(pWriter->aData); - sqlite3_free(pWriter->zMalloc); - fts3NodeFree(pWriter->pTree); - sqlite3_free(pWriter); - } -} - -/* -** The first value in the apVal[] array is assumed to contain an integer. -** This function tests if there exist any documents with docid values that -** are different from that integer. i.e. if deleting the document with docid -** pRowid would mean the FTS3 table were empty. -** -** If successful, *pisEmpty is set to true if the table is empty except for -** document pRowid, or false otherwise, and SQLITE_OK is returned. If an -** error occurs, an SQLite error code is returned. -*/ -static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ - sqlite3_stmt *pStmt; - int rc; - if( p->zContentTbl ){ - /* If using the content=xxx option, assume the table is never empty */ - *pisEmpty = 0; - rc = SQLITE_OK; - }else{ - rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pisEmpty = sqlite3_column_int(pStmt, 0); - } - rc = sqlite3_reset(pStmt); - } - } - return rc; -} - -/* -** Set *pnMax to the largest segment level in the database for the index -** iIndex. -** -** Segment levels are stored in the 'level' column of the %_segdir table. -** -** Return SQLITE_OK if successful, or an SQLite error code if not. -*/ -static int fts3SegmentMaxLevel( - Fts3Table *p, - int iLangid, - int iIndex, - sqlite3_int64 *pnMax -){ - sqlite3_stmt *pStmt; - int rc; - assert( iIndex>=0 && iIndexnIndex ); - - /* Set pStmt to the compiled version of: - ** - ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? - ** - ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). - */ - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int64(pStmt, 2, - getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) - ); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnMax = sqlite3_column_int64(pStmt, 0); - } - return sqlite3_reset(pStmt); -} - -/* -** iAbsLevel is an absolute level that may be assumed to exist within -** the database. This function checks if it is the largest level number -** within its index. Assuming no error occurs, *pbMax is set to 1 if -** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK -** is returned. If an error occurs, an error code is returned and the -** final value of *pbMax is undefined. -*/ -static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ - - /* Set pStmt to the compiled version of: - ** - ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? - ** - ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). - */ - sqlite3_stmt *pStmt; - int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); - sqlite3_bind_int64(pStmt, 2, - ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL - ); - - *pbMax = 0; - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; - } - return sqlite3_reset(pStmt); -} - -/* -** Delete all entries in the %_segments table associated with the segment -** opened with seg-reader pSeg. This function does not affect the contents -** of the %_segdir table. -*/ -static int fts3DeleteSegment( - Fts3Table *p, /* FTS table handle */ - Fts3SegReader *pSeg /* Segment to delete */ -){ - int rc = SQLITE_OK; /* Return code */ - if( pSeg->iStartBlock ){ - sqlite3_stmt *pDelete; /* SQL statement to delete rows */ - rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); - sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); - } - } - return rc; -} - -/* -** This function is used after merging multiple segments into a single large -** segment to delete the old, now redundant, segment b-trees. Specifically, -** it: -** -** 1) Deletes all %_segments entries for the segments associated with -** each of the SegReader objects in the array passed as the third -** argument, and -** -** 2) deletes all %_segdir entries with level iLevel, or all %_segdir -** entries regardless of level if (iLevel<0). -** -** SQLITE_OK is returned if successful, otherwise an SQLite error code. -*/ -static int fts3DeleteSegdir( - Fts3Table *p, /* Virtual table handle */ - int iLangid, /* Language id */ - int iIndex, /* Index for p->aIndex */ - int iLevel, /* Level of %_segdir entries to delete */ - Fts3SegReader **apSegment, /* Array of SegReader objects */ - int nReader /* Size of array apSegment */ -){ - int rc = SQLITE_OK; /* Return Code */ - int i; /* Iterator variable */ - sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ - - for(i=0; rc==SQLITE_OK && i=0 || iLevel==FTS3_SEGCURSOR_ALL ); - if( iLevel==FTS3_SEGCURSOR_ALL ){ - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int64(pDelete, 2, - getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) - ); - } - }else{ - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64( - pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) - ); - } - } - - if( rc==SQLITE_OK ){ - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); - } - - return rc; -} - -/* -** When this function is called, buffer *ppList (size *pnList bytes) contains -** a position list that may (or may not) feature multiple columns. This -** function adjusts the pointer *ppList and the length *pnList so that they -** identify the subset of the position list that corresponds to column iCol. -** -** If there are no entries in the input position list for column iCol, then -** *pnList is set to zero before returning. -** -** If parameter bZero is non-zero, then any part of the input list following -** the end of the output list is zeroed before returning. -*/ -static void fts3ColumnFilter( - int iCol, /* Column to filter on */ - int bZero, /* Zero out anything following *ppList */ - char **ppList, /* IN/OUT: Pointer to position list */ - int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ -){ - char *pList = *ppList; - int nList = *pnList; - char *pEnd = &pList[nList]; - int iCurrent = 0; - char *p = pList; - - assert( iCol>=0 ); - while( 1 ){ - char c = 0; - while( ppMsr->nBuffer ){ - char *pNew; - pMsr->nBuffer = nList*2; - pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); - if( !pNew ) return SQLITE_NOMEM; - pMsr->aBuffer = pNew; - } - - memcpy(pMsr->aBuffer, pList, nList); - return SQLITE_OK; -} - -SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ - sqlite3_int64 *piDocid, /* OUT: Docid value */ - char **paPoslist, /* OUT: Pointer to position list */ - int *pnPoslist /* OUT: Size of position list in bytes */ -){ - int nMerge = pMsr->nAdvance; - Fts3SegReader **apSegment = pMsr->apSegment; - int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( - p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp - ); - - if( nMerge==0 ){ - *paPoslist = 0; - return SQLITE_OK; - } - - while( 1 ){ - Fts3SegReader *pSeg; - pSeg = pMsr->apSegment[0]; - - if( pSeg->pOffsetList==0 ){ - *paPoslist = 0; - break; - }else{ - int rc; - char *pList; - int nList; - int j; - sqlite3_int64 iDocid = apSegment[0]->iDocid; - - rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); - j = 1; - while( rc==SQLITE_OK - && jpOffsetList - && apSegment[j]->iDocid==iDocid - ){ - rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); - j++; - } - if( rc!=SQLITE_OK ) return rc; - fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); - - if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pMsr, pList, nList+1); - if( rc!=SQLITE_OK ) return rc; - assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); - pList = pMsr->aBuffer; - } - - if( pMsr->iColFilter>=0 ){ - fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); - } - - if( nList>0 ){ - *paPoslist = pList; - *piDocid = iDocid; - *pnPoslist = nList; - break; - } - } - } - - return SQLITE_OK; -} - -static int fts3SegReaderStart( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pCsr, /* Cursor object */ - const char *zTerm, /* Term searched for (or NULL) */ - int nTerm /* Length of zTerm in bytes */ -){ - int i; - int nSeg = pCsr->nSegment; - - /* If the Fts3SegFilter defines a specific term (or term prefix) to search - ** for, then advance each segment iterator until it points to a term of - ** equal or greater value than the specified term. This prevents many - ** unnecessary merge/sort operations for the case where single segment - ** b-tree leaf nodes contain more than one term. - */ - for(i=0; pCsr->bRestart==0 && inSegment; i++){ - int res = 0; - Fts3SegReader *pSeg = pCsr->apSegment[i]; - do { - int rc = fts3SegReaderNext(p, pSeg, 0); - if( rc!=SQLITE_OK ) return rc; - }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 ); - - if( pSeg->bLookup && res!=0 ){ - fts3SegReaderSetEof(pSeg); - } - } - fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); - - return SQLITE_OK; -} - -SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pCsr, /* Cursor object */ - Fts3SegFilter *pFilter /* Restrictions on range of iteration */ -){ - pCsr->pFilter = pFilter; - return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); -} - -SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pCsr, /* Cursor object */ - int iCol, /* Column to match on. */ - const char *zTerm, /* Term to iterate through a doclist for */ - int nTerm /* Number of bytes in zTerm */ -){ - int i; - int rc; - int nSegment = pCsr->nSegment; - int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( - p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp - ); - - assert( pCsr->pFilter==0 ); - assert( zTerm && nTerm>0 ); - - /* Advance each segment iterator until it points to the term zTerm/nTerm. */ - rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); - if( rc!=SQLITE_OK ) return rc; - - /* Determine how many of the segments actually point to zTerm/nTerm. */ - for(i=0; iapSegment[i]; - if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ - break; - } - } - pCsr->nAdvance = i; - - /* Advance each of the segments to point to the first docid. */ - for(i=0; inAdvance; i++){ - rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); - if( rc!=SQLITE_OK ) return rc; - } - fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); - - assert( iCol<0 || iColnColumn ); - pCsr->iColFilter = iCol; - - return SQLITE_OK; -} - -/* -** This function is called on a MultiSegReader that has been started using -** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also -** have been made. Calling this function puts the MultiSegReader in such -** a state that if the next two calls are: -** -** sqlite3Fts3SegReaderStart() -** sqlite3Fts3SegReaderStep() -** -** then the entire doclist for the term is available in -** MultiSegReader.aDoclist/nDoclist. +** Generate code for the start of the iLevel-th loop in the WHERE clause +** implementation described by pWInfo. */ -SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ - int i; /* Used to iterate through segment-readers */ +static Bitmask codeOneLoopStart( + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + Bitmask notReady /* Which tables are currently available */ +){ + int j, k; /* Loop counters */ + int iCur; /* The VDBE cursor for the table */ + int addrNxt; /* Where to jump to continue with the next IN case */ + int omitTable; /* True if we use the index only */ + int bRev; /* True if we need to scan in reverse order */ + WhereLevel *pLevel; /* The where level to be coded */ + WhereLoop *pLoop; /* The WhereLoop object being coded */ + WhereClause *pWC; /* Decomposition of the entire WHERE clause */ + WhereTerm *pTerm; /* A WHERE clause term */ + Parse *pParse; /* Parsing context */ + sqlite3 *db; /* Database connection */ + Vdbe *v; /* The prepared stmt under constructions */ + struct SrcList_item *pTabItem; /* FROM clause term being coded */ + int addrBrk; /* Jump here to break out of the loop */ + int addrCont; /* Jump here to continue with next cycle */ + int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ + int iReleaseReg = 0; /* Temp register to free before returning */ - assert( pCsr->zTerm==0 ); - assert( pCsr->nTerm==0 ); - assert( pCsr->aDoclist==0 ); - assert( pCsr->nDoclist==0 ); + pParse = pWInfo->pParse; + v = pParse->pVdbe; + pWC = &pWInfo->sWC; + db = pParse->db; + pLevel = &pWInfo->a[iLevel]; + pLoop = pLevel->pWLoop; + pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + iCur = pTabItem->iCursor; + pLevel->notReady = notReady & ~getMask(&pWInfo->sMaskSet, iCur); + bRev = (pWInfo->revMask>>iLevel)&1; + omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 + && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; + VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); - pCsr->nAdvance = 0; - pCsr->bRestart = 1; - for(i=0; inSegment; i++){ - pCsr->apSegment[i]->pOffsetList = 0; - pCsr->apSegment[i]->nOffsetList = 0; - pCsr->apSegment[i]->iDocid = 0; + /* Create labels for the "break" and "continue" instructions + ** for the current loop. Jump to addrBrk to break out of a loop. + ** Jump to cont to go immediately to the next iteration of the + ** loop. + ** + ** When there is an IN operator, we also have a "addrNxt" label that + ** means to continue with the next IN value combination. When + ** there are no IN operators in the constraints, the "addrNxt" label + ** is the same as "addrBrk". + */ + addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); + + /* If this is the right table of a LEFT OUTER JOIN, allocate and + ** initialize a memory cell that records if this table matches any + ** row of the left table of the join. + */ + if( pLevel->iFrom>0 && (pTabItem[0].jointype & JT_LEFT)!=0 ){ + pLevel->iLeftJoin = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); + VdbeComment((v, "init LEFT JOIN no-match flag")); } - return SQLITE_OK; -} + /* Special case of a FROM clause subquery implemented as a co-routine */ + if( pTabItem->viaCoroutine ){ + int regYield = pTabItem->regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); + VdbeCoverage(v); + VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + pLevel->op = OP_Goto; + }else +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Case 1: The table is a virtual-table. Use the VFilter and VNext + ** to access the data. + */ + int iReg; /* P3 Value for OP_VFilter */ + int addrNotFound; + int nConstraint = pLoop->nLTerm; -SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pCsr /* Cursor object */ -){ - int rc = SQLITE_OK; + sqlite3ExprCachePush(pParse); + iReg = sqlite3GetTempRange(pParse, nConstraint+2); + addrNotFound = pLevel->addrBrk; + for(j=0; jaLTerm[j]; + if( pTerm==0 ) continue; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + } + } + sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); + sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, + pLoop->u.vtab.idxStr, + pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); + VdbeCoverage(v); + pLoop->u.vtab.needFree = 0; + for(j=0; ju.vtab.omitMask>>j)&1 ){ + disableTerm(pLevel, pLoop->aLTerm[j]); + } + } + pLevel->op = OP_VNext; + pLevel->p1 = iCur; + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); + sqlite3ExprCachePop(pParse); + }else +#endif /* SQLITE_OMIT_VIRTUALTABLE */ - int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); - int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); - int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); - int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); - int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); - int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); + if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 + ){ + /* Case 2: We can directly reference a single row using an + ** equality comparison against the ROWID field. Or + ** we reference multiple rows using a "rowid IN (...)" + ** construct. + */ + assert( pLoop->u.btree.nEq==1 ); + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->pExpr!=0 ); + assert( omitTable==0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + iReleaseReg = ++pParse->nMem; + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); + if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); + addrNxt = pLevel->addrNxt; + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + VdbeCoverage(v); + sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + VdbeComment((v, "pk")); + pLevel->op = OP_Noop; + }else if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 + ){ + /* Case 3: We have an inequality comparison against the ROWID field. + */ + int testOp = OP_Noop; + int start; + int memEndValue = 0; + WhereTerm *pStart, *pEnd; - Fts3SegReader **apSegment = pCsr->apSegment; - int nSegment = pCsr->nSegment; - Fts3SegFilter *pFilter = pCsr->pFilter; - int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( - p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp - ); + assert( omitTable==0 ); + j = 0; + pStart = pEnd = 0; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; + if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; + assert( pStart!=0 || pEnd!=0 ); + if( bRev ){ + pTerm = pStart; + pStart = pEnd; + pEnd = pTerm; + } + if( pStart ){ + Expr *pX; /* The expression that defines the start bound */ + int r1, rTemp; /* Registers for holding the start boundary */ - if( pCsr->nSegment==0 ) return SQLITE_OK; + /* The following constant maps TK_xx codes into corresponding + ** seek opcodes. It depends on a particular ordering of TK_xx + */ + const u8 aMoveOp[] = { + /* TK_GT */ OP_SeekGT, + /* TK_LE */ OP_SeekLE, + /* TK_LT */ OP_SeekLT, + /* TK_GE */ OP_SeekGE + }; + assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ + assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ + assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ - do { - int nMerge; - int i; - - /* Advance the first pCsr->nAdvance entries in the apSegment[] array - ** forward. Then sort the list in order of current term again. - */ - for(i=0; inAdvance; i++){ - Fts3SegReader *pSeg = apSegment[i]; - if( pSeg->bLookup ){ - fts3SegReaderSetEof(pSeg); + assert( (pStart->wtFlags & TERM_VNULL)==0 ); + testcase( pStart->wtFlags & TERM_VIRTUAL ); + pX = pStart->pExpr; + assert( pX!=0 ); + testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ + r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); + sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); + VdbeComment((v, "pk")); + VdbeCoverageIf(v, pX->op==TK_GT); + VdbeCoverageIf(v, pX->op==TK_LE); + VdbeCoverageIf(v, pX->op==TK_LT); + VdbeCoverageIf(v, pX->op==TK_GE); + sqlite3ExprCacheAffinityChange(pParse, r1, 1); + sqlite3ReleaseTempReg(pParse, rTemp); + disableTerm(pLevel, pStart); + }else{ + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + } + if( pEnd ){ + Expr *pX; + pX = pEnd->pExpr; + assert( pX!=0 ); + assert( (pEnd->wtFlags & TERM_VNULL)==0 ); + testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ + testcase( pEnd->wtFlags & TERM_VIRTUAL ); + memEndValue = ++pParse->nMem; + sqlite3ExprCode(pParse, pX->pRight, memEndValue); + if( pX->op==TK_LT || pX->op==TK_GT ){ + testOp = bRev ? OP_Le : OP_Ge; }else{ - rc = fts3SegReaderNext(p, pSeg, 0); + testOp = bRev ? OP_Lt : OP_Gt; } - if( rc!=SQLITE_OK ) return rc; + disableTerm(pLevel, pEnd); } - fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); - pCsr->nAdvance = 0; + start = sqlite3VdbeCurrentAddr(v); + pLevel->op = bRev ? OP_Prev : OP_Next; + pLevel->p1 = iCur; + pLevel->p2 = start; + assert( pLevel->p5==0 ); + if( testOp!=OP_Noop ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); + VdbeCoverageIf(v, testOp==OP_Le); + VdbeCoverageIf(v, testOp==OP_Lt); + VdbeCoverageIf(v, testOp==OP_Ge); + VdbeCoverageIf(v, testOp==OP_Gt); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); + } + }else if( pLoop->wsFlags & WHERE_INDEXED ){ + /* Case 4: A scan using an index. + ** + ** The WHERE clause may contain zero or more equality + ** terms ("==" or "IN" operators) that refer to the N + ** left-most columns of the index. It may also contain + ** inequality constraints (>, <, >= or <=) on the indexed + ** column that immediately follows the N equalities. Only + ** the right-most column can be an inequality - the rest must + ** use the "==" and "IN" operators. For example, if the + ** index is on (x,y,z), then the following clauses are all + ** optimized: + ** + ** x=5 + ** x=5 AND y=10 + ** x=5 AND y<10 + ** x=5 AND y>5 AND y<10 + ** x=5 AND y=5 AND z<=10 + ** + ** The z<10 term of the following cannot be used, only + ** the x=5 term: + ** + ** x=5 AND z<10 + ** + ** N may be zero if there are inequality constraints. + ** If there are no inequality constraints, then N is at + ** least one. + ** + ** This case is also used when there are no WHERE clause + ** constraints but an index is selected anyway, in order + ** to force the output order to conform to an ORDER BY. + */ + static const u8 aStartOp[] = { + 0, + 0, + OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ + OP_Last, /* 3: (!start_constraints && startEq && bRev) */ + OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ + OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ + OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ + OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ + }; + static const u8 aEndOp[] = { + OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ + OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ + OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ + OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ + }; + u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ + int regBase; /* Base register holding constraint values */ + WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ + WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ + int startEq; /* True if range start uses ==, >= or <= */ + int endEq; /* True if range end uses ==, >= or <= */ + int start_constraints; /* Start of range is constrained */ + int nConstraint; /* Number of constraint terms */ + Index *pIdx; /* The index we will be using */ + int iIdxCur; /* The VDBE cursor for the index */ + int nExtraReg = 0; /* Number of extra registers needed */ + int op; /* Instruction opcode */ + char *zStartAff; /* Affinity for start of range constraint */ + char cEndAff = 0; /* Affinity for end of range constraint */ + u8 bSeekPastNull = 0; /* True to seek past initial nulls */ + u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ - /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ - assert( rc==SQLITE_OK ); - if( apSegment[0]->aNode==0 ) break; + pIdx = pLoop->u.btree.pIndex; + iIdxCur = pLevel->iIdxCur; + assert( nEq>=pLoop->nSkip ); - pCsr->nTerm = apSegment[0]->nTerm; - pCsr->zTerm = apSegment[0]->zTerm; + /* If this loop satisfies a sort order (pOrderBy) request that + ** was passed to this function to implement a "SELECT min(x) ..." + ** query, then the caller will only allow the loop to run for + ** a single iteration. This means that the first row returned + ** should not have a NULL value stored in 'x'. If column 'x' is + ** the first one after the nEq equality constraints in the index, + ** this requires some special handling. + */ + assert( pWInfo->pOrderBy==0 + || pWInfo->pOrderBy->nExpr==1 + || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); + if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 + && pWInfo->nOBSat>0 + && (pIdx->nKeyCol>nEq) + ){ + assert( pLoop->nSkip==0 ); + bSeekPastNull = 1; + nExtraReg = 1; + } - /* If this is a prefix-search, and if the term that apSegment[0] points - ** to does not share a suffix with pFilter->zTerm/nTerm, then all - ** required callbacks have been made. In this case exit early. - ** - ** Similarly, if this is a search for an exact match, and the first term - ** of segment apSegment[0] is not a match, exit early. + /* Find any inequality constraint terms for the start and end + ** of the range. */ - if( pFilter->zTerm && !isScan ){ - if( pCsr->nTermnTerm - || (!isPrefix && pCsr->nTerm>pFilter->nTerm) - || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) + j = nEq; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ + pRangeStart = pLoop->aLTerm[j++]; + nExtraReg = 1; + /* Like optimization range constraints always occur in pairs */ + assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || + (pLoop->wsFlags & WHERE_TOP_LIMIT)!=0 ); + } + if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ + pRangeEnd = pLoop->aLTerm[j++]; + nExtraReg = 1; + if( (pRangeEnd->wtFlags & TERM_LIKEOPT)!=0 ){ + assert( pRangeStart!=0 ); /* LIKE opt constraints */ + assert( pRangeStart->wtFlags & TERM_LIKEOPT ); /* occur in pairs */ + pLevel->iLikeRepCntr = ++pParse->nMem; + testcase( bRev ); + testcase( pIdx->aSortOrder[nEq]==SQLITE_SO_DESC ); + sqlite3VdbeAddOp2(v, OP_Integer, + bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC), + pLevel->iLikeRepCntr); + VdbeComment((v, "LIKE loop counter")); + pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v); + } + if( pRangeStart==0 + && (j = pIdx->aiColumn[nEq])>=0 + && pIdx->pTable->aCol[j].notNull==0 ){ - break; + bSeekPastNull = 1; } } + assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); - nMerge = 1; - while( nMergeaNode - && apSegment[nMerge]->nTerm==pCsr->nTerm - && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) + /* Generate code to evaluate all constraint terms using == or IN + ** and store the values of those terms in an array of registers + ** starting at regBase. + */ + regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); + assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); + if( zStartAff ) cEndAff = zStartAff[nEq]; + addrNxt = pLevel->addrNxt; + + /* If we are doing a reverse order scan on an ascending index, or + ** a forward order scan on a descending index, interchange the + ** start and end terms (pRangeStart and pRangeEnd). + */ + if( (nEqnKeyCol && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) + || (bRev && pIdx->nKeyCol==nEq) ){ - nMerge++; + SWAP(WhereTerm *, pRangeEnd, pRangeStart); + SWAP(u8, bSeekPastNull, bStopAtNull); } - assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); - if( nMerge==1 - && !isIgnoreEmpty - && !isFirst - && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) - ){ - pCsr->nDoclist = apSegment[0]->nDoclist; - if( fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); - pCsr->aDoclist = pCsr->aBuffer; - }else{ - pCsr->aDoclist = apSegment[0]->aDoclist; - } - if( rc==SQLITE_OK ) rc = SQLITE_ROW; - }else{ - int nDoclist = 0; /* Size of doclist */ - sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ + testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); + startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); + endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); + start_constraints = pRangeStart || nEq>0; - /* The current term of the first nMerge entries in the array - ** of Fts3SegReader objects is the same. The doclists must be merged - ** and a single term returned with the merged doclist. - */ - for(i=0; ipExpr->pRight; + sqlite3ExprCode(pParse, pRight, regBase+nEq); + whereLikeOptimizationStringFixup(v, pLevel, pRangeStart); + if( (pRangeStart->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); } - fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp); - while( apSegment[0]->pOffsetList ){ - int j; /* Number of segments that share a docid */ - char *pList = 0; - int nList = 0; - int nByte; - sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); - j = 1; - while( jpOffsetList - && apSegment[j]->iDocid==iDocid - ){ - fts3SegReaderNextDocid(p, apSegment[j], 0, 0); - j++; - } - - if( isColFilter ){ - fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); + if( zStartAff ){ + if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_NONE){ + /* Since the comparison is to be performed with no conversions + ** applied to the operands, set the affinity to apply to pRight to + ** SQLITE_AFF_NONE. */ + zStartAff[nEq] = SQLITE_AFF_NONE; } - - if( !isIgnoreEmpty || nList>0 ){ - - /* Calculate the 'docid' delta value to write into the merged - ** doclist. */ - sqlite3_int64 iDelta; - if( p->bDescIdx && nDoclist>0 ){ - iDelta = iPrev - iDocid; - }else{ - iDelta = iDocid - iPrev; - } - assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); - assert( nDoclist>0 || iDelta==iDocid ); - - nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); - if( nDoclist+nByte>pCsr->nBuffer ){ - char *aNew; - pCsr->nBuffer = (nDoclist+nByte)*2; - aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); - if( !aNew ){ - return SQLITE_NOMEM; - } - pCsr->aBuffer = aNew; - } - - if( isFirst ){ - char *a = &pCsr->aBuffer[nDoclist]; - int nWrite; - - nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); - if( nWrite ){ - iPrev = iDocid; - nDoclist += nWrite; - } - }else{ - nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); - iPrev = iDocid; - if( isRequirePos ){ - memcpy(&pCsr->aBuffer[nDoclist], pList, nList); - nDoclist += nList; - pCsr->aBuffer[nDoclist++] = '\0'; - } - } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){ + zStartAff[nEq] = SQLITE_AFF_NONE; } + } + nConstraint++; + testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); + }else if( bSeekPastNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + nConstraint++; + startEq = 0; + start_constraints = 1; + } + codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; + assert( op!=0 ); + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); - fts3SegReaderSort(apSegment, nMerge, j, xCmp); + /* Load the value for the inequality constraint at the end of the + ** range (if any). + */ + nConstraint = nEq; + if( pRangeEnd ){ + Expr *pRight = pRangeEnd->pExpr->pRight; + sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); + sqlite3ExprCode(pParse, pRight, regBase+nEq); + whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); + if( (pRangeEnd->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); } - if( nDoclist>0 ){ - pCsr->aDoclist = pCsr->aBuffer; - pCsr->nDoclist = nDoclist; - rc = SQLITE_ROW; + if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_NONE + && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff) + ){ + codeApplyAffinity(pParse, regBase+nEq, 1, &cEndAff); } + nConstraint++; + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); + }else if( bStopAtNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; + nConstraint++; } - pCsr->nAdvance = nMerge; - }while( rc==SQLITE_OK ); - - return rc; -} + sqlite3DbFree(db, zStartAff); + /* Top of the loop body */ + pLevel->p2 = sqlite3VdbeCurrentAddr(v); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( - Fts3MultiSegReader *pCsr /* Cursor object */ -){ - if( pCsr ){ - int i; - for(i=0; inSegment; i++){ - sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); + /* Check if the index cursor is past the end of the range. */ + if( nConstraint ){ + op = aEndOp[bRev*2 + endEq]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); } - sqlite3_free(pCsr->apSegment); - sqlite3_free(pCsr->aBuffer); - - pCsr->nSegment = 0; - pCsr->apSegment = 0; - pCsr->aBuffer = 0; - } -} -/* -** Decode the "end_block" field, selected by column iCol of the SELECT -** statement passed as the first argument. -** -** The "end_block" field may contain either an integer, or a text field -** containing the text representation of two non-negative integers separated -** by one or more space (0x20) characters. In the first case, set *piEndBlock -** to the integer value and *pnByte to zero before returning. In the second, -** set *piEndBlock to the first value and *pnByte to the second. -*/ -static void fts3ReadEndBlockField( - sqlite3_stmt *pStmt, - int iCol, - i64 *piEndBlock, - i64 *pnByte -){ - const unsigned char *zText = sqlite3_column_text(pStmt, iCol); - if( zText ){ - int i; - int iMul = 1; - i64 iVal = 0; - for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ - iVal = iVal*10 + (zText[i] - '0'); + /* Seek the table cursor, if required */ + disableTerm(pLevel, pRangeStart); + disableTerm(pLevel, pRangeEnd); + if( omitTable ){ + /* pIdx is a covering index. No need to access the main table. */ + }else if( HasRowid(pIdx->pTable) ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ + }else if( iCur!=iIdxCur ){ + Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); + iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); + } + sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, + iRowidReg, pPk->nKeyCol); VdbeCoverage(v); } - *piEndBlock = iVal; - while( zText[i]==' ' ) i++; - iVal = 0; - if( zText[i]=='-' ){ - i++; - iMul = -1; + + /* Record the instruction used to terminate the loop. Disable + ** WHERE clause terms made redundant by the index range scan. + */ + if( pLoop->wsFlags & WHERE_ONEROW ){ + pLevel->op = OP_Noop; + }else if( bRev ){ + pLevel->op = OP_Prev; + }else{ + pLevel->op = OP_Next; } - for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ - iVal = iVal*10 + (zText[i] - '0'); + pLevel->p1 = iIdxCur; + pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; + if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + }else{ + assert( pLevel->p5==0 ); } - *pnByte = (iVal * (i64)iMul); - } -} - - -/* -** A segment of size nByte bytes has just been written to absolute level -** iAbsLevel. Promote any segments that should be promoted as a result. -*/ -static int fts3PromoteSegments( - Fts3Table *p, /* FTS table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level just updated */ - sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ -){ - int rc = SQLITE_OK; - sqlite3_stmt *pRange; + }else - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); +#ifndef SQLITE_OMIT_OR_OPTIMIZATION + if( pLoop->wsFlags & WHERE_MULTI_OR ){ + /* Case 5: Two or more separately indexed terms connected by OR + ** + ** Example: + ** + ** CREATE TABLE t1(a,b,c,d); + ** CREATE INDEX i1 ON t1(a); + ** CREATE INDEX i2 ON t1(b); + ** CREATE INDEX i3 ON t1(c); + ** + ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) + ** + ** In the example, there are three indexed terms connected by OR. + ** The top of the loop looks like this: + ** + ** Null 1 # Zero the rowset in reg 1 + ** + ** Then, for each indexed term, the following. The arguments to + ** RowSetTest are such that the rowid of the current row is inserted + ** into the RowSet. If it is already present, control skips the + ** Gosub opcode and jumps straight to the code generated by WhereEnd(). + ** + ** sqlite3WhereBegin() + ** RowSetTest # Insert rowid into rowset + ** Gosub 2 A + ** sqlite3WhereEnd() + ** + ** Following the above, code to terminate the loop. Label A, the target + ** of the Gosub above, jumps to the instruction right after the Goto. + ** + ** Null 1 # Zero the rowset in reg 1 + ** Goto B # The loop is finished. + ** + ** A: # Return data, whatever. + ** + ** Return 2 # Jump back to the Gosub + ** + ** B: + ** + ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then + ** use an ephemeral index instead of a RowSet to record the primary + ** keys of the rows we have already seen. + ** + */ + WhereClause *pOrWc; /* The OR-clause broken out into subterms */ + SrcList *pOrTab; /* Shortened table list or OR-clause generation */ + Index *pCov = 0; /* Potential covering index (or NULL) */ + int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ - if( rc==SQLITE_OK ){ - int bOk = 0; - i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; - i64 nLimit = (nByte*3)/2; + int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ + int regRowset = 0; /* Register for RowSet object */ + int regRowid = 0; /* Register holding rowid */ + int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ + int iRetInit; /* Address of regReturn init */ + int untestedTerms = 0; /* Some terms not completely tested */ + int ii; /* Loop counter */ + u16 wctrlFlags; /* Flags for sub-WHERE clause */ + Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ + Table *pTab = pTabItem->pTab; + + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->eOperator & WO_OR ); + assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); + pOrWc = &pTerm->u.pOrInfo->wc; + pLevel->op = OP_Return; + pLevel->p1 = regReturn; - /* Loop through all entries in the %_segdir table corresponding to - ** segments in this index on levels greater than iAbsLevel. If there is - ** at least one such segment, and it is possible to determine that all - ** such segments are smaller than nLimit bytes in size, they will be - ** promoted to level iAbsLevel. */ - sqlite3_bind_int64(pRange, 1, iAbsLevel+1); - sqlite3_bind_int64(pRange, 2, iLast); - while( SQLITE_ROW==sqlite3_step(pRange) ){ - i64 nSize = 0, dummy; - fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); - if( nSize<=0 || nSize>nLimit ){ - /* If nSize==0, then the %_segdir.end_block field does not not - ** contain a size value. This happens if it was written by an - ** old version of FTS. In this case it is not possible to determine - ** the size of the segment, and so segment promotion does not - ** take place. */ - bOk = 0; - break; + /* Set up a new SrcList in pOrTab containing the table being scanned + ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. + ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). + */ + if( pWInfo->nLevel>1 ){ + int nNotReady; /* The number of notReady tables */ + struct SrcList_item *origSrc; /* Original list of tables */ + nNotReady = pWInfo->nLevel - iLevel - 1; + pOrTab = sqlite3StackAllocRaw(db, + sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); + if( pOrTab==0 ) return notReady; + pOrTab->nAlloc = (u8)(nNotReady + 1); + pOrTab->nSrc = pOrTab->nAlloc; + memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); + origSrc = pWInfo->pTabList->a; + for(k=1; k<=nNotReady; k++){ + memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); } - bOk = 1; + }else{ + pOrTab = pWInfo->pTabList; } - rc = sqlite3_reset(pRange); - if( bOk ){ - int iIdx = 0; - sqlite3_stmt *pUpdate1 = 0; - sqlite3_stmt *pUpdate2 = 0; + /* Initialize the rowset register to contain NULL. An SQL NULL is + ** equivalent to an empty rowset. Or, create an ephemeral index + ** capable of holding primary keys in the case of a WITHOUT ROWID. + ** + ** Also initialize regReturn to contain the address of the instruction + ** immediately following the OP_Return at the bottom of the loop. This + ** is required in a few obscure LEFT JOIN cases where control jumps + ** over the top of the loop into the body of it. In this case the + ** correct response for the end-of-loop code (the OP_Return) is to + ** fall through to the next instruction, just as an OP_Next does if + ** called on an uninitialized cursor. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + if( HasRowid(pTab) ){ + regRowset = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + regRowset = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } + regRowid = ++pParse->nMem; + } + iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); - if( rc==SQLITE_OK ){ - rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y + ** Then for every term xN, evaluate as the subexpression: xN AND z + ** That way, terms in y that are factored into the disjunction will + ** be picked up by the recursive calls to sqlite3WhereBegin() below. + ** + ** Actually, each subexpression is converted to "xN AND w" where w is + ** the "interesting" terms of z - terms that did not originate in the + ** ON or USING clause of a LEFT JOIN, and terms that are usable as + ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 + */ + if( pWC->nTerm>1 ){ + int iTerm; + for(iTerm=0; iTermnTerm; iTerm++){ + Expr *pExpr = pWC->a[iTerm].pExpr; + if( &pWC->a[iTerm] == pTerm ) continue; + if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; + if( (pWC->a[iTerm].wtFlags & TERM_VIRTUAL)!=0 ) continue; + if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); + pExpr = sqlite3ExprDup(db, pExpr, 0); + pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); } - if( rc==SQLITE_OK ){ - rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + if( pAndExpr ){ + pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); } + } - if( rc==SQLITE_OK ){ - - /* Loop through all %_segdir entries for segments in this index with - ** levels equal to or greater than iAbsLevel. As each entry is visited, - ** updated it to set (level = -1) and (idx = N), where N is 0 for the - ** oldest segment in the range, 1 for the next oldest, and so on. - ** - ** In other words, move all segments being promoted to level -1, - ** setting the "idx" fields as appropriate to keep them in the same - ** order. The contents of level -1 (which is never used, except - ** transiently here), will be moved back to level iAbsLevel below. */ - sqlite3_bind_int64(pRange, 1, iAbsLevel); - while( SQLITE_ROW==sqlite3_step(pRange) ){ - sqlite3_bind_int(pUpdate1, 1, iIdx++); - sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); - sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); - sqlite3_step(pUpdate1); - rc = sqlite3_reset(pUpdate1); - if( rc!=SQLITE_OK ){ - sqlite3_reset(pRange); - break; - } + /* Run a separate WHERE clause for each term of the OR clause. After + ** eliminating duplicates from other WHERE clauses, the action for each + ** sub-WHERE clause is to to invoke the main loop body as a subroutine. + */ + wctrlFlags = WHERE_OMIT_OPEN_CLOSE + | WHERE_FORCE_TABLE + | WHERE_ONETABLE_ONLY + | WHERE_NO_AUTOINDEX; + for(ii=0; iinTerm; ii++){ + WhereTerm *pOrTerm = &pOrWc->a[ii]; + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ + WhereInfo *pSubWInfo; /* Info for single OR-term scan */ + Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ + int j1 = 0; /* Address of jump operation */ + if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ + pAndExpr->pLeft = pOrExpr; + pOrExpr = pAndExpr; } - } - if( rc==SQLITE_OK ){ - rc = sqlite3_reset(pRange); - } + /* Loop through table entries that match term pOrTerm. */ + WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, + wctrlFlags, iCovCur); + assert( pSubWInfo || pParse->nErr || db->mallocFailed ); + if( pSubWInfo ){ + WhereLoop *pSubLoop; + int addrExplain = explainOneScan( + pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 + ); + addScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain); - /* Move level -1 to level iAbsLevel */ - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); - sqlite3_step(pUpdate2); - rc = sqlite3_reset(pUpdate2); - } - } - } + /* This is the sub-WHERE clause body. First skip over + ** duplicate rows from prior sub-WHERE clauses, and record the + ** rowid (or PRIMARY KEY) for the current row so that the same + ** row will be skipped in subsequent sub-WHERE clauses. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + int r; + int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); + if( HasRowid(pTab) ){ + r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); + j1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, r,iSet); + VdbeCoverage(v); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + int nPk = pPk->nKeyCol; + int iPk; + /* Read the PK into an array of temp registers. */ + r = sqlite3GetTempRange(pParse, nPk); + for(iPk=0; iPkaiColumn[iPk]; + sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur, r+iPk, 0); + } - return rc; -} + /* Check if the temp table already contains this key. If so, + ** the row has already been included in the result set and + ** can be ignored (by jumping past the Gosub below). Otherwise, + ** insert the key into the temp table and proceed with processing + ** the row. + ** + ** Use some of the same optimizations as OP_RowSetTest: If iSet + ** is zero, assume that the key cannot already be present in + ** the temp table. And if iSet is -1, assume that there is no + ** need to insert the key into the temp table, as it will never + ** be tested for. */ + if( iSet ){ + j1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); + VdbeCoverage(v); + } + if( iSet>=0 ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); + sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); + if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } -/* -** Merge all level iLevel segments in the database into a single -** iLevel+1 segment. Or, if iLevel<0, merge all segments into a -** single segment with a level equal to the numerically largest level -** currently present in the database. -** -** If this function is called with iLevel<0, but there is only one -** segment in the database, SQLITE_DONE is returned immediately. -** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, -** an SQLite error code is returned. -*/ -static int fts3SegmentMerge( - Fts3Table *p, - int iLangid, /* Language id to merge */ - int iIndex, /* Index in p->aIndex[] to merge */ - int iLevel /* Level to merge */ -){ - int rc; /* Return code */ - int iIdx = 0; /* Index of new segment */ - sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ - SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ - Fts3SegFilter filter; /* Segment term filter condition */ - Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ - int bIgnoreEmpty = 0; /* True to ignore empty segments */ - i64 iMaxLevel = 0; /* Max level number for this index/langid */ + /* Release the array of temp registers */ + sqlite3ReleaseTempRange(pParse, r, nPk); + } + } - assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel>=0 - ); - assert( iLevel=0 && iIndexnIndex ); + /* Invoke the main loop body as a subroutine */ + sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); - rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); - if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + /* Jump here (skipping the main loop body subroutine) if the + ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ + if( j1 ) sqlite3VdbeJumpHere(v, j1); - if( iLevel!=FTS3_SEGCURSOR_PENDING ){ - rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); - if( rc!=SQLITE_OK ) goto finished; - } + /* The pSubWInfo->untestedTerms flag means that this OR term + ** contained one or more AND term from a notReady table. The + ** terms from the notReady table could not be tested and will + ** need to be tested later. + */ + if( pSubWInfo->untestedTerms ) untestedTerms = 1; - if( iLevel==FTS3_SEGCURSOR_ALL ){ - /* This call is to merge all segments in the database to a single - ** segment. The level of the new segment is equal to the numerically - ** greatest segment level currently present in the database for this - ** index. The idx of the new segment is always 0. */ - if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){ - rc = SQLITE_DONE; - goto finished; + /* If all of the OR-connected terms are optimized using the same + ** index, and the index is opened using the same cursor number + ** by each call to sqlite3WhereBegin() made by this loop, it may + ** be possible to use that index as a covering index. + ** + ** If the call to sqlite3WhereBegin() above resulted in a scan that + ** uses an index, and this is either the first OR-connected term + ** processed or the index is the same as that used by all previous + ** terms, set pCov to the candidate covering index. Otherwise, set + ** pCov to NULL to indicate that no candidate covering index will + ** be available. + */ + pSubLoop = pSubWInfo->a[0].pWLoop; + assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); + if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 + && (ii==0 || pSubLoop->u.btree.pIndex==pCov) + && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) + ){ + assert( pSubWInfo->a[0].iIdxCur==iCovCur ); + pCov = pSubLoop->u.btree.pIndex; + wctrlFlags |= WHERE_REOPEN_IDX; + }else{ + pCov = 0; + } + + /* Finish the loop through table entries that match term pOrTerm. */ + sqlite3WhereEnd(pSubWInfo); + } + } + } + pLevel->u.pCovidx = pCov; + if( pCov ) pLevel->iIdxCur = iCovCur; + if( pAndExpr ){ + pAndExpr->pLeft = 0; + sqlite3ExprDelete(db, pAndExpr); } - iNewLevel = iMaxLevel; - bIgnoreEmpty = 1; + sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); + sqlite3VdbeResolveLabel(v, iLoopBody); - }else{ - /* This call is to merge all segments at level iLevel. find the next - ** available segment index at level iLevel+1. The call to - ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to - ** a single iLevel+2 segment if necessary. */ - assert( FTS3_SEGCURSOR_PENDING==-1 ); - iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); - bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); - } - if( rc!=SQLITE_OK ) goto finished; + if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); + if( !untestedTerms ) disableTerm(pLevel, pTerm); + }else +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ - assert( csr.nSegment>0 ); - assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); - assert( iNewLevelisRecursive ){ + /* Tables marked isRecursive have only a single row that is stored in + ** a pseudo-cursor. No need to Rewind or Next such cursors. */ + pLevel->op = OP_Noop; + }else{ + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + } + } - memset(&filter, 0, sizeof(Fts3SegFilter)); - filter.flags = FTS3_SEGMENT_REQUIRE_POS; - filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pLevel->addrVisit = sqlite3VdbeCurrentAddr(v); +#endif - rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); - while( SQLITE_OK==rc ){ - rc = sqlite3Fts3SegReaderStep(p, &csr); - if( rc!=SQLITE_ROW ) break; - rc = fts3SegWriterAdd(p, &pWriter, 1, - csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); + /* Insert code to test every subexpression that can be completely + ** computed using the current set of tables. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + int skipLikeAddr = 0; + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + testcase( pWInfo->untestedTerms==0 + && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); + pWInfo->untestedTerms = 1; + continue; + } + pE = pTerm->pExpr; + assert( pE!=0 ); + if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ + continue; + } + if( pTerm->wtFlags & TERM_LIKECOND ){ + assert( pLevel->iLikeRepCntr>0 ); + skipLikeAddr = sqlite3VdbeAddOp1(v, OP_IfNot, pLevel->iLikeRepCntr); + VdbeCoverage(v); + } + sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); + if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); + pTerm->wtFlags |= TERM_CODED; } - if( rc!=SQLITE_OK ) goto finished; - assert( pWriter || bIgnoreEmpty ); - if( iLevel!=FTS3_SEGCURSOR_PENDING ){ - rc = fts3DeleteSegdir( - p, iLangid, iIndex, iLevel, csr.apSegment, csr.nSegment - ); - if( rc!=SQLITE_OK ) goto finished; + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE, *pEAlt; + WhereTerm *pAlt; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; + if( pTerm->leftCursor!=iCur ) continue; + if( pLevel->iLeftJoin ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); + pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); + if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; + testcase( pAlt->eOperator & WO_EQ ); + testcase( pAlt->eOperator & WO_IN ); + VdbeModuleComment((v, "begin transitive constraint")); + pEAlt = sqlite3StackAllocRaw(db, sizeof(*pEAlt)); + if( pEAlt ){ + *pEAlt = *pAlt->pExpr; + pEAlt->pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, pEAlt, addrCont, SQLITE_JUMPIFNULL); + sqlite3StackFree(db, pEAlt); + } } - if( pWriter ){ - rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); - if( rc==SQLITE_OK ){ - if( iLevel==FTS3_SEGCURSOR_PENDING || iNewLevelnLeafData); + + /* For a LEFT OUTER JOIN, generate code that will record the fact that + ** at least one row of the right table has matched the left table. + */ + if( pLevel->iLeftJoin ){ + pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); + VdbeComment((v, "record LEFT JOIN hit")); + sqlite3ExprCacheClear(pParse); + for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + assert( pWInfo->untestedTerms ); + continue; } + assert( pTerm->pExpr ); + sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); + pTerm->wtFlags |= TERM_CODED; } } - finished: - fts3SegWriterFree(pWriter); - sqlite3Fts3SegReaderFinish(&csr); - return rc; + return pLevel->notReady; } - -/* -** Flush the contents of pendingTerms to level 0 segments. +#ifdef WHERETRACE_ENABLED +/* +** Print the content of a WhereTerm object */ -SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ - int rc = SQLITE_OK; - int i; - - for(i=0; rc==SQLITE_OK && inIndex; i++){ - rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; +static void whereTermPrint(WhereTerm *pTerm, int iTerm){ + if( pTerm==0 ){ + sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); + }else{ + char zType[4]; + memcpy(zType, "...", 4); + if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; + if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + sqlite3DebugPrintf("TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x\n", + iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb, + pTerm->eOperator); + sqlite3TreeViewExpr(0, pTerm->pExpr, 0); } - sqlite3Fts3PendingTermsClear(p); +} +#endif - /* Determine the auto-incr-merge setting if unknown. If enabled, - ** estimate the number of leaf blocks of content to be written - */ - if( rc==SQLITE_OK && p->bHasStat - && p->nAutoincrmerge==0xff && p->nLeafAdd>0 - ){ - sqlite3_stmt *pStmt = 0; - rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); - rc = sqlite3_step(pStmt); - if( rc==SQLITE_ROW ){ - p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); - if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; - }else if( rc==SQLITE_DONE ){ - p->nAutoincrmerge = 0; +#ifdef WHERETRACE_ENABLED +/* +** Print a WhereLoop object for debugging purposes +*/ +static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ + WhereInfo *pWInfo = pWC->pWInfo; + int nb = 1+(pWInfo->pTabList->nSrc+7)/8; + struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, + p->iTab, nb, p->maskSelf, nb, p->prereq); + sqlite3DebugPrintf(" %12s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + const char *zName; + if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ + if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ + int i = sqlite3Strlen30(zName) - 1; + while( zName[i]!='_' ) i--; + zName += i; } - rc = sqlite3_reset(pStmt); + sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq); + }else{ + sqlite3DebugPrintf("%20s",""); + } + }else{ + char *z; + if( p->u.vtab.idxStr ){ + z = sqlite3_mprintf("(%d,\"%s\",%x)", + p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); + }else{ + z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); + } + sqlite3DebugPrintf(" %-19s", z); + sqlite3_free(z); + } + if( p->wsFlags & WHERE_SKIPSCAN ){ + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); + }else{ + sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); + } + sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); + if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + int i; + for(i=0; inLTerm; i++){ + whereTermPrint(p->aLTerm[i], i); } } - return rc; } +#endif /* -** Encode N integers as varints into a blob. +** Convert bulk memory into a valid WhereLoop that can be passed +** to whereLoopClear harmlessly. */ -static void fts3EncodeIntArray( - int N, /* The number of integers to encode */ - u32 *a, /* The integer values */ - char *zBuf, /* Write the BLOB here */ - int *pNBuf /* Write number of bytes if zBuf[] used here */ -){ - int i, j; - for(i=j=0; iaLTerm = p->aLTermSpace; + p->nLTerm = 0; + p->nLSlot = ArraySize(p->aLTermSpace); + p->wsFlags = 0; } /* -** Decode a blob of varints into N integers +** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. */ -static void fts3DecodeIntArray( - int N, /* The number of integers to decode */ - u32 *a, /* Write the integer values */ - const char *zBuf, /* The BLOB containing the varints */ - int nBuf /* size of the BLOB */ -){ - int i, j; - UNUSED_PARAMETER(nBuf); - for(i=j=0; iwsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){ + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){ + sqlite3_free(p->u.vtab.idxStr); + p->u.vtab.needFree = 0; + p->u.vtab.idxStr = 0; + }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ + sqlite3DbFree(db, p->u.btree.pIndex->zColAff); + sqlite3DbFree(db, p->u.btree.pIndex); + p->u.btree.pIndex = 0; + } } } /* -** Insert the sizes (in tokens) for each column of the document -** with docid equal to p->iPrevDocid. The sizes are encoded as -** a blob of varints. +** Deallocate internal memory used by a WhereLoop object */ -static void fts3InsertDocsize( - int *pRC, /* Result code */ - Fts3Table *p, /* Table into which to insert */ - u32 *aSz /* Sizes of each column, in tokens */ -){ - char *pBlob; /* The BLOB encoding of the document size */ - int nBlob; /* Number of bytes in the BLOB */ - sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ - int rc; /* Result code from subfunctions */ - - if( *pRC ) return; - pBlob = sqlite3_malloc( 10*p->nColumn ); - if( pBlob==0 ){ - *pRC = SQLITE_NOMEM; - return; - } - fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); - rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); - if( rc ){ - sqlite3_free(pBlob); - *pRC = rc; - return; - } - sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); - sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); - sqlite3_step(pStmt); - *pRC = sqlite3_reset(pStmt); +static void whereLoopClear(sqlite3 *db, WhereLoop *p){ + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + whereLoopClearUnion(db, p); + whereLoopInit(p); } /* -** Record 0 of the %_stat table contains a blob consisting of N varints, -** where N is the number of user defined columns in the fts3 table plus -** two. If nCol is the number of user defined columns, then values of the -** varints are set as follows: -** -** Varint 0: Total number of rows in the table. -** -** Varint 1..nCol: For each column, the total number of tokens stored in -** the column for all rows of the table. -** -** Varint 1+nCol: The total size, in bytes, of all text values in all -** columns of all rows of the table. -** +** Increase the memory allocation for pLoop->aLTerm[] to be at least n. */ -static void fts3UpdateDocTotals( - int *pRC, /* The result code */ - Fts3Table *p, /* Table being updated */ - u32 *aSzIns, /* Size increases */ - u32 *aSzDel, /* Size decreases */ - int nChng /* Change in the number of documents */ -){ - char *pBlob; /* Storage for BLOB written into %_stat */ - int nBlob; /* Size of BLOB written into %_stat */ - u32 *a; /* Array of integers that becomes the BLOB */ - sqlite3_stmt *pStmt; /* Statement for reading and writing */ - int i; /* Loop counter */ - int rc; /* Result code from subfunctions */ - - const int nStat = p->nColumn+2; +static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ + WhereTerm **paNew; + if( p->nLSlot>=n ) return SQLITE_OK; + n = (n+7)&~7; + paNew = sqlite3DbMallocRaw(db, sizeof(p->aLTerm[0])*n); + if( paNew==0 ) return SQLITE_NOMEM; + memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + p->aLTerm = paNew; + p->nLSlot = n; + return SQLITE_OK; +} - if( *pRC ) return; - a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); - if( a==0 ){ - *pRC = SQLITE_NOMEM; - return; - } - pBlob = (char*)&a[nStat]; - rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); - if( rc ){ - sqlite3_free(a); - *pRC = rc; - return; - } - sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - fts3DecodeIntArray(nStat, a, - sqlite3_column_blob(pStmt, 0), - sqlite3_column_bytes(pStmt, 0)); - }else{ - memset(a, 0, sizeof(u32)*(nStat) ); - } - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ){ - sqlite3_free(a); - *pRC = rc; - return; - } - if( nChng<0 && a[0]<(u32)(-nChng) ){ - a[0] = 0; - }else{ - a[0] += nChng; - } - for(i=0; inColumn+1; i++){ - u32 x = a[i+1]; - if( x+aSzIns[i] < aSzDel[i] ){ - x = 0; - }else{ - x = x + aSzIns[i] - aSzDel[i]; - } - a[i+1] = x; +/* +** Transfer content from the second pLoop into the first. +*/ +static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ + whereLoopClearUnion(db, pTo); + if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ + memset(&pTo->u, 0, sizeof(pTo->u)); + return SQLITE_NOMEM; } - fts3EncodeIntArray(nStat, a, pBlob, &nBlob); - rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); - if( rc ){ - sqlite3_free(a); - *pRC = rc; - return; + memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); + memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); + if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ + pFrom->u.vtab.needFree = 0; + }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){ + pFrom->u.btree.pIndex = 0; } - sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); - sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); - sqlite3_step(pStmt); - *pRC = sqlite3_reset(pStmt); - sqlite3_free(a); + return SQLITE_OK; } /* -** Merge the entire database so that there is one segment for each -** iIndex/iLangid combination. +** Delete a WhereLoop object */ -static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ - int bSeenDone = 0; - int rc; - sqlite3_stmt *pAllLangid = 0; - - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); - if( rc==SQLITE_OK ){ - int rc2; - sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); - sqlite3_bind_int(pAllLangid, 2, p->nIndex); - while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ - int i; - int iLangid = sqlite3_column_int(pAllLangid, 0); - for(i=0; rc==SQLITE_OK && inIndex; i++){ - rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_DONE ){ - bSeenDone = 1; - rc = SQLITE_OK; - } - } - } - rc2 = sqlite3_reset(pAllLangid); - if( rc==SQLITE_OK ) rc = rc2; - } - - sqlite3Fts3SegmentsClose(p); - sqlite3Fts3PendingTermsClear(p); - - return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ + whereLoopClear(db, p); + sqlite3DbFree(db, p); } /* -** This function is called when the user executes the following statement: -** -** INSERT INTO () VALUES('rebuild'); -** -** The entire FTS index is discarded and rebuilt. If the table is one -** created using the content=xxx option, then the new index is based on -** the current contents of the xxx table. Otherwise, it is rebuilt based -** on the contents of the %_content table. +** Free a WhereInfo structure */ -static int fts3DoRebuild(Fts3Table *p){ - int rc; /* Return Code */ - - rc = fts3DeleteAll(p, 0); - if( rc==SQLITE_OK ){ - u32 *aSz = 0; - u32 *aSzIns = 0; - u32 *aSzDel = 0; - sqlite3_stmt *pStmt = 0; - int nEntry = 0; - - /* Compose and prepare an SQL statement to loop through the content table */ - char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - } - - if( rc==SQLITE_OK ){ - int nByte = sizeof(u32) * (p->nColumn+1)*3; - aSz = (u32 *)sqlite3_malloc(nByte); - if( aSz==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(aSz, 0, nByte); - aSzIns = &aSz[p->nColumn+1]; - aSzDel = &aSzIns[p->nColumn+1]; - } - } - - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - int iCol; - int iLangid = langidFromSelect(p, pStmt); - rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); - memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); - for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ - if( p->abNotindexed[iCol]==0 ){ - const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); - rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); - aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); - } - } - if( p->bHasDocsize ){ - fts3InsertDocsize(&rc, p, aSz); - } - if( rc!=SQLITE_OK ){ - sqlite3_finalize(pStmt); - pStmt = 0; - }else{ - nEntry++; - for(iCol=0; iCol<=p->nColumn; iCol++){ - aSzIns[iCol] += aSz[iCol]; - } +static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ + if( ALWAYS(pWInfo) ){ + int i; + for(i=0; inLevel; i++){ + WhereLevel *pLevel = &pWInfo->a[i]; + if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){ + sqlite3DbFree(db, pLevel->u.in.aInLoop); } } - if( p->bFts4 ){ - fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); - } - sqlite3_free(aSz); - - if( pStmt ){ - int rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ){ - rc = rc2; - } + whereClauseClear(&pWInfo->sWC); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); } + sqlite3DbFree(db, pWInfo); } - - return rc; } - /* -** This function opens a cursor used to read the input data for an -** incremental merge operation. Specifically, it opens a cursor to scan -** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute -** level iAbsLevel. +** Return TRUE if all of the following are true: +** +** (1) X has the same or lower cost that Y +** (2) X is a proper subset of Y +** (3) X skips at least as many columns as Y +** +** By "proper subset" we mean that X uses fewer WHERE clause terms +** than Y and that every WHERE clause term used by X is also used +** by Y. +** +** If X is a proper subset of Y then Y is a better choice and ought +** to have a lower cost. This routine returns TRUE when that cost +** relationship is inverted and needs to be adjusted. The third rule +** was added because if X uses skip-scan less than Y it still might +** deserve a lower cost even if it is a proper subset of Y. */ -static int fts3IncrmergeCsr( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level to open */ - int nSeg, /* Number of segments to merge */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ +static int whereLoopCheaperProperSubset( + const WhereLoop *pX, /* First WhereLoop to compare */ + const WhereLoop *pY /* Compare against this WhereLoop */ ){ - int rc; /* Return Code */ - sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ - int nByte; /* Bytes allocated at pCsr->apSegment[] */ - - /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ - memset(pCsr, 0, sizeof(*pCsr)); - nByte = sizeof(Fts3SegReader *) * nSeg; - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); - - if( pCsr->apSegment==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(pCsr->apSegment, 0, nByte); - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + int i, j; + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* X is not a subset of Y */ } - if( rc==SQLITE_OK ){ - int i; - int rc2; - sqlite3_bind_int64(pStmt, 1, iAbsLevel); - assert( pCsr->nSegment==0 ); - for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] - ); - pCsr->nSegment++; + if( pY->nSkip > pX->nSkip ) return 0; + if( pX->rRun >= pY->rRun ){ + if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ + if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ + } + for(i=pX->nLTerm-1; i>=0; i--){ + if( pX->aLTerm[i]==0 ) continue; + for(j=pY->nLTerm-1; j>=0; j--){ + if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } - rc2 = sqlite3_reset(pStmt); - if( rc==SQLITE_OK ) rc = rc2; + if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ } - - return rc; + return 1; /* All conditions meet */ } -typedef struct IncrmergeWriter IncrmergeWriter; -typedef struct NodeWriter NodeWriter; -typedef struct Blob Blob; -typedef struct NodeReader NodeReader; - /* -** An instance of the following structure is used as a dynamic buffer -** to build up nodes or other blobs of data in. +** Try to adjust the cost of WhereLoop pTemplate upwards or downwards so +** that: ** -** The function blobGrowBuffer() is used to extend the allocation. -*/ -struct Blob { - char *a; /* Pointer to allocation */ - int n; /* Number of valid bytes of data in a[] */ - int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ -}; - -/* -** This structure is used to build up buffers containing segment b-tree -** nodes (blocks). -*/ -struct NodeWriter { - sqlite3_int64 iBlock; /* Current block id */ - Blob key; /* Last key written to the current block */ - Blob block; /* Current block image */ -}; - -/* -** An object of this type contains the state required to create or append -** to an appendable b-tree segment. -*/ -struct IncrmergeWriter { - int nLeafEst; /* Space allocated for leaf blocks */ - int nWork; /* Number of leaf pages flushed */ - sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ - int iIdx; /* Index of *output* segment in iAbsLevel+1 */ - sqlite3_int64 iStart; /* Block number of first allocated block */ - sqlite3_int64 iEnd; /* Block number of last allocated block */ - sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ - u8 bNoLeafData; /* If true, store 0 for segment size */ - NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; -}; - -/* -** An object of the following type is used to read data from a single -** FTS segment node. See the following functions: +** (1) pTemplate costs less than any other WhereLoops that are a proper +** subset of pTemplate ** -** nodeReaderInit() -** nodeReaderNext() -** nodeReaderRelease() -*/ -struct NodeReader { - const char *aNode; - int nNode; - int iOff; /* Current offset within aNode[] */ - - /* Output variables. Containing the current node entry. */ - sqlite3_int64 iChild; /* Pointer to child node */ - Blob term; /* Current term */ - const char *aDoclist; /* Pointer to doclist */ - int nDoclist; /* Size of doclist in bytes */ -}; - -/* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, if the allocation at pBlob->a is not already at least nMin -** bytes in size, extend (realloc) it to be so. +** (2) pTemplate costs more than any other WhereLoops for which pTemplate +** is a proper subset. ** -** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a -** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc -** to reflect the new size of the pBlob->a[] buffer. +** To say "WhereLoop X is a proper subset of Y" means that X uses fewer +** WHERE clause terms than Y and that every WHERE clause term used by X is +** also used by Y. */ -static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ - if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ - int nAlloc = nMin; - char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); - if( a ){ - pBlob->nAlloc = nAlloc; - pBlob->a = a; - }else{ - *pRc = SQLITE_NOMEM; +static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ + if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + for(; p; p=p->pNextLoop){ + if( p->iTab!=pTemplate->iTab ) continue; + if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( whereLoopCheaperProperSubset(p, pTemplate) ){ + /* Adjust pTemplate cost downward so that it is cheaper than its + ** subset p. */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut - 1; + }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ + /* Adjust pTemplate cost upward so that it is costlier than p since + ** pTemplate is a proper subset of p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut + 1; } } } /* -** Attempt to advance the node-reader object passed as the first argument to -** the next entry on the node. +** Search the list of WhereLoops in *ppPrev looking for one that can be +** supplanted by pTemplate. ** -** Return an error code if an error occurs (SQLITE_NOMEM is possible). -** Otherwise return SQLITE_OK. If there is no next entry on the node -** (e.g. because the current entry is the last) set NodeReader->aNode to -** NULL to indicate EOF. Otherwise, populate the NodeReader structure output -** variables for the new entry. +** Return NULL if the WhereLoop list contains an entry that can supplant +** pTemplate, in other words if pTemplate does not belong on the list. +** +** If pX is a WhereLoop that pTemplate can supplant, then return the +** link that points to pX. +** +** If pTemplate cannot supplant any existing element of the list but needs +** to be added to the list, then return a pointer to the tail of the list. */ -static int nodeReaderNext(NodeReader *p){ - int bFirst = (p->term.n==0); /* True for first term on the node */ - int nPrefix = 0; /* Bytes to copy from previous term */ - int nSuffix = 0; /* Bytes to append to the prefix */ - int rc = SQLITE_OK; /* Return code */ - - assert( p->aNode ); - if( p->iChild && bFirst==0 ) p->iChild++; - if( p->iOff>=p->nNode ){ - /* EOF */ - p->aNode = 0; - }else{ - if( bFirst==0 ){ - p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); +static WhereLoop **whereLoopFindLesser( + WhereLoop **ppPrev, + const WhereLoop *pTemplate +){ + WhereLoop *p; + for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ + if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ + /* If either the iTab or iSortIdx values for two WhereLoop are different + ** then those WhereLoops need to be considered separately. Neither is + ** a candidate to replace the other. */ + continue; } - p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + /* In the current implementation, the rSetup value is either zero + ** or the cost of building an automatic index (NlogN) and the NlogN + ** is the same for compatible WhereLoops. */ + assert( p->rSetup==0 || pTemplate->rSetup==0 + || p->rSetup==pTemplate->rSetup ); - blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); - if( rc==SQLITE_OK ){ - memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); - p->term.n = nPrefix+nSuffix; - p->iOff += nSuffix; - if( p->iChild==0 ){ - p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); - p->aDoclist = &p->aNode[p->iOff]; - p->iOff += p->nDoclist; - } - } - } + /* whereLoopAddBtree() always generates and inserts the automatic index + ** case first. Hence compatible candidate WhereLoops never have a larger + ** rSetup. Call this SETUP-INVARIANT */ + assert( p->rSetup>=pTemplate->rSetup ); - assert( p->iOff<=p->nNode ); + /* Any loop using an appliation-defined index (or PRIMARY KEY or + ** UNIQUE constraint) with one or more == constraints is better + ** than an automatic index. Unless it is a skip-scan. */ + if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 + && (pTemplate->nSkip)==0 + && (pTemplate->wsFlags & WHERE_INDEXED)!=0 + && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 + && (p->prereq & pTemplate->prereq)==pTemplate->prereq + ){ + break; + } - return rc; -} + /* If existing WhereLoop p is better than pTemplate, pTemplate can be + ** discarded. WhereLoop p is better if: + ** (1) p has no more dependencies than pTemplate, and + ** (2) p has an equal or lower cost than pTemplate + */ + if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */ + && p->rSetup<=pTemplate->rSetup /* (2a) */ + && p->rRun<=pTemplate->rRun /* (2b) */ + && p->nOut<=pTemplate->nOut /* (2c) */ + ){ + return 0; /* Discard pTemplate */ + } -/* -** Release all dynamic resources held by node-reader object *p. -*/ -static void nodeReaderRelease(NodeReader *p){ - sqlite3_free(p->term.a); + /* If pTemplate is always better than p, then cause p to be overwritten + ** with pTemplate. pTemplate is better than p if: + ** (1) pTemplate has no more dependences than p, and + ** (2) pTemplate has an equal or lower cost than p. + */ + if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ + && p->rRun>=pTemplate->rRun /* (2a) */ + && p->nOut>=pTemplate->nOut /* (2b) */ + ){ + assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ + break; /* Cause p to be overwritten by pTemplate */ + } + } + return ppPrev; } /* -** Initialize a node-reader object to read the node in buffer aNode/nNode. +** Insert or replace a WhereLoop entry using the template supplied. ** -** If successful, SQLITE_OK is returned and the NodeReader object set to -** point to the first entry on the node (if any). Otherwise, an SQLite -** error code is returned. +** An existing WhereLoop entry might be overwritten if the new template +** is better and has fewer dependencies. Or the template will be ignored +** and no insert will occur if an existing WhereLoop is faster and has +** fewer dependencies than the template. Otherwise a new WhereLoop is +** added based on the template. +** +** If pBuilder->pOrSet is not NULL then we care about only the +** prerequisites and rRun and nOut costs of the N best loops. That +** information is gathered in the pBuilder->pOrSet object. This special +** processing mode is used only for OR clause processing. +** +** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we +** still might overwrite similar loops with the new template if the +** new template is better. Loops may be overwritten if the following +** conditions are met: +** +** (1) They have the same iTab. +** (2) They have the same iSortIdx. +** (3) The template has same or fewer dependencies than the current loop +** (4) The template has the same or lower cost than the current loop */ -static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ - memset(p, 0, sizeof(NodeReader)); - p->aNode = aNode; - p->nNode = nNode; +static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ + WhereLoop **ppPrev, *p; + WhereInfo *pWInfo = pBuilder->pWInfo; + sqlite3 *db = pWInfo->pParse->db; - /* Figure out if this is a leaf or an internal node. */ - if( p->aNode[0] ){ - /* An internal node. */ - p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); + /* If pBuilder->pOrSet is defined, then only keep track of the costs + ** and prereqs. + */ + if( pBuilder->pOrSet!=0 ){ +#if WHERETRACE_ENABLED + u16 n = pBuilder->pOrSet->n; + int x = +#endif + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + pTemplate->nOut); +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + return SQLITE_OK; + } + + /* Look for an existing WhereLoop to replace with pTemplate + */ + whereLoopAdjustCost(pWInfo->pLoops, pTemplate); + ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); + + if( ppPrev==0 ){ + /* There already exists a WhereLoop on the list that is better + ** than pTemplate, so just ignore pTemplate */ +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(" skip: "); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + return SQLITE_OK; }else{ - p->iOff = 1; + p = *ppPrev; } - return nodeReaderNext(p); + /* If we reach this point it means that either p[] should be overwritten + ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new + ** WhereLoop and insert it. + */ +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + if( p!=0 ){ + sqlite3DebugPrintf("replace: "); + whereLoopPrint(p, pBuilder->pWC); + } + sqlite3DebugPrintf(" add: "); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + if( p==0 ){ + /* Allocate a new WhereLoop to add to the end of the list */ + *ppPrev = p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); + if( p==0 ) return SQLITE_NOMEM; + whereLoopInit(p); + p->pNextLoop = 0; + }else{ + /* We will be overwriting WhereLoop p[]. But before we do, first + ** go through the rest of the list and delete any other entries besides + ** p[] that are also supplated by pTemplate */ + WhereLoop **ppTail = &p->pNextLoop; + WhereLoop *pToDel; + while( *ppTail ){ + ppTail = whereLoopFindLesser(ppTail, pTemplate); + if( ppTail==0 ) break; + pToDel = *ppTail; + if( pToDel==0 ) break; + *ppTail = pToDel->pNextLoop; +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(" delete: "); + whereLoopPrint(pToDel, pBuilder->pWC); + } +#endif + whereLoopDelete(db, pToDel); + } + } + whereLoopXfer(db, p, pTemplate); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + Index *pIndex = p->u.btree.pIndex; + if( pIndex && pIndex->tnum==0 ){ + p->u.btree.pIndex = 0; + } + } + return SQLITE_OK; } /* -** This function is called while writing an FTS segment each time a leaf o -** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed -** to be greater than the largest key on the node just written, but smaller -** than or equal to the first key that will be written to the next leaf -** node. +** Adjust the WhereLoop.nOut value downward to account for terms of the +** WHERE clause that reference the loop but which are not used by an +** index. +* +** For every WHERE clause term that is not used by the index +** and which has a truth probability assigned by one of the likelihood(), +** likely(), or unlikely() SQL functions, reduce the estimated number +** of output rows by the probability specified. ** -** The block id of the leaf node just written to disk may be found in -** (pWriter->aNodeWriter[0].iBlock) when this function is called. +** TUNING: For every WHERE clause term that is not used by the index +** and which does not have an assigned truth probability, heuristics +** described below are used to try to estimate the truth probability. +** TODO --> Perhaps this is something that could be improved by better +** table statistics. +** +** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75% +** value corresponds to -1 in LogEst notation, so this means decrement +** the WhereLoop.nOut field for every such WHERE clause term. +** +** Heuristic 2: If there exists one or more WHERE clause terms of the +** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the +** final output row estimate is no greater than 1/4 of the total number +** of rows in the table. In other words, assume that x==EXPR will filter +** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the +** "x" column is boolean or else -1 or 0 or 1 is a common default value +** on the "x" column and so in that case only cap the output row estimate +** at 1/2 instead of 1/4. */ -static int fts3IncrmergePush( - Fts3Table *p, /* Fts3 table handle */ - IncrmergeWriter *pWriter, /* Writer object */ - const char *zTerm, /* Term to write to internal node */ - int nTerm /* Bytes at zTerm */ +static void whereLoopOutputAdjust( + WhereClause *pWC, /* The WHERE clause */ + WhereLoop *pLoop, /* The loop to adjust downward */ + LogEst nRow /* Number of rows in the entire table */ ){ - sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; - int iLayer; - - assert( nTerm>0 ); - for(iLayer=1; ALWAYS(iLayeraNodeWriter[iLayer]; - int rc = SQLITE_OK; - int nPrefix; - int nSuffix; - int nSpace; - - /* Figure out how much space the key will consume if it is written to - ** the current node of layer iLayer. Due to the prefix compression, - ** the space required changes depending on which node the key is to - ** be added to. */ - nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); - nSuffix = nTerm - nPrefix; - nSpace = sqlite3Fts3VarintLen(nPrefix); - nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; - - if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ - /* If the current node of layer iLayer contains zero keys, or if adding - ** the key to it will not cause it to grow to larger than nNodeSize - ** bytes in size, write the key here. */ - - Blob *pBlk = &pNode->block; - if( pBlk->n==0 ){ - blobGrowBuffer(pBlk, p->nNodeSize, &rc); - if( rc==SQLITE_OK ){ - pBlk->a[0] = (char)iLayer; - pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); - } - } - blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); - blobGrowBuffer(&pNode->key, nTerm, &rc); - - if( rc==SQLITE_OK ){ - if( pNode->key.n ){ - pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); - } - pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); - memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); - pBlk->n += nSuffix; + WhereTerm *pTerm, *pX; + Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); + int i, j, k; + LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ - memcpy(pNode->key.a, zTerm, nTerm); - pNode->key.n = nTerm; + assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); + for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ + if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; + if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; + if( (pTerm->prereqAll & notAllowed)!=0 ) continue; + for(j=pLoop->nLTerm-1; j>=0; j--){ + pX = pLoop->aLTerm[j]; + if( pX==0 ) continue; + if( pX==pTerm ) break; + if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; + } + if( j<0 ){ + if( pTerm->truthProb<=0 ){ + /* If a truth probability is specified using the likelihood() hints, + ** then use the probability provided by the application. */ + pLoop->nOut += pTerm->truthProb; + }else{ + /* In the absence of explicit truth probabilities, use heuristics to + ** guess a reasonable truth probability. */ + pLoop->nOut--; + if( pTerm->eOperator&WO_EQ ){ + Expr *pRight = pTerm->pExpr->pRight; + if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + k = 10; + }else{ + k = 20; + } + if( iReduceiBlock, pNode->block.a, pNode->block.n); - - assert( pNode->block.nAlloc>=p->nNodeSize ); - pNode->block.a[0] = (char)iLayer; - pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); - - iNextPtr = pNode->iBlock; - pNode->iBlock++; - pNode->key.n = 0; } - - if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; - iPtr = iNextPtr; } - - assert( 0 ); - return 0; + if( pLoop->nOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce; } /* -** Append a term and (optionally) doclist to the FTS segment node currently -** stored in blob *pNode. The node need not contain any terms, but the -** header must be written before this function is called. -** -** A node header is a single 0x00 byte for a leaf node, or a height varint -** followed by the left-hand-child varint for an internal node. -** -** The term to be appended is passed via arguments zTerm/nTerm. For a -** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal -** node, both aDoclist and nDoclist must be passed 0. -** -** If the size of the value in blob pPrev is zero, then this is the first -** term written to the node. Otherwise, pPrev contains a copy of the -** previous term. Before this function returns, it is updated to contain a -** copy of zTerm/nTerm. +** Adjust the cost C by the costMult facter T. This only occurs if +** compiled with -DSQLITE_ENABLE_COSTMULT +*/ +#ifdef SQLITE_ENABLE_COSTMULT +# define ApplyCostMultiplier(C,T) C += T +#else +# define ApplyCostMultiplier(C,T) +#endif + +/* +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the +** index pIndex. Try to match one more. ** -** It is assumed that the buffer associated with pNode is already large -** enough to accommodate the new entry. The buffer associated with pPrev -** is extended by this function if requrired. +** When this function is called, pBuilder->pNew->nOut contains the +** number of rows expected to be visited by filtering using the nEq +** terms only. If it is modified, this value is restored before this +** function returns. ** -** If an error (i.e. OOM condition) occurs, an SQLite error code is -** returned. Otherwise, SQLITE_OK. +** If pProbe->tnum==0, that means pIndex is a fake index used for the +** INTEGER PRIMARY KEY. */ -static int fts3AppendToNode( - Blob *pNode, /* Current node image to append to */ - Blob *pPrev, /* Buffer containing previous term written */ - const char *zTerm, /* New term to write */ - int nTerm, /* Size of zTerm in bytes */ - const char *aDoclist, /* Doclist (or NULL) to write */ - int nDoclist /* Size of aDoclist in bytes */ +static int whereLoopAddBtreeIndex( + WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ + struct SrcList_item *pSrc, /* FROM clause term being analyzed */ + Index *pProbe, /* An index on pSrc */ + LogEst nInMul /* log(Number of iterations due to IN) */ ){ + WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + sqlite3 *db = pParse->db; /* Database connection malloc context */ + WhereLoop *pNew; /* Template WhereLoop under construction */ + WhereTerm *pTerm; /* A WhereTerm under consideration */ + int opMask; /* Valid operators for constraints */ + WhereScan scan; /* Iterator for WHERE terms */ + Bitmask saved_prereq; /* Original value of pNew->prereq */ + u16 saved_nLTerm; /* Original value of pNew->nLTerm */ + u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ + u16 saved_nSkip; /* Original value of pNew->nSkip */ + u32 saved_wsFlags; /* Original value of pNew->wsFlags */ + LogEst saved_nOut; /* Original value of pNew->nOut */ + int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ - int bFirst = (pPrev->n==0); /* True if this is the first term written */ - int nPrefix; /* Size of term prefix in bytes */ - int nSuffix; /* Size of term suffix in bytes */ + LogEst rSize; /* Number of rows in the table */ + LogEst rLogSize; /* Logarithm of table size */ + WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ - /* Node must have already been started. There must be a doclist for a - ** leaf node, and there must not be a doclist for an internal node. */ - assert( pNode->n>0 ); - assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); + pNew = pBuilder->pNew; + if( db->mallocFailed ) return SQLITE_NOMEM; - blobGrowBuffer(pPrev, nTerm, &rc); - if( rc!=SQLITE_OK ) return rc; + assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); + assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); + if( pNew->wsFlags & WHERE_BTM_LIMIT ){ + opMask = WO_LT|WO_LE; + }else if( pProbe->tnum<=0 || (pSrc->jointype & JT_LEFT)!=0 ){ + opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE; + }else{ + opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; + } + if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); - nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); - nSuffix = nTerm - nPrefix; - memcpy(pPrev->a, zTerm, nTerm); - pPrev->n = nTerm; + assert( pNew->u.btree.nEqnColumn ); + iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - if( bFirst==0 ){ - pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); - } - pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); - memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); - pNode->n += nSuffix; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, + opMask, pProbe); + saved_nEq = pNew->u.btree.nEq; + saved_nSkip = pNew->nSkip; + saved_nLTerm = pNew->nLTerm; + saved_wsFlags = pNew->wsFlags; + saved_prereq = pNew->prereq; + saved_nOut = pNew->nOut; + pNew->rSetup = 0; + rSize = pProbe->aiRowLogEst[0]; + rLogSize = estLog(rSize); + for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ + LogEst rCostIdx; + LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ + int nIn = 0; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + int nRecValid = pBuilder->nRecValid; +#endif + if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) + && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) + ){ + continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ + } + if( pTerm->prereqRight & pNew->maskSelf ) continue; - if( aDoclist ){ - pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); - memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); - pNode->n += nDoclist; - } + /* Do not allow the upper bound of a LIKE optimization range constraint + ** to mix with a lower range bound from some other source */ + if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; - assert( pNode->n<=pNode->nAlloc ); + pNew->wsFlags = saved_wsFlags; + pNew->u.btree.nEq = saved_nEq; + pNew->nLTerm = saved_nLTerm; + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTerm; + pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; - return SQLITE_OK; -} + assert( nInMul==0 + || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 + || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 + || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 + ); -/* -** Append the current term and doclist pointed to by cursor pCsr to the -** appendable b-tree segment opened for writing by pWriter. -** -** Return SQLITE_OK if successful, or an SQLite error code otherwise. -*/ -static int fts3IncrmergeAppend( - Fts3Table *p, /* Fts3 table handle */ - IncrmergeWriter *pWriter, /* Writer object */ - Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ -){ - const char *zTerm = pCsr->zTerm; - int nTerm = pCsr->nTerm; - const char *aDoclist = pCsr->aDoclist; - int nDoclist = pCsr->nDoclist; - int rc = SQLITE_OK; /* Return code */ - int nSpace; /* Total space in bytes required on leaf */ - int nPrefix; /* Size of prefix shared with previous term */ - int nSuffix; /* Size of suffix (nTerm - nPrefix) */ - NodeWriter *pLeaf; /* Object used to write leaf nodes */ + if( eOp & WO_IN ){ + Expr *pExpr = pTerm->pExpr; + pNew->wsFlags |= WHERE_COLUMN_IN; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ + nIn = 46; assert( 46==sqlite3LogEst(25) ); + }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ + /* "x IN (value, value, ...)" */ + nIn = sqlite3LogEst(pExpr->x.pList->nExpr); + } + assert( nIn>0 ); /* RHS always has 2 or more terms... The parser + ** changes "x IN (?)" into "x=?". */ - pLeaf = &pWriter->aNodeWriter[0]; - nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); - nSuffix = nTerm - nPrefix; + }else if( eOp & (WO_EQ) ){ + pNew->wsFlags |= WHERE_COLUMN_EQ; + if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ + if( iCol>=0 && pProbe->uniqNotNull==0 ){ + pNew->wsFlags |= WHERE_UNQ_WANTED; + }else{ + pNew->wsFlags |= WHERE_ONEROW; + } + } + }else if( eOp & WO_ISNULL ){ + pNew->wsFlags |= WHERE_COLUMN_NULL; + }else if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pBtm = pTerm; + pTop = 0; + if( pTerm->wtFlags & TERM_LIKEOPT ){ + /* Range contraints that come from the LIKE optimization are + ** always used in pairs. */ + pTop = &pTerm[1]; + assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); + assert( pTop->wtFlags & TERM_LIKEOPT ); + assert( pTop->eOperator==WO_LT ); + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTop; + pNew->wsFlags |= WHERE_TOP_LIMIT; + } + }else{ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pTop = pTerm; + pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? + pNew->aLTerm[pNew->nLTerm-2] : 0; + } - nSpace = sqlite3Fts3VarintLen(nPrefix); - nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; - nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + /* At this point pNew->nOut is set to the number of rows expected to + ** be visited by the index scan before considering term pTerm, or the + ** values of nIn and nInMul. In other words, assuming that all + ** "x IN(...)" terms are replaced with "x = ?". This block updates + ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ + assert( pNew->nOut==saved_nOut ); + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + ** data, using some other estimate. */ + whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); + }else{ + int nEq = ++pNew->u.btree.nEq; + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); - /* If the current block is not empty, and if adding this term/doclist - ** to the current block would make it larger than Fts3Table.nNodeSize - ** bytes, write this block out to the database. */ - if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ - rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); - pWriter->nWork++; + assert( pNew->nOut==saved_nOut ); + if( pTerm->truthProb<=0 && iCol>=0 ){ + assert( (eOp & WO_IN) || nIn==0 ); + testcase( eOp & WO_IN ); + pNew->nOut += pTerm->truthProb; + pNew->nOut -= nIn; + }else{ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + tRowcnt nOut = 0; + if( nInMul==0 + && pProbe->nSample + && pNew->u.btree.nEq<=pProbe->nSampleCol + && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + ){ + Expr *pExpr = pTerm->pExpr; + if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + testcase( eOp & WO_EQ ); + testcase( eOp & WO_ISNULL ); + rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); + }else{ + rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); + } + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ + if( nOut ){ + pNew->nOut = sqlite3LogEst(nOut); + if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + pNew->nOut -= nIn; + } + } + if( nOut==0 ) +#endif + { + pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); + if( eOp & WO_ISNULL ){ + /* TUNING: If there is no likelihood() value, assume that a + ** "col IS NULL" expression matches twice as many rows + ** as (col=?). */ + pNew->nOut += 10; + } + } + } + } - /* Add the current term to the parent node. The term added to the - ** parent must: - ** - ** a) be greater than the largest term on the leaf node just written - ** to the database (still available in pLeaf->key), and - ** - ** b) be less than or equal to the term about to be added to the new - ** leaf node (zTerm/nTerm). - ** - ** In other words, it must be the prefix of zTerm 1 byte longer than - ** the common prefix (if any) of zTerm and pWriter->zTerm. - */ - if( rc==SQLITE_OK ){ - rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); + /* Set rCostIdx to the cost of visiting selected rows in index. Add + ** it to pNew->rRun, which is currently set to the cost of the index + ** seek only. Then, if this is a non-covering index, add the cost of + ** visiting the rows in the main table. */ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } + ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); - /* Advance to the next output block */ - pLeaf->iBlock++; - pLeaf->key.n = 0; - pLeaf->block.n = 0; + nOutUnadjusted = pNew->nOut; + pNew->rRun += nInMul + nIn; + pNew->nOut += nInMul + nIn; + whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); + rc = whereLoopInsert(pBuilder, pNew); - nSuffix = nTerm; - nSpace = 1; - nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; - nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; - } + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + pNew->nOut = saved_nOut; + }else{ + pNew->nOut = nOutUnadjusted; + } - pWriter->nLeafData += nSpace; - blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); - if( rc==SQLITE_OK ){ - if( pLeaf->block.n==0 ){ - pLeaf->block.n = 1; - pLeaf->block.a[0] = '\0'; + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 + && pNew->u.btree.nEqnColumn + ){ + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } - rc = fts3AppendToNode( - &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist - ); + pNew->nOut = saved_nOut; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + pBuilder->nRecValid = nRecValid; +#endif + } + pNew->prereq = saved_prereq; + pNew->u.btree.nEq = saved_nEq; + pNew->nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + pNew->nOut = saved_nOut; + pNew->nLTerm = saved_nLTerm; + + /* Consider using a skip-scan if there are no WHERE clause constraints + ** available for the left-most terms of the index, and if the average + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); + if( saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->noSkipScan==0 + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK + ){ + LogEst nIter; + pNew->u.btree.nEq++; + pNew->nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; + pNew->u.btree.nEq = saved_nEq; + pNew->nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; } return rc; } /* -** This function is called to release all dynamic resources held by the -** merge-writer object pWriter, and if no error has occurred, to flush -** all outstanding node buffers held by pWriter to disk. -** -** If *pRc is not SQLITE_OK when this function is called, then no attempt -** is made to write any data to disk. Instead, this function serves only -** to release outstanding resources. +** Return True if it is possible that pIndex might be useful in +** implementing the ORDER BY clause in pBuilder. ** -** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while -** flushing buffers to disk, *pRc is set to an SQLite error code before -** returning. +** Return False if pBuilder does not contain an ORDER BY clause or +** if there is no way for pIndex to be useful in implementing that +** ORDER BY clause. */ -static void fts3IncrmergeRelease( - Fts3Table *p, /* FTS3 table handle */ - IncrmergeWriter *pWriter, /* Merge-writer object */ - int *pRc /* IN/OUT: Error code */ +static int indexMightHelpWithOrderBy( + WhereLoopBuilder *pBuilder, + Index *pIndex, + int iCursor ){ - int i; /* Used to iterate through non-root layers */ - int iRoot; /* Index of root in pWriter->aNodeWriter */ - NodeWriter *pRoot; /* NodeWriter for root node */ - int rc = *pRc; /* Error code */ - - /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment - ** root node. If the segment fits entirely on a single leaf node, iRoot - ** will be set to 0. If the root node is the parent of the leaves, iRoot - ** will be 1. And so on. */ - for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ - NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; - if( pNode->block.n>0 ) break; - assert( *pRc || pNode->block.nAlloc==0 ); - assert( *pRc || pNode->key.nAlloc==0 ); - sqlite3_free(pNode->block.a); - sqlite3_free(pNode->key.a); - } - - /* Empty output segment. This is a no-op. */ - if( iRoot<0 ) return; + ExprList *pOB; + int ii, jj; - /* The entire output segment fits on a single node. Normally, this means - ** the node would be stored as a blob in the "root" column of the %_segdir - ** table. However, this is not permitted in this case. The problem is that - ** space has already been reserved in the %_segments table, and so the - ** start_block and end_block fields of the %_segdir table must be populated. - ** And, by design or by accident, released versions of FTS cannot handle - ** segments that fit entirely on the root node with start_block!=0. - ** - ** Instead, create a synthetic root node that contains nothing but a - ** pointer to the single content node. So that the segment consists of a - ** single leaf and a single interior (root) node. - ** - ** Todo: Better might be to defer allocating space in the %_segments - ** table until we are sure it is needed. - */ - if( iRoot==0 ){ - Blob *pBlock = &pWriter->aNodeWriter[1].block; - blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); - if( rc==SQLITE_OK ){ - pBlock->a[0] = 0x01; - pBlock->n = 1 + sqlite3Fts3PutVarint( - &pBlock->a[1], pWriter->aNodeWriter[0].iBlock - ); + if( pIndex->bUnordered ) return 0; + if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); + if( pExpr->op!=TK_COLUMN ) return 0; + if( pExpr->iTable==iCursor ){ + if( pExpr->iColumn<0 ) return 1; + for(jj=0; jjnKeyCol; jj++){ + if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; + } } - iRoot = 1; } - pRoot = &pWriter->aNodeWriter[iRoot]; + return 0; +} - /* Flush all currently outstanding nodes to disk. */ - for(i=0; iaNodeWriter[i]; - if( pNode->block.n>0 && rc==SQLITE_OK ){ - rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); +/* +** Return a bitmask where 1s indicate that the corresponding column of +** the table is used by an index. Only the first 63 columns are considered. +*/ +static Bitmask columnsInIndex(Index *pIdx){ + Bitmask m = 0; + int j; + for(j=pIdx->nColumn-1; j>=0; j--){ + int x = pIdx->aiColumn[j]; + if( x>=0 ){ + testcase( x==BMS-1 ); + testcase( x==BMS-2 ); + if( xblock.a); - sqlite3_free(pNode->key.a); } + return m; +} - /* Write the %_segdir record. */ - if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir(p, - pWriter->iAbsLevel+1, /* level */ - pWriter->iIdx, /* idx */ - pWriter->iStart, /* start_block */ - pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ - pWriter->iEnd, /* end_block */ - (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ - pRoot->block.a, pRoot->block.n /* root */ - ); +/* Check to see if a partial index with pPartIndexWhere can be used +** in the current query. Return true if it can be and false if not. +*/ +static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ + int i; + WhereTerm *pTerm; + for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + Expr *pExpr = pTerm->pExpr; + if( sqlite3ExprImpliesExpr(pExpr, pWhere, iTab) + && (!ExprHasProperty(pExpr, EP_FromJoin) || pExpr->iRightJoinTable==iTab) + ){ + return 1; + } } - sqlite3_free(pRoot->block.a); - sqlite3_free(pRoot->key.a); - - *pRc = rc; + return 0; } /* -** Compare the term in buffer zLhs (size in bytes nLhs) with that in -** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of -** the other, it is considered to be smaller than the other. +** Add all WhereLoop objects for a single table of the join where the table +** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be +** a b-tree table, not a virtual table. ** -** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve -** if it is greater. +** The costs (WhereLoop.rRun) of the b-tree loops added by this function +** are calculated as follows: +** +** For a full scan, assuming the table (or index) contains nRow rows: +** +** cost = nRow * 3.0 // full-table scan +** cost = nRow * K // scan of covering index +** cost = nRow * (K+3.0) // scan of non-covering index +** +** where K is a value between 1.1 and 3.0 set based on the relative +** estimated average size of the index and table records. +** +** For an index scan, where nVisit is the number of index rows visited +** by the scan, and nSeek is the number of seek operations required on +** the index b-tree: +** +** cost = nSeek * (log(nRow) + K * nVisit) // covering index +** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index +** +** Normally, nSeek is 1. nSeek values greater than 1 come about if the +** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when +** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. +** +** The estimated values (nRow, nVisit, nSeek) often contain a large amount +** of uncertainty. For this reason, scoring is designed to pick plans that +** "do the least harm" if the estimates are inaccurate. For example, a +** log(nRow) factor is omitted from a non-covering index scan in order to +** bias the scoring in favor of using an index, since the worst-case +** performance of using an index is far better than the worst-case performance +** of a full table scan. */ -static int fts3TermCmp( - const char *zLhs, int nLhs, /* LHS of comparison */ - const char *zRhs, int nRhs /* RHS of comparison */ +static int whereLoopAddBtree( + WhereLoopBuilder *pBuilder, /* WHERE clause information */ + Bitmask mExtra /* Extra prerequesites for using this table */ ){ - int nCmp = MIN(nLhs, nRhs); - int res; + WhereInfo *pWInfo; /* WHERE analysis context */ + Index *pProbe; /* An index we are evaluating */ + Index sPk; /* A fake index object for the primary key */ + LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ + i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ + SrcList *pTabList; /* The FROM clause */ + struct SrcList_item *pSrc; /* The FROM clause btree term to add */ + WhereLoop *pNew; /* Template WhereLoop object */ + int rc = SQLITE_OK; /* Return code */ + int iSortIdx = 1; /* Index number */ + int b; /* A boolean value */ + LogEst rSize; /* number of rows in the table */ + LogEst rLogSize; /* Logarithm of the number of rows in the table */ + WhereClause *pWC; /* The parsed WHERE clause */ + Table *pTab; /* Table being queried */ + + pNew = pBuilder->pNew; + pWInfo = pBuilder->pWInfo; + pTabList = pWInfo->pTabList; + pSrc = pTabList->a + pNew->iTab; + pTab = pSrc->pTab; + pWC = pBuilder->pWC; + assert( !IsVirtual(pSrc->pTab) ); - res = memcmp(zLhs, zRhs, nCmp); - if( res==0 ) res = nLhs - nRhs; + if( pSrc->pIndex ){ + /* An INDEXED BY clause specifies a particular index to use */ + pProbe = pSrc->pIndex; + }else if( !HasRowid(pTab) ){ + pProbe = pTab->pIndex; + }else{ + /* There is no INDEXED BY clause. Create a fake Index object in local + ** variable sPk to represent the rowid primary key index. Make this + ** fake index the first in a chain of Index objects with all of the real + ** indices to follow */ + Index *pFirst; /* First of real indices on the table */ + memset(&sPk, 0, sizeof(Index)); + sPk.nKeyCol = 1; + sPk.nColumn = 1; + sPk.aiColumn = &aiColumnPk; + sPk.aiRowLogEst = aiRowEstPk; + sPk.onError = OE_Replace; + sPk.pTable = pTab; + sPk.szIdxRow = pTab->szTabRow; + aiRowEstPk[0] = pTab->nRowLogEst; + aiRowEstPk[1] = 0; + pFirst = pSrc->pTab->pIndex; + if( pSrc->notIndexed==0 ){ + /* The real indices of the table are only considered if the + ** NOT INDEXED qualifier is omitted from the FROM clause */ + sPk.pNext = pFirst; + } + pProbe = &sPk; + } + rSize = pTab->nRowLogEst; + rLogSize = estLog(rSize); - return res; -} +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX + /* Automatic indexes */ + if( !pBuilder->pOrSet + && (pWInfo->wctrlFlags & WHERE_NO_AUTOINDEX)==0 + && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 + && pSrc->pIndex==0 + && !pSrc->viaCoroutine + && !pSrc->notIndexed + && HasRowid(pTab) + && !pSrc->isCorrelated + && !pSrc->isRecursive + ){ + /* Generate auto-index WhereLoops */ + WhereTerm *pTerm; + WhereTerm *pWCEnd = pWC->a + pWC->nTerm; + for(pTerm=pWC->a; rc==SQLITE_OK && pTermprereqRight & pNew->maskSelf ) continue; + if( termCanDriveIndex(pTerm, pSrc, 0) ){ + pNew->u.btree.nEq = 1; + pNew->nSkip = 0; + pNew->u.btree.pIndex = 0; + pNew->nLTerm = 1; + pNew->aLTerm[0] = pTerm; + /* TUNING: One-time cost for computing the automatic index is + ** estimated to be X*N*log2(N) where N is the number of rows in + ** the table being indexed and where X is 7 (LogEst=28) for normal + ** tables or 1.375 (LogEst=4) for views and subqueries. The value + ** of X is smaller for views and subqueries so that the query planner + ** will be more aggressive about generating automatic indexes for + ** those objects, since there is no opportunity to add schema + ** indexes on subqueries and views. */ + pNew->rSetup = rLogSize + rSize + 4; + if( pTab->pSelect==0 && (pTab->tabFlags & TF_Ephemeral)==0 ){ + pNew->rSetup += 24; + } + ApplyCostMultiplier(pNew->rSetup, pTab->costMult); + /* TUNING: Each index lookup yields 20 rows in the table. This + ** is more than the usual guess of 10 rows, since we have no way + ** of knowing how selective the index will ultimately be. It would + ** not be unreasonable to make this value much larger. */ + pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); + pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); + pNew->wsFlags = WHERE_AUTO_INDEX; + pNew->prereq = mExtra | pTerm->prereqRight; + rc = whereLoopInsert(pBuilder, pNew); + } + } + } +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + + /* Loop over all indices + */ + for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ + if( pProbe->pPartIdxWhere!=0 + && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){ + testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ + continue; /* Partial index inappropriate for this query */ + } + rSize = pProbe->aiRowLogEst[0]; + pNew->u.btree.nEq = 0; + pNew->nSkip = 0; + pNew->nLTerm = 0; + pNew->iSortIdx = 0; + pNew->rSetup = 0; + pNew->prereq = mExtra; + pNew->nOut = rSize; + pNew->u.btree.pIndex = pProbe; + b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); + /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ + assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 ); + if( pProbe->tnum<=0 ){ + /* Integer primary key index */ + pNew->wsFlags = WHERE_IPK; + /* Full table scan */ + pNew->iSortIdx = b ? iSortIdx : 0; + /* TUNING: Cost of full table scan is (N*3.0). */ + pNew->rRun = rSize + 16; + ApplyCostMultiplier(pNew->rRun, pTab->costMult); + whereLoopOutputAdjust(pWC, pNew, rSize); + rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = rSize; + if( rc ) break; + }else{ + Bitmask m; + if( pProbe->isCovering ){ + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; + m = 0; + }else{ + m = pSrc->colUsed & ~columnsInIndex(pProbe); + pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; + } -/* -** Query to see if the entry in the %_segments table with blockid iEnd is -** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before -** returning. Otherwise, set *pbRes to 0. -** -** Or, if an error occurs while querying the database, return an SQLite -** error code. The final value of *pbRes is undefined in this case. -** -** This is used to test if a segment is an "appendable" segment. If it -** is, then a NULL entry has been inserted into the %_segments table -** with blockid %_segdir.end_block. -*/ -static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ - int bRes = 0; /* Result to set *pbRes to */ - sqlite3_stmt *pCheck = 0; /* Statement to query database with */ - int rc; /* Return code */ + /* Full scan via index */ + if( b + || !HasRowid(pTab) + || ( m==0 + && pProbe->bUnordered==0 + && (pProbe->szIdxRowszTabRow) + && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 + && sqlite3GlobalConfig.bUseCis + && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan) + ) + ){ + pNew->iSortIdx = b ? iSortIdx : 0; - rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pCheck, 1, iEnd); - if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; - rc = sqlite3_reset(pCheck); + /* The cost of visiting the index rows is N*K, where K is + ** between 1.1 and 3.0, depending on the relative sizes of the + ** index and table rows. If this is a non-covering index scan, + ** also add the cost of visiting table rows (N*3.0). */ + pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; + if( m!=0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); + } + ApplyCostMultiplier(pNew->rRun, pTab->costMult); + whereLoopOutputAdjust(pWC, pNew, rSize); + rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = rSize; + if( rc ) break; + } + } + + rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3Stat4ProbeFree(pBuilder->pRec); + pBuilder->nRecValid = 0; + pBuilder->pRec = 0; +#endif + + /* If there was an INDEXED BY clause, then only that one index is + ** considered. */ + if( pSrc->pIndex ) break; } - - *pbRes = bRes; return rc; } +#ifndef SQLITE_OMIT_VIRTUALTABLE /* -** This function is called when initializing an incremental-merge operation. -** It checks if the existing segment with index value iIdx at absolute level -** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the -** merge-writer object *pWriter is initialized to write to it. -** -** An existing segment can be appended to by an incremental merge if: -** -** * It was initially created as an appendable segment (with all required -** space pre-allocated), and -** -** * The first key read from the input (arguments zKey and nKey) is -** greater than the largest key currently stored in the potential -** output segment. +** Add all WhereLoop objects for a table of the join identified by +** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. */ -static int fts3IncrmergeLoad( - Fts3Table *p, /* Fts3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ - int iIdx, /* Index of candidate output segment */ - const char *zKey, /* First key to write */ - int nKey, /* Number of bytes in nKey */ - IncrmergeWriter *pWriter /* Populate this object */ +static int whereLoopAddVirtual( + WhereLoopBuilder *pBuilder, /* WHERE clause information */ + Bitmask mExtra ){ - int rc; /* Return code */ - sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ + WhereInfo *pWInfo; /* WHERE analysis context */ + Parse *pParse; /* The parsing context */ + WhereClause *pWC; /* The WHERE clause */ + struct SrcList_item *pSrc; /* The FROM clause term to search */ + Table *pTab; + sqlite3 *db; + sqlite3_index_info *pIdxInfo; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_constraint_usage *pUsage; + WhereTerm *pTerm; + int i, j; + int iTerm, mxTerm; + int nConstraint; + int seenIn = 0; /* True if an IN operator is seen */ + int seenVar = 0; /* True if a non-constant constraint is seen */ + int iPhase; /* 0: const w/o IN, 1: const, 2: no IN, 2: IN */ + WhereLoop *pNew; + int rc = SQLITE_OK; - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); - if( rc==SQLITE_OK ){ - sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ - sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ - sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ - const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ - int nRoot = 0; /* Size of aRoot[] in bytes */ - int rc2; /* Return code from sqlite3_reset() */ - int bAppendable = 0; /* Set to true if segment is appendable */ + pWInfo = pBuilder->pWInfo; + pParse = pWInfo->pParse; + db = pParse->db; + pWC = pBuilder->pWC; + pNew = pBuilder->pNew; + pSrc = &pWInfo->pTabList->a[pNew->iTab]; + pTab = pSrc->pTab; + assert( IsVirtual(pTab) ); + pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pBuilder->pOrderBy); + if( pIdxInfo==0 ) return SQLITE_NOMEM; + pNew->prereq = 0; + pNew->rSetup = 0; + pNew->wsFlags = WHERE_VIRTUALTABLE; + pNew->nLTerm = 0; + pNew->u.vtab.needFree = 0; + pUsage = pIdxInfo->aConstraintUsage; + nConstraint = pIdxInfo->nConstraint; + if( whereLoopResize(db, pNew, nConstraint) ){ + sqlite3DbFree(db, pIdxInfo); + return SQLITE_NOMEM; + } - /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ - sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); - sqlite3_bind_int(pSelect, 2, iIdx); - if( sqlite3_step(pSelect)==SQLITE_ROW ){ - iStart = sqlite3_column_int64(pSelect, 1); - iLeafEnd = sqlite3_column_int64(pSelect, 2); - fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); - if( pWriter->nLeafData<0 ){ - pWriter->nLeafData = pWriter->nLeafData * -1; + for(iPhase=0; iPhase<=3; iPhase++){ + if( !seenIn && (iPhase&1)!=0 ){ + iPhase++; + if( iPhase>3 ) break; + } + if( !seenVar && iPhase>1 ) break; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pIdxCons++){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + switch( iPhase ){ + case 0: /* Constants without IN operator */ + pIdxCons->usable = 0; + if( (pTerm->eOperator & WO_IN)!=0 ){ + seenIn = 1; + } + if( pTerm->prereqRight!=0 ){ + seenVar = 1; + }else if( (pTerm->eOperator & WO_IN)==0 ){ + pIdxCons->usable = 1; + } + break; + case 1: /* Constants with IN operators */ + assert( seenIn ); + pIdxCons->usable = (pTerm->prereqRight==0); + break; + case 2: /* Variables without IN */ + assert( seenVar ); + pIdxCons->usable = (pTerm->eOperator & WO_IN)==0; + break; + default: /* Variables with IN */ + assert( seenVar && seenIn ); + pIdxCons->usable = 1; + break; } - pWriter->bNoLeafData = (pWriter->nLeafData==0); - nRoot = sqlite3_column_bytes(pSelect, 4); - aRoot = sqlite3_column_blob(pSelect, 4); - }else{ - return sqlite3_reset(pSelect); } - - /* Check for the zero-length marker in the %_segments table */ - rc = fts3IsAppendable(p, iEnd, &bAppendable); - - /* Check that zKey/nKey is larger than the largest key the candidate */ - if( rc==SQLITE_OK && bAppendable ){ - char *aLeaf = 0; - int nLeaf = 0; - - rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); - if( rc==SQLITE_OK ){ - NodeReader reader; - for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); - rc==SQLITE_OK && reader.aNode; - rc = nodeReaderNext(&reader) + memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); + if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->needToFreeIdxStr = 0; + pIdxInfo->orderByConsumed = 0; + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; + pIdxInfo->estimatedRows = 25; + rc = vtabBestIndex(pParse, pTab, pIdxInfo); + if( rc ) goto whereLoopAddVtab_exit; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + pNew->prereq = mExtra; + mxTerm = -1; + assert( pNew->nLSlot>=nConstraint ); + for(i=0; iaLTerm[i] = 0; + pNew->u.vtab.omitMask = 0; + for(i=0; i=0 ){ + j = pIdxCons->iTermOffset; + if( iTerm>=nConstraint + || j<0 + || j>=pWC->nTerm + || pNew->aLTerm[iTerm]!=0 ){ - assert( reader.aNode ); - } - if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ - bAppendable = 0; - } - nodeReaderRelease(&reader); + rc = SQLITE_ERROR; + sqlite3ErrorMsg(pParse, "%s.xBestIndex() malfunction", pTab->zName); + goto whereLoopAddVtab_exit; + } + testcase( iTerm==nConstraint-1 ); + testcase( j==0 ); + testcase( j==pWC->nTerm-1 ); + pTerm = &pWC->a[j]; + pNew->prereq |= pTerm->prereqRight; + assert( iTermnLSlot ); + pNew->aLTerm[iTerm] = pTerm; + if( iTerm>mxTerm ) mxTerm = iTerm; + testcase( iTerm==15 ); + testcase( iTerm==16 ); + if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ + if( pUsage[i].omit==0 ){ + /* Do not attempt to use an IN constraint if the virtual table + ** says that the equivalent EQ constraint cannot be safely omitted. + ** If we do attempt to use such a constraint, some rows might be + ** repeated in the output. */ + break; + } + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; + } + } + } + if( i>=nConstraint ){ + pNew->nLTerm = mxTerm+1; + assert( pNew->nLTerm<=pNew->nLSlot ); + pNew->u.vtab.idxNum = pIdxInfo->idxNum; + pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; + pIdxInfo->needToFreeIdxStr = 0; + pNew->u.vtab.idxStr = pIdxInfo->idxStr; + pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? + pIdxInfo->nOrderBy : 0); + pNew->rSetup = 0; + pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); + pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); + whereLoopInsert(pBuilder, pNew); + if( pNew->u.vtab.needFree ){ + sqlite3_free(pNew->u.vtab.idxStr); + pNew->u.vtab.needFree = 0; } - sqlite3_free(aLeaf); } + } - if( rc==SQLITE_OK && bAppendable ){ - /* It is possible to append to this segment. Set up the IncrmergeWriter - ** object to do so. */ - int i; - int nHeight = (int)aRoot[0]; - NodeWriter *pNode; - - pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; - pWriter->iStart = iStart; - pWriter->iEnd = iEnd; - pWriter->iAbsLevel = iAbsLevel; - pWriter->iIdx = iIdx; - - for(i=nHeight+1; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; - } +whereLoopAddVtab_exit: + if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); + sqlite3DbFree(db, pIdxInfo); + return rc; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ - pNode = &pWriter->aNodeWriter[nHeight]; - pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; - blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); - if( rc==SQLITE_OK ){ - memcpy(pNode->block.a, aRoot, nRoot); - pNode->block.n = nRoot; - } +/* +** Add WhereLoop entries to handle OR terms. This works for either +** btrees or virtual tables. +*/ +static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ + WhereInfo *pWInfo = pBuilder->pWInfo; + WhereClause *pWC; + WhereLoop *pNew; + WhereTerm *pTerm, *pWCEnd; + int rc = SQLITE_OK; + int iCur; + WhereClause tempWC; + WhereLoopBuilder sSubBuild; + WhereOrSet sSum, sCur; + struct SrcList_item *pItem; + + pWC = pBuilder->pWC; + pWCEnd = pWC->a + pWC->nTerm; + pNew = pBuilder->pNew; + memset(&sSum, 0, sizeof(sSum)); + pItem = pWInfo->pTabList->a + pNew->iTab; + iCur = pItem->iCursor; - for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ - NodeReader reader; - pNode = &pWriter->aNodeWriter[i]; + for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 + && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 + ){ + WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; + WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; + WhereTerm *pOrTerm; + int once = 1; + int i, j; + + sSubBuild = *pBuilder; + sSubBuild.pOrderBy = 0; + sSubBuild.pOrSet = &sCur; - rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); - while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); - blobGrowBuffer(&pNode->key, reader.term.n, &rc); + WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); + for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ + sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; + }else if( pOrTerm->leftCursor==iCur ){ + tempWC.pWInfo = pWC->pWInfo; + tempWC.pOuter = pWC; + tempWC.op = TK_AND; + tempWC.nTerm = 1; + tempWC.a = pOrTerm; + sSubBuild.pWC = &tempWC; + }else{ + continue; + } + sCur.n = 0; +#ifdef WHERETRACE_ENABLED + WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", + (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); + if( sqlite3WhereTrace & 0x400 ){ + for(i=0; inTerm; i++){ + whereTermPrint(&sSubBuild.pWC->a[i], i); + } + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pItem->pTab) ){ + rc = whereLoopAddVirtual(&sSubBuild, mExtra); + }else +#endif + { + rc = whereLoopAddBtree(&sSubBuild, mExtra); + } if( rc==SQLITE_OK ){ - memcpy(pNode->key.a, reader.term.a, reader.term.n); - pNode->key.n = reader.term.n; - if( i>0 ){ - char *aBlock = 0; - int nBlock = 0; - pNode = &pWriter->aNodeWriter[i-1]; - pNode->iBlock = reader.iChild; - rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); - blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); - if( rc==SQLITE_OK ){ - memcpy(pNode->block.a, aBlock, nBlock); - pNode->block.n = nBlock; + rc = whereLoopAddOr(&sSubBuild, mExtra); + } + assert( rc==SQLITE_OK || sCur.n==0 ); + if( sCur.n==0 ){ + sSum.n = 0; + break; + }else if( once ){ + whereOrMove(&sSum, &sCur); + once = 0; + }else{ + WhereOrSet sPrev; + whereOrMove(&sPrev, &sSum); + sSum.n = 0; + for(i=0; inLTerm = 1; + pNew->aLTerm[0] = pTerm; + pNew->wsFlags = WHERE_MULTI_OR; + pNew->rSetup = 0; + pNew->iSortIdx = 0; + memset(&pNew->u, 0, sizeof(pNew->u)); + for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun + 1; + pNew->nOut = sSum.a[i].nOut; + pNew->prereq = sSum.a[i].prereq; + rc = whereLoopInsert(pBuilder, pNew); + } + WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); } - - rc2 = sqlite3_reset(pSelect); - if( rc==SQLITE_OK ) rc = rc2; } - return rc; } /* -** Determine the largest segment index value that exists within absolute -** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus -** one before returning SQLITE_OK. Or, if there are no segments at all -** within level iAbsLevel, set *piIdx to zero. -** -** If an error occurs, return an SQLite error code. The final value of -** *piIdx is undefined in this case. +** Add all WhereLoop objects for all tables */ -static int fts3IncrmergeOutputIdx( - Fts3Table *p, /* FTS Table handle */ - sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ - int *piIdx /* OUT: Next free index at iAbsLevel+1 */ -){ - int rc; - sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ +static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo = pBuilder->pWInfo; + Bitmask mExtra = 0; + Bitmask mPrior = 0; + int iTab; + SrcList *pTabList = pWInfo->pTabList; + struct SrcList_item *pItem; + sqlite3 *db = pWInfo->pParse->db; + int nTabList = pWInfo->nLevel; + int rc = SQLITE_OK; + u8 priorJoinType = 0; + WhereLoop *pNew; - rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); - sqlite3_step(pOutputIdx); - *piIdx = sqlite3_column_int(pOutputIdx, 0); - rc = sqlite3_reset(pOutputIdx); + /* Loop over the tables in the join, from left to right */ + pNew = pBuilder->pNew; + whereLoopInit(pNew); + for(iTab=0, pItem=pTabList->a; iTabiTab = iTab; + pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor); + if( ((pItem->jointype|priorJoinType) & (JT_LEFT|JT_CROSS))!=0 ){ + mExtra = mPrior; + } + priorJoinType = pItem->jointype; + if( IsVirtual(pItem->pTab) ){ + rc = whereLoopAddVirtual(pBuilder, mExtra); + }else{ + rc = whereLoopAddBtree(pBuilder, mExtra); + } + if( rc==SQLITE_OK ){ + rc = whereLoopAddOr(pBuilder, mExtra); + } + mPrior |= pNew->maskSelf; + if( rc || db->mallocFailed ) break; } - + whereLoopClear(db, pNew); return rc; } -/* -** Allocate an appendable output segment on absolute level iAbsLevel+1 -** with idx value iIdx. -** -** In the %_segdir table, a segment is defined by the values in three -** columns: -** -** start_block -** leaves_end_block -** end_block -** -** When an appendable segment is allocated, it is estimated that the -** maximum number of leaf blocks that may be required is the sum of the -** number of leaf blocks consumed by the input segments, plus the number -** of input segments, multiplied by two. This value is stored in stack -** variable nLeafEst. -** -** A total of 16*nLeafEst blocks are allocated when an appendable segment -** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous -** array of leaf nodes starts at the first block allocated. The array -** of interior nodes that are parents of the leaf nodes start at block -** (start_block + (1 + end_block - start_block) / 16). And so on. +/* +** Examine a WherePath (with the addition of the extra WhereLoop of the 5th +** parameters) to see if it outputs rows in the requested ORDER BY +** (or GROUP BY) without requiring a separate sort operation. Return N: +** +** N>0: N terms of the ORDER BY clause are satisfied +** N==0: No terms of the ORDER BY clause are satisfied +** N<0: Unknown yet how many terms of ORDER BY might be satisfied. ** -** In the actual code below, the value "16" is replaced with the -** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. +** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as +** strict. With GROUP BY and DISTINCT the only requirement is that +** equivalent rows appear immediately adjacent to one another. GROUP BY +** and DISTINCT do not require rows to appear in any particular order as long +** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT +** the pOrderBy terms can be matched in any order. With ORDER BY, the +** pOrderBy terms must be matched in strict left-to-right order. */ -static int fts3IncrmergeWriter( - Fts3Table *p, /* Fts3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ - int iIdx, /* Index of new output segment */ - Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ - IncrmergeWriter *pWriter /* Populate this object */ +static i8 wherePathSatisfiesOrderBy( + WhereInfo *pWInfo, /* The WHERE clause */ + ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ + WherePath *pPath, /* The WherePath to check */ + u16 wctrlFlags, /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */ + u16 nLoop, /* Number of entries in pPath->aLoop[] */ + WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ + Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ ){ - int rc; /* Return Code */ - int i; /* Iterator variable */ - int nLeafEst = 0; /* Blocks allocated for leaf nodes */ - sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ - sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ + u8 revSet; /* True if rev is known */ + u8 rev; /* Composite sort order */ + u8 revIdx; /* Index sort order */ + u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ + u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */ + u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ + u16 nKeyCol; /* Number of key columns in pIndex */ + u16 nColumn; /* Total number of ordered columns in the index */ + u16 nOrderBy; /* Number terms in the ORDER BY clause */ + int iLoop; /* Index of WhereLoop in pPath being processed */ + int i, j; /* Loop counters */ + int iCur; /* Cursor number for current WhereLoop */ + int iColumn; /* A column number within table iCur */ + WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + Expr *pOBExpr; /* An expression from the ORDER BY clause */ + CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */ + Index *pIndex; /* The index associated with pLoop */ + sqlite3 *db = pWInfo->pParse->db; /* Database connection */ + Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ + Bitmask obDone; /* Mask of all ORDER BY terms */ + Bitmask orderDistinctMask; /* Mask of all well-ordered loops */ + Bitmask ready; /* Mask of inner loops */ - /* Calculate nLeafEst. */ - rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); - sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); - if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ - nLeafEst = sqlite3_column_int(pLeafEst, 0); - } - rc = sqlite3_reset(pLeafEst); - } - if( rc!=SQLITE_OK ) return rc; + /* + ** We say the WhereLoop is "one-row" if it generates no more than one + ** row of output. A WhereLoop is one-row if all of the following are true: + ** (a) All index columns match with WHERE_COLUMN_EQ. + ** (b) The index is unique + ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row. + ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags. + ** + ** We say the WhereLoop is "order-distinct" if the set of columns from + ** that WhereLoop that are in the ORDER BY clause are different for every + ** row of the WhereLoop. Every one-row WhereLoop is automatically + ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause + ** is not order-distinct. To be order-distinct is not quite the same as being + ** UNIQUE since a UNIQUE column or index can have multiple rows that + ** are NULL and NULL values are equivalent for the purpose of order-distinct. + ** To be order-distinct, the columns must be UNIQUE and NOT NULL. + ** + ** The rowid for a table is always UNIQUE and NOT NULL so whenever the + ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is + ** automatically order-distinct. + */ - /* Calculate the first block to use in the output segment */ - rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ - pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); - pWriter->iEnd = pWriter->iStart - 1; - pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; - } - rc = sqlite3_reset(pFirstBlock); - } - if( rc!=SQLITE_OK ) return rc; + assert( pOrderBy!=0 ); + if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; - /* Insert the marker in the %_segments table to make sure nobody tries - ** to steal the space just allocated. This is also used to identify - ** appendable segments. */ - rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); - if( rc!=SQLITE_OK ) return rc; + nOrderBy = pOrderBy->nExpr; + testcase( nOrderBy==BMS-1 ); + if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */ + isOrderDistinct = 1; + obDone = MASKBIT(nOrderBy)-1; + orderDistinctMask = 0; + ready = 0; + for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; + pLoop = iLoopaLoop[iLoop] : pLast; + if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ + if( pLoop->u.vtab.isOrdered ) obSat = obDone; + break; + } + iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; - pWriter->iAbsLevel = iAbsLevel; - pWriter->nLeafEst = nLeafEst; - pWriter->iIdx = iIdx; + /* Mark off any ORDER BY term X that is a column in the table of + ** the current loop for which there is term in the WHERE + ** clause of the form X IS NULL or X=? that reference only outer + ** loops. + */ + for(i=0; ia[i].pExpr); + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, + ~ready, WO_EQ|WO_ISNULL, 0); + if( pTerm==0 ) continue; + if( (pTerm->eOperator&WO_EQ)!=0 && pOBExpr->iColumn>=0 ){ + const char *z1, *z2; + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); + if( !pColl ) pColl = db->pDfltColl; + z1 = pColl->zName; + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pTerm->pExpr); + if( !pColl ) pColl = db->pDfltColl; + z2 = pColl->zName; + if( sqlite3StrICmp(z1, z2)!=0 ) continue; + } + obSat |= MASKBIT(i); + } - /* Set up the array of NodeWriter objects */ - for(i=0; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; - } - return SQLITE_OK; -} + if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ + if( pLoop->wsFlags & WHERE_IPK ){ + pIndex = 0; + nKeyCol = 0; + nColumn = 1; + }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ + return 0; + }else{ + nKeyCol = pIndex->nKeyCol; + nColumn = pIndex->nColumn; + assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); + assert( pIndex->aiColumn[nColumn-1]==(-1) || !HasRowid(pIndex->pTable)); + isOrderDistinct = IsUniqueIndex(pIndex); + } -/* -** Remove an entry from the %_segdir table. This involves running the -** following two statements: -** -** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx -** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx -** -** The DELETE statement removes the specific %_segdir level. The UPDATE -** statement ensures that the remaining segments have contiguously allocated -** idx values. -*/ -static int fts3RemoveSegdirEntry( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ - int iIdx /* Index of %_segdir entry to delete */ -){ - int rc; /* Return code */ - sqlite3_stmt *pDelete = 0; /* DELETE statement */ + /* Loop through all columns of the index and deal with the ones + ** that are not constrained by == or IN. + */ + rev = revSet = 0; + distinctColumns = 0; + for(j=0; ju.btree.nEq + && pLoop->nSkip==0 + && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 + ){ + if( i & WO_ISNULL ){ + testcase( isOrderDistinct ); + isOrderDistinct = 0; + } + continue; + } - return rc; -} + /* Get the column number in the table (iColumn) and sort order + ** (revIdx) for the j-th column of the index. + */ + if( pIndex ){ + iColumn = pIndex->aiColumn[j]; + revIdx = pIndex->aSortOrder[j]; + if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; + }else{ + iColumn = -1; + revIdx = 0; + } -/* -** One or more segments have just been removed from absolute level iAbsLevel. -** Update the 'idx' values of the remaining segments in the level so that -** the idx values are a contiguous sequence starting from 0. -*/ -static int fts3RepackSegdirLevel( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iAbsLevel /* Absolute level to repack */ -){ - int rc; /* Return code */ - int *aIdx = 0; /* Array of remaining idx values */ - int nIdx = 0; /* Valid entries in aIdx[] */ - int nAlloc = 0; /* Allocated size of aIdx[] */ - int i; /* Iterator variable */ - sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ - sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ + /* An unconstrained column that might be NULL means that this + ** WhereLoop is not well-ordered + */ + if( isOrderDistinct + && iColumn>=0 + && j>=pLoop->u.btree.nEq + && pIndex->pTable->aCol[iColumn].notNull==0 + ){ + isOrderDistinct = 0; + } - rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); - if( rc==SQLITE_OK ){ - int rc2; - sqlite3_bind_int64(pSelect, 1, iAbsLevel); - while( SQLITE_ROW==sqlite3_step(pSelect) ){ - if( nIdx>=nAlloc ){ - int *aNew; - nAlloc += 16; - aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); - if( !aNew ){ - rc = SQLITE_NOMEM; + /* Find the ORDER BY term that corresponds to the j-th column + ** of the index and mark that ORDER BY term off + */ + bOnce = 1; + isMatch = 0; + for(i=0; bOnce && ia[i].pExpr); + testcase( wctrlFlags & WHERE_GROUPBY ); + testcase( wctrlFlags & WHERE_DISTINCTBY ); + if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + if( iColumn>=0 ){ + pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); + if( !pColl ) pColl = db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; + } + isMatch = 1; break; } - aIdx = aNew; + if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){ + /* Make sure the sort order is compatible in an ORDER BY clause. + ** Sort order is irrelevant for a GROUP BY clause. */ + if( revSet ){ + if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0; + }else{ + rev = revIdx ^ pOrderBy->a[i].sortOrder; + if( rev ) *pRevMask |= MASKBIT(iLoop); + revSet = 1; + } + } + if( isMatch ){ + if( iColumn<0 ){ + testcase( distinctColumns==0 ); + distinctColumns = 1; + } + obSat |= MASKBIT(i); + }else{ + /* No match found */ + if( j==0 || jbIgnoreSavepoint==0 ); - p->bIgnoreSavepoint = 1; - for(i=0; rc==SQLITE_OK && imaskSelf; + for(i=0; ia[i].pExpr; + mTerm = exprTableUsage(&pWInfo->sMaskSet,p); + if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( (mTerm&~orderDistinctMask)==0 ){ + obSat |= MASKBIT(i); + } + } + } + } /* End the loop over all WhereLoops from outer-most down to inner-most */ + if( obSat==obDone ) return (i8)nOrderBy; + if( !isOrderDistinct ){ + for(i=nOrderBy-1; i>0; i--){ + Bitmask m = MASKBIT(i) - 1; + if( (obSat&m)==m ) return i; } + return 0; } - p->bIgnoreSavepoint = 0; + return -1; +} - sqlite3_free(aIdx); - return rc; + +/* +** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(), +** the planner assumes that the specified pOrderBy list is actually a GROUP +** BY clause - and so any order that groups rows as required satisfies the +** request. +** +** Normally, in this case it is not possible for the caller to determine +** whether or not the rows are really being delivered in sorted order, or +** just in some other order that provides the required grouping. However, +** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then +** this function may be called on the returned WhereInfo object. It returns +** true if the rows really will be sorted in the specified order, or false +** otherwise. +** +** For example, assuming: +** +** CREATE INDEX i1 ON t1(x, Y); +** +** then +** +** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1 +** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0 +*/ +SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo *pWInfo){ + assert( pWInfo->wctrlFlags & WHERE_GROUPBY ); + assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); + return pWInfo->sorted; } -static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ - pNode->a[0] = (char)iHeight; - if( iChild ){ - assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); - pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); - }else{ - assert( pNode->nAlloc>=1 ); - pNode->n = 1; - } +#ifdef WHERETRACE_ENABLED +/* For debugging use only: */ +static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ + static char zName[65]; + int i; + for(i=0; iaLoop[i]->cId; } + if( pLast ) zName[i++] = pLast->cId; + zName[i] = 0; + return zName; } +#endif /* -** The first two arguments are a pointer to and the size of a segment b-tree -** node. The node may be a leaf or an internal node. -** -** This function creates a new node image in blob object *pNew by copying -** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) -** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. +** Return the cost of sorting nRow rows, assuming that the keys have +** nOrderby columns and that the first nSorted columns are already in +** order. */ -static int fts3TruncateNode( - const char *aNode, /* Current node image */ - int nNode, /* Size of aNode in bytes */ - Blob *pNew, /* OUT: Write new node image here */ - const char *zTerm, /* Omit all terms smaller than this */ - int nTerm, /* Size of zTerm in bytes */ - sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +static LogEst whereSortingCost( + WhereInfo *pWInfo, + LogEst nRow, + int nOrderBy, + int nSorted ){ - NodeReader reader; /* Reader object */ - Blob prev = {0, 0, 0}; /* Previous term written to new node */ - int rc = SQLITE_OK; /* Return code */ - int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ - - /* Allocate required output space */ - blobGrowBuffer(pNew, nNode, &rc); - if( rc!=SQLITE_OK ) return rc; - pNew->n = 0; + /* TUNING: Estimated cost of a full external sort, where N is + ** the number of rows to sort is: + ** + ** cost = (3.0 * N * log(N)). + ** + ** Or, if the order-by clause has X terms but only the last Y + ** terms are out of order, then block-sorting will reduce the + ** sorting cost to: + ** + ** cost = (3.0 * N * log(N)) * (Y/X) + ** + ** The (Y/X) term is implemented using stack variable rScale + ** below. */ + LogEst rScale, rSortCost; + assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); + rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; + rSortCost = nRow + estLog(nRow) + rScale + 16; - /* Populate new node buffer */ - for(rc = nodeReaderInit(&reader, aNode, nNode); - rc==SQLITE_OK && reader.aNode; - rc = nodeReaderNext(&reader) - ){ - if( pNew->n==0 ){ - int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); - if( res<0 || (bLeaf==0 && res==0) ) continue; - fts3StartNode(pNew, (int)aNode[0], reader.iChild); - *piBlock = reader.iChild; - } - rc = fts3AppendToNode( - pNew, &prev, reader.term.a, reader.term.n, - reader.aDoclist, reader.nDoclist - ); - if( rc!=SQLITE_OK ) break; - } - if( pNew->n==0 ){ - fts3StartNode(pNew, (int)aNode[0], reader.iChild); - *piBlock = reader.iChild; + /* TUNING: The cost of implementing DISTINCT using a B-TREE is + ** similar but with a larger constant of proportionality. + ** Multiply by an additional factor of 3.0. */ + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + rSortCost += 16; } - assert( pNew->n<=pNew->nAlloc ); - nodeReaderRelease(&reader); - sqlite3_free(prev.a); - return rc; + return rSortCost; } /* -** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute -** level iAbsLevel. This may involve deleting entries from the %_segments -** table, and modifying existing entries in both the %_segments and %_segdir -** tables. +** Given the list of WhereLoop objects at pWInfo->pLoops, this routine +** attempts to find the lowest cost path that visits each WhereLoop +** once. This path is then loaded into the pWInfo->a[].pWLoop fields. ** -** SQLITE_OK is returned if the segment is updated successfully. Or an -** SQLite error code otherwise. +** Assume that the total number of output rows that will need to be sorted +** will be nRowEst (in the 10*log2 representation). Or, ignore sorting +** costs if nRowEst==0. +** +** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation +** error occurs. */ -static int fts3TruncateSegment( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ - int iIdx, /* Index within level of segment to modify */ - const char *zTerm, /* Remove terms smaller than this */ - int nTerm /* Number of bytes in buffer zTerm */ -){ - int rc = SQLITE_OK; /* Return code */ - Blob root = {0,0,0}; /* New root page image */ - Blob block = {0,0,0}; /* Buffer used for any other block */ - sqlite3_int64 iBlock = 0; /* Block id */ - sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ - sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ - sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ - - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); - if( rc==SQLITE_OK ){ - int rc2; /* sqlite3_reset() return code */ - sqlite3_bind_int64(pFetch, 1, iAbsLevel); - sqlite3_bind_int(pFetch, 2, iIdx); - if( SQLITE_ROW==sqlite3_step(pFetch) ){ - const char *aRoot = sqlite3_column_blob(pFetch, 4); - int nRoot = sqlite3_column_bytes(pFetch, 4); - iOldStart = sqlite3_column_int64(pFetch, 1); - rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); - } - rc2 = sqlite3_reset(pFetch); - if( rc==SQLITE_OK ) rc = rc2; - } +static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ + int mxChoice; /* Maximum number of simultaneous paths tracked */ + int nLoop; /* Number of terms in the join */ + Parse *pParse; /* Parsing context */ + sqlite3 *db; /* The database connection */ + int iLoop; /* Loop counter over the terms of the join */ + int ii, jj; /* Loop counters */ + int mxI = 0; /* Index of next entry to replace */ + int nOrderBy; /* Number of ORDER BY clause terms */ + LogEst mxCost = 0; /* Maximum cost of a set of paths */ + LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */ + int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ + WherePath *aFrom; /* All nFrom paths at the previous level */ + WherePath *aTo; /* The nTo best paths at the current level */ + WherePath *pFrom; /* An element of aFrom[] that we are working on */ + WherePath *pTo; /* An element of aTo[] that we are working on */ + WhereLoop *pWLoop; /* One of the WhereLoop objects */ + WhereLoop **pX; /* Used to divy up the pSpace memory */ + LogEst *aSortCost = 0; /* Sorting and partial sorting costs */ + char *pSpace; /* Temporary memory used by this routine */ + int nSpace; /* Bytes of space allocated at pSpace */ - while( rc==SQLITE_OK && iBlock ){ - char *aBlock = 0; - int nBlock = 0; - iNewStart = iBlock; + pParse = pWInfo->pParse; + db = pParse->db; + nLoop = pWInfo->nLevel; + /* TUNING: For simple queries, only the best path is tracked. + ** For 2-way joins, the 5 best paths are followed. + ** For joins of 3 or more tables, track the 10 best paths */ + mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); + assert( nLoop<=pWInfo->pTabList->nSrc ); + WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); - rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); - if( rc==SQLITE_OK ){ - rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); - } - if( rc==SQLITE_OK ){ - rc = fts3WriteSegment(p, iNewStart, block.a, block.n); - } - sqlite3_free(aBlock); + /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this + ** case the purpose of this call is to estimate the number of rows returned + ** by the overall query. Once this estimate has been obtained, the caller + ** will invoke this function a second time, passing the estimate as the + ** nRowEst parameter. */ + if( pWInfo->pOrderBy==0 || nRowEst==0 ){ + nOrderBy = 0; + }else{ + nOrderBy = pWInfo->pOrderBy->nExpr; } - /* Variable iNewStart now contains the first valid leaf node. */ - if( rc==SQLITE_OK && iNewStart ){ - sqlite3_stmt *pDel = 0; - rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pDel, 1, iOldStart); - sqlite3_bind_int64(pDel, 2, iNewStart-1); - sqlite3_step(pDel); - rc = sqlite3_reset(pDel); - } + /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ + nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; + nSpace += sizeof(LogEst) * nOrderBy; + pSpace = sqlite3DbMallocRaw(db, nSpace); + if( pSpace==0 ) return SQLITE_NOMEM; + aTo = (WherePath*)pSpace; + aFrom = aTo+mxChoice; + memset(aFrom, 0, sizeof(aFrom[0])); + pX = (WhereLoop**)(aFrom+mxChoice); + for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){ + pFrom->aLoop = pX; + } + if( nOrderBy ){ + /* If there is an ORDER BY clause and it is not being ignored, set up + ** space for the aSortCost[] array. Each element of the aSortCost array + ** is either zero - meaning it has not yet been initialized - or the + ** cost of sorting nRowEst rows of data where the first X terms of + ** the ORDER BY clause are already in order, where X is the array + ** index. */ + aSortCost = (LogEst*)pX; + memset(aSortCost, 0, sizeof(LogEst) * nOrderBy); } + assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] ); + assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX ); - if( rc==SQLITE_OK ){ - sqlite3_stmt *pChomp = 0; - rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pChomp, 1, iNewStart); - sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); - sqlite3_bind_int64(pChomp, 3, iAbsLevel); - sqlite3_bind_int(pChomp, 4, iIdx); - sqlite3_step(pChomp); - rc = sqlite3_reset(pChomp); - } + /* Seed the search with a single WherePath containing zero WhereLoops. + ** + ** TUNING: Do not let the number of iterations go above 28. If the cost + ** of computing an automatic index is not paid back within the first 28 + ** rows, then do not use the automatic index. */ + aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) ); + nFrom = 1; + assert( aFrom[0].isOrdered==0 ); + if( nOrderBy ){ + /* If nLoop is zero, then there are no FROM terms in the query. Since + ** in this case the query may return a maximum of one row, the results + ** are already in the requested order. Set isOrdered to nOrderBy to + ** indicate this. Or, if nLoop is greater than zero, set isOrdered to + ** -1, indicating that the result set may or may not be ordered, + ** depending on the loops added to the current plan. */ + aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy; } - sqlite3_free(root.a); - sqlite3_free(block.a); - return rc; -} + /* Compute successively longer WherePaths using the previous generation + ** of WherePaths as the basis for the next. Keep track of the mxChoice + ** best paths at each generation */ + for(iLoop=0; iLooppLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ + LogEst rCost; /* Cost of path (pFrom+pWLoop) */ + LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ + i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ + Bitmask maskNew; /* Mask of src visited by (..) */ + Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ -/* -** This function is called after an incrmental-merge operation has run to -** merge (or partially merge) two or more segments from absolute level -** iAbsLevel. -** -** Each input segment is either removed from the db completely (if all of -** its data was copied to the output segment by the incrmerge operation) -** or modified in place so that it no longer contains those entries that -** have been duplicated in the output segment. -*/ -static int fts3IncrmergeChomp( - Fts3Table *p, /* FTS table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ - Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ - int *pnRem /* Number of segments not deleted */ -){ - int i; - int nRem = 0; - int rc = SQLITE_OK; + if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; + if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + /* At this point, pWLoop is a candidate to be the next loop. + ** Compute its cost */ + rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); + rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); + nOut = pFrom->nRow + pWLoop->nOut; + maskNew = pFrom->maskLoop | pWLoop->maskSelf; + if( isOrdered<0 ){ + isOrdered = wherePathSatisfiesOrderBy(pWInfo, + pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, + iLoop, pWLoop, &revMask); + }else{ + revMask = pFrom->revLoop; + } + if( isOrdered>=0 && isOrderednSegment-1; i>=0 && rc==SQLITE_OK; i--){ - Fts3SegReader *pSeg = 0; - int j; + WHERETRACE(0x002, + ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", + aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy, + rUnsorted, rCost)); + }else{ + rCost = rUnsorted; + } - /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding - ** somewhere in the pCsr->apSegment[] array. */ - for(j=0; ALWAYS(jnSegment); j++){ - pSeg = pCsr->apSegment[j]; - if( pSeg->iIdx==i ) break; + /* Check to see if pWLoop should be added to the set of + ** mxChoice best-so-far paths. + ** + ** First look for an existing path among best-so-far paths + ** that covers the same set of loops and has the same isOrdered + ** setting as the current path candidate. + ** + ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent + ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range + ** of legal values for isOrdered, -1..64. + */ + for(jj=0, pTo=aTo; jjmaskLoop==maskNew + && ((pTo->isOrdered^isOrdered)&0x80)==0 + ){ + testcase( jj==nTo-1 ); + break; + } + } + if( jj>=nTo ){ + /* None of the existing best-so-far paths match the candidate. */ + if( nTo>=mxChoice + && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted)) + ){ + /* The current candidate is no better than any of the mxChoice + ** paths currently in the best-so-far buffer. So discard + ** this candidate as not viable. */ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + isOrdered>=0 ? isOrdered+'0' : '?'); + } +#endif + continue; + } + /* If we reach this points it means that the new candidate path + ** needs to be added to the set of best-so-far paths. */ + if( nTo=0 ? isOrdered+'0' : '?'); + } +#endif + }else{ + /* Control reaches here if best-so-far path pTo=aTo[jj] covers the + ** same set of loops and has the sam isOrdered setting as the + ** candidate path. Check to see if the candidate should replace + ** pTo or if the candidate should be skipped */ + if( pTo->rCostrCost==rCost && pTo->nRow<=nOut) ){ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf( + "Skip %s cost=%-3d,%3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + isOrdered>=0 ? isOrdered+'0' : '?'); + sqlite3DebugPrintf(" vs %s cost=%-3d,%d order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + } +#endif + /* Discard the candidate path from further consideration */ + testcase( pTo->rCost==rCost ); + continue; + } + testcase( pTo->rCost==rCost+1 ); + /* Control reaches here if the candidate path is better than the + ** pTo path. Replace pTo with the candidate. */ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf( + "Update %s cost=%-3d,%3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + isOrdered>=0 ? isOrdered+'0' : '?'); + sqlite3DebugPrintf(" was %s cost=%-3d,%3d order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + } +#endif + } + /* pWLoop is a winner. Add it to the set of best so far */ + pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; + pTo->revLoop = revMask; + pTo->nRow = nOut; + pTo->rCost = rCost; + pTo->rUnsorted = rUnsorted; + pTo->isOrdered = isOrdered; + memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); + pTo->aLoop[iLoop] = pWLoop; + if( nTo>=mxChoice ){ + mxI = 0; + mxCost = aTo[0].rCost; + mxUnsorted = aTo[0].nRow; + for(jj=1, pTo=&aTo[1]; jjrCost>mxCost + || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) + ){ + mxCost = pTo->rCost; + mxUnsorted = pTo->rUnsorted; + mxI = jj; + } + } + } + } } - assert( jnSegment && pSeg->iIdx==i ); - if( pSeg->aNode==0 ){ - /* Seg-reader is at EOF. Remove the entire input segment. */ - rc = fts3DeleteSegment(p, pSeg); - if( rc==SQLITE_OK ){ - rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); +#ifdef WHERETRACE_ENABLED /* >=2 */ + if( sqlite3WhereTrace & 0x02 ){ + sqlite3DebugPrintf("---- after round %d ----\n", iLoop); + for(ii=0, pTo=aTo; iirCost, pTo->nRow, + pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); + if( pTo->isOrdered>0 ){ + sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); + }else{ + sqlite3DebugPrintf("\n"); + } } - *pnRem = 0; - }else{ - /* The incremental merge did not copy all the data from this - ** segment to the upper level. The segment is modified in place - ** so that it contains no keys smaller than zTerm/nTerm. */ - const char *zTerm = pSeg->zTerm; - int nTerm = pSeg->nTerm; - rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); - nRem++; } - } +#endif - if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ - rc = fts3RepackSegdirLevel(p, iAbsLevel); + /* Swap the roles of aFrom and aTo for the next generation */ + pFrom = aTo; + aTo = aFrom; + aFrom = pFrom; + nFrom = nTo; } - *pnRem = nRem; - return rc; -} - -/* -** Store an incr-merge hint in the database. -*/ -static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ - sqlite3_stmt *pReplace = 0; - int rc; /* Return code */ - - rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); - sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); - sqlite3_step(pReplace); - rc = sqlite3_reset(pReplace); + if( nFrom==0 ){ + sqlite3ErrorMsg(pParse, "no query solution"); + sqlite3DbFree(db, pSpace); + return SQLITE_ERROR; } - - return rc; -} - -/* -** Load an incr-merge hint from the database. The incr-merge hint, if one -** exists, is stored in the rowid==1 row of the %_stat table. -** -** If successful, populate blob *pHint with the value read from the %_stat -** table and return SQLITE_OK. Otherwise, if an error occurs, return an -** SQLite error code. -*/ -static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ - sqlite3_stmt *pSelect = 0; - int rc; - - pHint->n = 0; - rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); - if( rc==SQLITE_OK ){ - int rc2; - sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); - if( SQLITE_ROW==sqlite3_step(pSelect) ){ - const char *aHint = sqlite3_column_blob(pSelect, 0); - int nHint = sqlite3_column_bytes(pSelect, 0); - if( aHint ){ - blobGrowBuffer(pHint, nHint, &rc); - if( rc==SQLITE_OK ){ - memcpy(pHint->a, aHint, nHint); - pHint->n = nHint; - } + + /* Find the lowest cost path. pFrom will be left pointing to that path */ + pFrom = aFrom; + for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; + } + assert( pWInfo->nLevel==nLoop ); + /* Load the lowest cost path into pWInfo */ + for(iLoop=0; iLoopa + iLoop; + pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop]; + pLevel->iFrom = pWLoop->iTab; + pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; + } + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 + && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 + && pWInfo->eDistinct==WHERE_DISTINCT_NOOP + && nRowEst + ){ + Bitmask notUsed; + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, + WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); + if( rc==pWInfo->pResultSet->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } + } + if( pWInfo->pOrderBy ){ + if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ + if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } + }else{ + pWInfo->nOBSat = pFrom->isOrdered; + if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0; + pWInfo->revMask = pFrom->revLoop; + } + if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) + && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0 + ){ + Bitmask revMask = 0; + int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, + pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask + ); + assert( pWInfo->sorted==0 ); + if( nOrder==pWInfo->pOrderBy->nExpr ){ + pWInfo->sorted = 1; + pWInfo->revMask = revMask; } } - rc2 = sqlite3_reset(pSelect); - if( rc==SQLITE_OK ) rc = rc2; } - return rc; + + pWInfo->nRowOut = pFrom->nRow; + + /* Free temporary memory and return success */ + sqlite3DbFree(db, pSpace); + return SQLITE_OK; } /* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, append an entry to the hint stored in blob *pHint. Each entry -** consists of two varints, the absolute level number of the input segments -** and the number of input segments. +** Most queries use only a single table (they are not joins) and have +** simple == constraints against indexed fields. This routine attempts +** to plan those simple cases using much less ceremony than the +** general-purpose query planner, and thereby yield faster sqlite3_prepare() +** times for the common case. ** -** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, -** set *pRc to an SQLite error code before returning. +** Return non-zero on success, if this query can be handled by this +** no-frills query planner. Return zero if this query needs the +** general-purpose query planner. */ -static void fts3IncrmergeHintPush( - Blob *pHint, /* Hint blob to append to */ - i64 iAbsLevel, /* First varint to store in hint */ - int nInput, /* Second varint to store in hint */ - int *pRc /* IN/OUT: Error code */ -){ - blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); - if( *pRc==SQLITE_OK ){ - pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); - pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); +static int whereShortCut(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo; + struct SrcList_item *pItem; + WhereClause *pWC; + WhereTerm *pTerm; + WhereLoop *pLoop; + int iCur; + int j; + Table *pTab; + Index *pIdx; + + pWInfo = pBuilder->pWInfo; + if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0; + assert( pWInfo->pTabList->nSrc>=1 ); + pItem = pWInfo->pTabList->a; + pTab = pItem->pTab; + if( IsVirtual(pTab) ) return 0; + if( pItem->zIndex ) return 0; + iCur = pItem->iCursor; + pWC = &pWInfo->sWC; + pLoop = pBuilder->pNew; + pLoop->wsFlags = 0; + pLoop->nSkip = 0; + pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); + if( pTerm ){ + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; + pLoop->aLTerm[0] = pTerm; + pLoop->nLTerm = 1; + pLoop->u.btree.nEq = 1; + /* TUNING: Cost of a rowid lookup is 10 */ + pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */ + }else{ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pLoop->aLTermSpace==pLoop->aLTerm ); + if( !IsUniqueIndex(pIdx) + || pIdx->pPartIdxWhere!=0 + || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) + ) continue; + for(j=0; jnKeyCol; j++){ + pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 0, WO_EQ, pIdx); + if( pTerm==0 ) break; + pLoop->aLTerm[j] = pTerm; + } + if( j!=pIdx->nKeyCol ) continue; + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; + if( pIdx->isCovering || (pItem->colUsed & ~columnsInIndex(pIdx))==0 ){ + pLoop->wsFlags |= WHERE_IDX_ONLY; + } + pLoop->nLTerm = j; + pLoop->u.btree.nEq = j; + pLoop->u.btree.pIndex = pIdx; + /* TUNING: Cost of a unique index lookup is 15 */ + pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */ + break; + } + } + if( pLoop->wsFlags ){ + pLoop->nOut = (LogEst)1; + pWInfo->a[0].pWLoop = pLoop; + pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); + pWInfo->a[0].iTabCur = iCur; + pWInfo->nRowOut = 1; + if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } +#ifdef SQLITE_DEBUG + pLoop->cId = '0'; +#endif + return 1; } + return 0; } /* -** Read the last entry (most recently pushed) from the hint blob *pHint -** and then remove the entry. Write the two values read to *piAbsLevel and -** *pnInput before returning. +** Generate the beginning of the loop used for WHERE clause processing. +** The return value is a pointer to an opaque structure that contains +** information needed to terminate the loop. Later, the calling routine +** should invoke sqlite3WhereEnd() with the return value of this function +** in order to complete the WHERE clause processing. ** -** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does -** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. +** If an error occurs, this routine returns NULL. +** +** The basic idea is to do a nested loop, one loop for each table in +** the FROM clause of a select. (INSERT and UPDATE statements are the +** same as a SELECT with only a single table in the FROM clause.) For +** example, if the SQL is this: +** +** SELECT * FROM t1, t2, t3 WHERE ...; +** +** Then the code generated is conceptually like the following: +** +** foreach row1 in t1 do \ Code generated +** foreach row2 in t2 do |-- by sqlite3WhereBegin() +** foreach row3 in t3 do / +** ... +** end \ Code generated +** end |-- by sqlite3WhereEnd() +** end / +** +** Note that the loops might not be nested in the order in which they +** appear in the FROM clause if a different order is better able to make +** use of indices. Note also that when the IN operator appears in +** the WHERE clause, it might result in additional nested loops for +** scanning through all values on the right-hand side of the IN. +** +** There are Btree cursors associated with each table. t1 uses cursor +** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor. +** And so forth. This routine generates code to open those VDBE cursors +** and sqlite3WhereEnd() generates the code to close them. +** +** The code that sqlite3WhereBegin() generates leaves the cursors named +** in pTabList pointing at their appropriate entries. The [...] code +** can use OP_Column and OP_Rowid opcodes on these cursors to extract +** data from the various tables of the loop. +** +** If the WHERE clause is empty, the foreach loops must each scan their +** entire tables. Thus a three-way join is an O(N^3) operation. But if +** the tables have indices and there are terms in the WHERE clause that +** refer to those indices, a complete table scan can be avoided and the +** code will run much faster. Most of the work of this routine is checking +** to see if there are indices that can be used to speed up the loop. +** +** Terms of the WHERE clause are also used to limit which rows actually +** make it to the "..." in the middle of the loop. After each "foreach", +** terms of the WHERE clause that use only terms in that loop and outer +** loops are evaluated and if false a jump is made around all subsequent +** inner loops (or around the "..." if the test occurs within the inner- +** most loop) +** +** OUTER JOINS +** +** An outer join of tables t1 and t2 is conceptally coded as follows: +** +** foreach row1 in t1 do +** flag = 0 +** foreach row2 in t2 do +** start: +** ... +** flag = 1 +** end +** if flag==0 then +** move the row2 cursor to a null row +** goto start +** fi +** end +** +** ORDER BY CLAUSE PROCESSING +** +** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause +** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement +** if there is one. If there is no ORDER BY clause or if this routine +** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. +** +** The iIdxCur parameter is the cursor number of an index. If +** WHERE_ONETABLE_ONLY is set, iIdxCur is the cursor number of an index +** to use for OR clause processing. The WHERE clause should use this +** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is +** the first cursor in an array of cursors for all indices. iIdxCur should +** be used to compute the appropriate cursor depending on which index is +** used. */ -static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ - const int nHint = pHint->n; - int i; +SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ + Expr *pWhere, /* The WHERE clause */ + ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ + ExprList *pResultSet, /* Result set of the query */ + u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ + int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ +){ + int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ + int nTabList; /* Number of elements in pTabList */ + WhereInfo *pWInfo; /* Will become the return value of this function */ + Vdbe *v = pParse->pVdbe; /* The virtual database engine */ + Bitmask notReady; /* Cursors that are not yet positioned */ + WhereLoopBuilder sWLB; /* The WhereLoop builder */ + WhereMaskSet *pMaskSet; /* The expression mask set */ + WhereLevel *pLevel; /* A single level in pWInfo->a[] */ + WhereLoop *pLoop; /* Pointer to a single WhereLoop object */ + int ii; /* Loop counter */ + sqlite3 *db; /* Database connection */ + int rc; /* Return code */ - i = pHint->n-2; - while( i>0 && (pHint->a[i-1] & 0x80) ) i--; - while( i>0 && (pHint->a[i-1] & 0x80) ) i--; - pHint->n = i; - i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); - i += fts3GetVarint32(&pHint->a[i], pnInput); - if( i!=nHint ) return FTS_CORRUPT_VTAB; + /* Variable initialization */ + db = pParse->db; + memset(&sWLB, 0, sizeof(sWLB)); - return SQLITE_OK; -} + /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ + testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); + if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; + sWLB.pOrderBy = pOrderBy; + /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via + ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ + if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ + wctrlFlags &= ~WHERE_WANT_DISTINCT; + } -/* -** Attempt an incremental merge that writes nMerge leaf blocks. -** -** Incremental merges happen nMin segments at a time. The segments -** to be merged are the nMin oldest segments (the ones with the smallest -** values for the _segdir.idx field) in the highest level that contains -** at least nMin segments. Multiple merges might occur in an attempt to -** write the quota of nMerge leaf blocks. -*/ -SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ - int rc; /* Return code */ - int nRem = nMerge; /* Number of leaf pages yet to be written */ - Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ - Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ - IncrmergeWriter *pWriter; /* Writer object */ - int nSeg = 0; /* Number of input segments */ - sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ - Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ - int bDirtyHint = 0; /* True if blob 'hint' has been modified */ + /* The number of tables in the FROM clause is limited by the number of + ** bits in a Bitmask + */ + testcase( pTabList->nSrc==BMS ); + if( pTabList->nSrc>BMS ){ + sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS); + return 0; + } - /* Allocate space for the cursor, filter and writer objects */ - const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); - pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); - if( !pWriter ) return SQLITE_NOMEM; - pFilter = (Fts3SegFilter *)&pWriter[1]; - pCsr = (Fts3MultiSegReader *)&pFilter[1]; + /* This function normally generates a nested loop for all tables in + ** pTabList. But if the WHERE_ONETABLE_ONLY flag is set, then we should + ** only generate code for the first table in pTabList and assume that + ** any cursors associated with subsequent tables are uninitialized. + */ + nTabList = (wctrlFlags & WHERE_ONETABLE_ONLY) ? 1 : pTabList->nSrc; - rc = fts3IncrmergeHintLoad(p, &hint); - while( rc==SQLITE_OK && nRem>0 ){ - const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; - sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ - int bUseHint = 0; /* True if attempting to append */ - int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ + /* Allocate and initialize the WhereInfo structure that will become the + ** return value. A single allocation is used to store the WhereInfo + ** struct, the contents of WhereInfo.a[], the WhereClause structure + ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte + ** field (type Bitmask) it must be aligned on an 8-byte boundary on + ** some architectures. Hence the ROUND8() below. + */ + nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); + pWInfo = sqlite3DbMallocZero(db, nByteWInfo + sizeof(WhereLoop)); + if( db->mallocFailed ){ + sqlite3DbFree(db, pWInfo); + pWInfo = 0; + goto whereBeginError; + } + pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; + pWInfo->nLevel = nTabList; + pWInfo->pParse = pParse; + pWInfo->pTabList = pTabList; + pWInfo->pOrderBy = pOrderBy; + pWInfo->pResultSet = pResultSet; + pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); + pWInfo->wctrlFlags = wctrlFlags; + pWInfo->savedNQueryLoop = pParse->nQueryLoop; + pMaskSet = &pWInfo->sMaskSet; + sWLB.pWInfo = pWInfo; + sWLB.pWC = &pWInfo->sWC; + sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo); + assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) ); + whereLoopInit(sWLB.pNew); +#ifdef SQLITE_DEBUG + sWLB.pNew->cId = '*'; +#endif - /* Search the %_segdir table for the absolute level with the smallest - ** relative level number that contains at least nMin segments, if any. - ** If one is found, set iAbsLevel to the absolute level number and - ** nSeg to nMin. If no level with at least nMin segments can be found, - ** set nSeg to -1. - */ - rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); - sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); - if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ - iAbsLevel = sqlite3_column_int64(pFindLevel, 0); - nSeg = sqlite3_column_int(pFindLevel, 1); - assert( nSeg>=2 ); - }else{ - nSeg = -1; + /* Split the WHERE clause into separate subexpressions where each + ** subexpression is separated by an AND operator. + */ + initMaskSet(pMaskSet); + whereClauseInit(&pWInfo->sWC, pWInfo); + whereSplit(&pWInfo->sWC, pWhere, TK_AND); + + /* Special case: a WHERE clause that is constant. Evaluate the + ** expression and either jump over all of the code or fall thru. + */ + for(ii=0; iinTerm; ii++){ + if( nTabList==0 || sqlite3ExprIsConstantNotJoin(sWLB.pWC->a[ii].pExpr) ){ + sqlite3ExprIfFalse(pParse, sWLB.pWC->a[ii].pExpr, pWInfo->iBreak, + SQLITE_JUMPIFNULL); + sWLB.pWC->a[ii].wtFlags |= TERM_CODED; } - rc = sqlite3_reset(pFindLevel); + } - /* If the hint read from the %_stat table is not empty, check if the - ** last entry in it specifies a relative level smaller than or equal - ** to the level identified by the block above (if any). If so, this - ** iteration of the loop will work on merging at the hinted level. - */ - if( rc==SQLITE_OK && hint.n ){ - int nHint = hint.n; - sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ - int nHintSeg = 0; /* Hint number of segments */ + /* Special case: No FROM clause + */ + if( nTabList==0 ){ + if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } + } - rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); - if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ - iAbsLevel = iHintAbsLevel; - nSeg = nHintSeg; - bUseHint = 1; - bDirtyHint = 1; - }else{ - /* This undoes the effect of the HintPop() above - so that no entry - ** is removed from the hint blob. */ - hint.n = nHint; - } + /* Assign a bit from the bitmask to every term in the FROM clause. + ** + ** When assigning bitmask values to FROM clause cursors, it must be + ** the case that if X is the bitmask for the N-th FROM clause term then + ** the bitmask for all FROM clause terms to the left of the N-th term + ** is (X-1). An expression from the ON clause of a LEFT JOIN can use + ** its Expr.iRightJoinTable value to find the bitmask of the right table + ** of the join. Subtracting one from the right table bitmask gives a + ** bitmask for all tables to the left of the join. Knowing the bitmask + ** for all tables to the left of a left join is important. Ticket #3015. + ** + ** Note that bitmasks are created for all pTabList->nSrc tables in + ** pTabList, not just the first nTabList tables. nTabList is normally + ** equal to pTabList->nSrc but might be shortened to 1 if the + ** WHERE_ONETABLE_ONLY flag is set. + */ + for(ii=0; iinSrc; ii++){ + createMask(pMaskSet, pTabList->a[ii].iCursor); + } +#ifndef NDEBUG + { + Bitmask toTheLeft = 0; + for(ii=0; iinSrc; ii++){ + Bitmask m = getMask(pMaskSet, pTabList->a[ii].iCursor); + assert( (m-1)==toTheLeft ); + toTheLeft |= m; } + } +#endif - /* If nSeg is less that zero, then there is no level with at least - ** nMin segments and no hint in the %_stat table. No work to do. - ** Exit early in this case. */ - if( nSeg<0 ) break; + /* Analyze all of the subexpressions. Note that exprAnalyze() might + ** add new virtual terms onto the end of the WHERE clause. We do not + ** want to analyze these virtual terms, so start analyzing at the end + ** and work forward so that the added virtual terms are never processed. + */ + exprAnalyzeAll(pTabList, &pWInfo->sWC); + if( db->mallocFailed ){ + goto whereBeginError; + } - /* Open a cursor to iterate through the contents of the oldest nSeg - ** indexes of absolute level iAbsLevel. If this cursor is opened using - ** the 'hint' parameters, it is possible that there are less than nSeg - ** segments available in level iAbsLevel. In this case, no work is - ** done on iAbsLevel - fall through to the next iteration of the loop - ** to start work on some other level. */ - memset(pWriter, 0, nAlloc); - pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ + /* The DISTINCT marking is pointless. Ignore it. */ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + }else if( pOrderBy==0 ){ + /* Try to ORDER BY the result set to make distinct processing easier */ + pWInfo->wctrlFlags |= WHERE_DISTINCTBY; + pWInfo->pOrderBy = pResultSet; + } + } - if( rc==SQLITE_OK ){ - rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); - assert( bUseHint==1 || bUseHint==0 ); - if( iIdx==0 || (bUseHint && iIdx==1) ){ - int bIgnore = 0; - rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); - if( bIgnore ){ - pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; - } - } + /* Construct the WhereLoop objects */ + WHERETRACE(0xffff,("*** Optimizer Start ***\n")); +#if defined(WHERETRACE_ENABLED) + /* Display all terms of the WHERE clause */ + if( sqlite3WhereTrace & 0x100 ){ + int i; + for(i=0; inTerm; i++){ + whereTermPrint(&sWLB.pWC->a[i], i); } + } +#endif - if( rc==SQLITE_OK ){ - rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ + rc = whereLoopAddAll(&sWLB); + if( rc ) goto whereBeginError; + + /* Display all of the WhereLoop objects if wheretrace is enabled */ +#ifdef WHERETRACE_ENABLED /* !=0 */ + if( sqlite3WhereTrace ){ + WhereLoop *p; + int i; + static char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; + for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ + p->cId = zLabel[i%sizeof(zLabel)]; + whereLoopPrint(p, sWLB.pWC); + } } - if( SQLITE_OK==rc && pCsr->nSegment==nSeg - && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) - && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) - ){ - if( bUseHint && iIdx>0 ){ - const char *zKey = pCsr->zTerm; - int nKey = pCsr->nTerm; - rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); - }else{ - rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); +#endif + + wherePathSolver(pWInfo, 0); + if( db->mallocFailed ) goto whereBeginError; + if( pWInfo->pOrderBy ){ + wherePathSolver(pWInfo, pWInfo->nRowOut+1); + if( db->mallocFailed ) goto whereBeginError; + } + } + if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ + pWInfo->revMask = (Bitmask)(-1); + } + if( pParse->nErr || NEVER(db->mallocFailed) ){ + goto whereBeginError; + } +#ifdef WHERETRACE_ENABLED /* !=0 */ + if( sqlite3WhereTrace ){ + sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); + if( pWInfo->nOBSat>0 ){ + sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); + } + switch( pWInfo->eDistinct ){ + case WHERE_DISTINCT_UNIQUE: { + sqlite3DebugPrintf(" DISTINCT=unique"); + break; } - - if( rc==SQLITE_OK && pWriter->nLeafEst ){ - fts3LogMerge(nSeg, iAbsLevel); - do { - rc = fts3IncrmergeAppend(p, pWriter, pCsr); - if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); - if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; - }while( rc==SQLITE_ROW ); - - /* Update or delete the input segments */ - if( rc==SQLITE_OK ){ - nRem -= (1 + pWriter->nWork); - rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); - if( nSeg!=0 ){ - bDirtyHint = 1; - fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); - } - } + case WHERE_DISTINCT_ORDERED: { + sqlite3DebugPrintf(" DISTINCT=ordered"); + break; } - - if( nSeg!=0 ){ - pWriter->nLeafData = pWriter->nLeafData * -1; + case WHERE_DISTINCT_UNORDERED: { + sqlite3DebugPrintf(" DISTINCT=unordered"); + break; } - fts3IncrmergeRelease(p, pWriter, &rc); - if( nSeg==0 && pWriter->bNoLeafData==0 ){ - fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } + sqlite3DebugPrintf("\n"); + for(ii=0; iinLevel; ii++){ + whereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC); + } + } +#endif + /* Attempt to omit tables from the join that do not effect the result */ + if( pWInfo->nLevel>=2 + && pResultSet!=0 + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + ){ + Bitmask tabUsed = exprListTableUsage(pMaskSet, pResultSet); + if( sWLB.pOrderBy ) tabUsed |= exprListTableUsage(pMaskSet, sWLB.pOrderBy); + while( pWInfo->nLevel>=2 ){ + WhereTerm *pTerm, *pEnd; + pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; + if( (pWInfo->pTabList->a[pLoop->iTab].jointype & JT_LEFT)==0 ) break; + if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 + && (pLoop->wsFlags & WHERE_ONEROW)==0 + ){ + break; + } + if( (tabUsed & pLoop->maskSelf)!=0 ) break; + pEnd = sWLB.pWC->a + sWLB.pWC->nTerm; + for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + ){ + break; + } } + if( pTerm drop loop %c not used\n", pLoop->cId)); + pWInfo->nLevel--; + nTabList--; } - - sqlite3Fts3SegReaderFinish(pCsr); } + WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); + pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; - /* Write the hint values into the %_stat table for the next incr-merger */ - if( bDirtyHint && rc==SQLITE_OK ){ - rc = fts3IncrmergeHintStore(p, &hint); + /* If the caller is an UPDATE or DELETE statement that is requesting + ** to use a one-pass algorithm, determine if this is appropriate. + ** The one-pass algorithm only works if the WHERE clause constrains + ** the statement to update a single row. + */ + assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (pWInfo->a[0].pWLoop->wsFlags & WHERE_ONEROW)!=0 ){ + pWInfo->okOnePass = 1; + if( HasRowid(pTabList->a[0].pTab) ){ + pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + } } - sqlite3_free(pWriter); - sqlite3_free(hint.a); - return rc; -} - -/* -** Convert the text beginning at *pz into an integer and return -** its value. Advance *pz to point to the first character past -** the integer. -*/ -static int fts3Getint(const char **pz){ - const char *z = *pz; - int i = 0; - while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; - *pz = z; - return i; -} - -/* -** Process statements of the form: -** -** INSERT INTO table(table) VALUES('merge=A,B'); -** -** A and B are integers that decode to be the number of leaf pages -** written for the merge, and the minimum number of segments on a level -** before it will be selected for a merge, respectively. -*/ -static int fts3DoIncrmerge( - Fts3Table *p, /* FTS3 table handle */ - const char *zParam /* Nul-terminated string containing "A,B" */ -){ - int rc; - int nMin = (FTS3_MERGE_COUNT / 2); - int nMerge = 0; - const char *z = zParam; - - /* Read the first integer value */ - nMerge = fts3Getint(&z); + /* Open all tables in the pTabList and any indices selected for + ** searching those tables. + */ + notReady = ~(Bitmask)0; + for(ii=0, pLevel=pWInfo->a; iia[pLevel->iFrom]; + pTab = pTabItem->pTab; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pLoop = pLevel->pWLoop; + if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ + /* Do nothing */ + }else +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + int iCur = pTabItem->iCursor; + sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); + }else if( IsVirtual(pTab) ){ + /* noop */ + }else +#endif + if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 + && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ + int op = OP_OpenRead; + if( pWInfo->okOnePass ){ + op = OP_OpenWrite; + pWInfo->aiCurOnePass[0] = pTabItem->iCursor; + }; + sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); + assert( pTabItem->iCursor==pLevel->iTabCur ); + testcase( !pWInfo->okOnePass && pTab->nCol==BMS-1 ); + testcase( !pWInfo->okOnePass && pTab->nCol==BMS ); + if( !pWInfo->okOnePass && pTab->nColcolUsed; + int n = 0; + for(; b; b=b>>1, n++){} + sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, + SQLITE_INT_TO_PTR(n), P4_INT32); + assert( n<=pTab->nCol ); + } + }else{ + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + } + if( pLoop->wsFlags & WHERE_INDEXED ){ + Index *pIx = pLoop->u.btree.pIndex; + int iIndexCur; + int op = OP_OpenRead; + /* iIdxCur is always set if to a positive value if ONEPASS is possible */ + assert( iIdxCur!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) + && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 + ){ + /* This is one term of an OR-optimization using the PRIMARY KEY of a + ** WITHOUT ROWID table. No need for a separate index */ + iIndexCur = pLevel->iTabCur; + op = 0; + }else if( pWInfo->okOnePass ){ + Index *pJ = pTabItem->pTab->pIndex; + iIndexCur = iIdxCur; + assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); + while( ALWAYS(pJ) && pJ!=pIx ){ + iIndexCur++; + pJ = pJ->pNext; + } + op = OP_OpenWrite; + pWInfo->aiCurOnePass[1] = iIndexCur; + }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ + iIndexCur = iIdxCur; + if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx; + }else{ + iIndexCur = pParse->nTab++; + } + pLevel->iIdxCur = iIndexCur; + assert( pIx->pSchema==pTab->pSchema ); + assert( iIndexCur>=0 ); + if( op ){ + sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIx); + if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 + && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 + ){ + sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */ + } + VdbeComment((v, "%s", pIx->zName)); + } + } + if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); + notReady &= ~getMask(&pWInfo->sMaskSet, pTabItem->iCursor); } + pWInfo->iTop = sqlite3VdbeCurrentAddr(v); + if( db->mallocFailed ) goto whereBeginError; - if( z[0]!='\0' || nMin<2 ){ - rc = SQLITE_ERROR; - }else{ - rc = SQLITE_OK; - if( !p->bHasStat ){ - assert( p->bFts4==0 ); - sqlite3Fts3CreateStatTable(&rc, p); + /* Generate the code to do the search. Each iteration of the for + ** loop below generates code for a single nested loop of the VM + ** program. + */ + notReady = ~(Bitmask)0; + for(ii=0; iia[ii]; + wsFlags = pLevel->pWLoop->wsFlags; +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX + if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ + constructAutomaticIndex(pParse, &pWInfo->sWC, + &pTabList->a[pLevel->iFrom], notReady, pLevel); + if( db->mallocFailed ) goto whereBeginError; } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); +#endif + addrExplain = explainOneScan( + pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags + ); + pLevel->addrBody = sqlite3VdbeCurrentAddr(v); + notReady = codeOneLoopStart(pWInfo, ii, notReady); + pWInfo->iContinue = pLevel->addrCont; + if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_ONETABLE_ONLY)==0 ){ + addScanStatus(v, pTabList, pLevel, addrExplain); } - sqlite3Fts3SegmentsClose(p); } - return rc; -} -/* -** Process statements of the form: -** -** INSERT INTO table(table) VALUES('automerge=X'); -** -** where X is an integer. X==0 means to turn automerge off. X!=0 means -** turn it on. The setting is persistent. -*/ -static int fts3DoAutoincrmerge( - Fts3Table *p, /* FTS3 table handle */ - const char *zParam /* Nul-terminated string containing boolean */ -){ - int rc = SQLITE_OK; - sqlite3_stmt *pStmt = 0; - p->nAutoincrmerge = fts3Getint(&zParam); - if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ - p->nAutoincrmerge = 8; - } - if( !p->bHasStat ){ - assert( p->bFts4==0 ); - sqlite3Fts3CreateStatTable(&rc, p); - if( rc ) return rc; + /* Done. */ + VdbeModuleComment((v, "Begin WHERE-core")); + return pWInfo; + + /* Jump here if malloc fails */ +whereBeginError: + if( pWInfo ){ + pParse->nQueryLoop = pWInfo->savedNQueryLoop; + whereInfoFree(db, pWInfo); } - rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); - if( rc ) return rc; - sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); - sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - return rc; + return 0; } /* -** Return a 64-bit checksum for the FTS index entry specified by the -** arguments to this function. +** Generate the end of the WHERE loop. See comments on +** sqlite3WhereBegin() for additional information. */ -static u64 fts3ChecksumEntry( - const char *zTerm, /* Pointer to buffer containing term */ - int nTerm, /* Size of zTerm in bytes */ - int iLangid, /* Language id for current row */ - int iIndex, /* Index (0..Fts3Table.nIndex-1) */ - i64 iDocid, /* Docid for current row. */ - int iCol, /* Column number */ - int iPos /* Position */ -){ +SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ + Parse *pParse = pWInfo->pParse; + Vdbe *v = pParse->pVdbe; int i; - u64 ret = (u64)iDocid; - - ret += (ret<<3) + iLangid; - ret += (ret<<3) + iIndex; - ret += (ret<<3) + iCol; - ret += (ret<<3) + iPos; - for(i=0; inIndex-1) */ - int *pRc /* OUT: Return code */ -){ - Fts3SegFilter filter; - Fts3MultiSegReader csr; - int rc; - u64 cksum = 0; - - assert( *pRc==SQLITE_OK ); - - memset(&filter, 0, sizeof(filter)); - memset(&csr, 0, sizeof(csr)); - filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; - filter.flags |= FTS3_SEGMENT_SCAN; + WhereLevel *pLevel; + WhereLoop *pLoop; + SrcList *pTabList = pWInfo->pTabList; + sqlite3 *db = pParse->db; - rc = sqlite3Fts3SegReaderCursor( - p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr - ); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + /* Generate loop termination code. + */ + VdbeModuleComment((v, "End WHERE-core")); + sqlite3ExprCacheClear(pParse); + for(i=pWInfo->nLevel-1; i>=0; i--){ + int addr; + pLevel = &pWInfo->a[i]; + pLoop = pLevel->pWLoop; + sqlite3VdbeResolveLabel(v, pLevel->addrCont); + if( pLevel->op!=OP_Noop ){ + sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); + sqlite3VdbeChangeP5(v, pLevel->p5); + VdbeCoverage(v); + VdbeCoverageIf(v, pLevel->op==OP_Next); + VdbeCoverageIf(v, pLevel->op==OP_Prev); + VdbeCoverageIf(v, pLevel->op==OP_VNext); + } + if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ + struct InLoop *pIn; + int j; + sqlite3VdbeResolveLabel(v, pLevel->addrNxt); + for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ + sqlite3VdbeJumpHere(v, pIn->addrInTop+1); + sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); + VdbeCoverage(v); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); + sqlite3VdbeJumpHere(v, pIn->addrInTop-1); + } + } + sqlite3VdbeResolveLabel(v, pLevel->addrBrk); + if( pLevel->addrSkip ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); + VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); + sqlite3VdbeJumpHere(v, pLevel->addrSkip); + sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); + } + if( pLevel->addrLikeRep ){ + int op; + if( sqlite3VdbeGetOp(v, pLevel->addrLikeRep-1)->p1 ){ + op = OP_DecrJumpZero; + }else{ + op = OP_JumpZeroIncr; + } + sqlite3VdbeAddOp2(v, op, pLevel->iLikeRepCntr, pLevel->addrLikeRep); + VdbeCoverage(v); + } + if( pLevel->iLeftJoin ){ + addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 + || (pLoop->wsFlags & WHERE_INDEXED)!=0 ); + if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ){ + sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); + } + if( pLoop->wsFlags & WHERE_INDEXED ){ + sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); + } + if( pLevel->op==OP_Return ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrFirst); + } + sqlite3VdbeJumpHere(v, addr); + } + VdbeModuleComment((v, "End WHERE-loop%d: %s", i, + pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); } - if( rc==SQLITE_OK ){ - while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ - char *pCsr = csr.aDoclist; - char *pEnd = &pCsr[csr.nDoclist]; + /* The "break" point is here, just past the end of the outer loop. + ** Set it. + */ + sqlite3VdbeResolveLabel(v, pWInfo->iBreak); - i64 iDocid = 0; - i64 iCol = 0; - i64 iPos = 0; + assert( pWInfo->nLevel<=pTabList->nSrc ); + for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ + int k, last; + VdbeOp *pOp; + Index *pIdx = 0; + struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom]; + Table *pTab = pTabItem->pTab; + assert( pTab!=0 ); + pLoop = pLevel->pWLoop; - pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); - while( pCsrviaCoroutine && !db->mallocFailed ){ + last = sqlite3VdbeCurrentAddr(v); + k = pLevel->addrBody; + pOp = sqlite3VdbeGetOp(v, k); + for(; kp1!=pLevel->iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + pOp->opcode = OP_Copy; + pOp->p1 = pOp->p2 + pTabItem->regResult; + pOp->p2 = pOp->p3; + pOp->p3 = 0; + }else if( pOp->opcode==OP_Rowid ){ + pOp->opcode = OP_Null; + pOp->p1 = 0; + pOp->p3 = 0; } } + continue; } - } - sqlite3Fts3SegReaderFinish(&csr); - - *pRc = rc; - return cksum; -} - -/* -** Check if the contents of the FTS index match the current contents of the -** content table. If no error occurs and the contents do match, set *pbOk -** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk -** to false before returning. -** -** If an error occurs (e.g. an OOM or IO error), return an SQLite error -** code. The final value of *pbOk is undefined in this case. -*/ -static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ - int rc = SQLITE_OK; /* Return code */ - u64 cksum1 = 0; /* Checksum based on FTS index contents */ - u64 cksum2 = 0; /* Checksum based on %_content contents */ - sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ - /* This block calculates the checksum according to the FTS index. */ - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); - if( rc==SQLITE_OK ){ - int rc2; - sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); - sqlite3_bind_int(pAllLangid, 2, p->nIndex); - while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ - int iLangid = sqlite3_column_int(pAllLangid, 0); - int i; - for(i=0; inIndex; i++){ - cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + /* Close all of the cursors that were opened by sqlite3WhereBegin. + ** Except, do not close cursors that will be reused by the OR optimization + ** (WHERE_OMIT_OPEN_CLOSE). And do not close the OP_OpenWrite cursors + ** created for the ONEPASS optimization. + */ + if( (pTab->tabFlags & TF_Ephemeral)==0 + && pTab->pSelect==0 + && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 + ){ + int ws = pLoop->wsFlags; + if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ + sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); + } + if( (ws & WHERE_INDEXED)!=0 + && (ws & (WHERE_IPK|WHERE_AUTO_INDEX))==0 + && pLevel->iIdxCur!=pWInfo->aiCurOnePass[1] + ){ + sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); } } - rc2 = sqlite3_reset(pAllLangid); - if( rc==SQLITE_OK ) rc = rc2; - } - /* This block calculates the checksum according to the %_content table */ - if( rc==SQLITE_OK ){ - sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; - sqlite3_stmt *pStmt = 0; - char *zSql; - - zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); + /* If this scan uses an index, make VDBE code substitutions to read data + ** from the index instead of from the table where possible. In some cases + ** this optimization prevents the table from ever being read, which can + ** yield a significant performance boost. + ** + ** Calls to the code generator in between sqlite3WhereBegin and + ** sqlite3WhereEnd will have created code that references the table + ** directly. This loop scans all that code looking for opcodes + ** that reference the table and converts them into opcodes that + ** reference the index. + */ + if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){ + pIdx = pLoop->u.btree.pIndex; + }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ + pIdx = pLevel->u.pCovidx; } - - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - i64 iDocid = sqlite3_column_int64(pStmt, 0); - int iLang = langidFromSelect(p, pStmt); - int iCol; - - for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ - if( p->abNotindexed[iCol]==0 ){ - const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); - int nText = sqlite3_column_bytes(pStmt, iCol+1); - sqlite3_tokenizer_cursor *pT = 0; - - rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT); - while( rc==SQLITE_OK ){ - char const *zToken; /* Buffer containing token */ - int nToken = 0; /* Number of bytes in token */ - int iDum1 = 0, iDum2 = 0; /* Dummy variables */ - int iPos = 0; /* Position of token in zText */ - - rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); - if( rc==SQLITE_OK ){ - int i; - cksum2 = cksum2 ^ fts3ChecksumEntry( - zToken, nToken, iLang, 0, iDocid, iCol, iPos - ); - for(i=1; inIndex; i++){ - if( p->aIndex[i].nPrefix<=nToken ){ - cksum2 = cksum2 ^ fts3ChecksumEntry( - zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos - ); - } - } - } + if( pIdx && !db->mallocFailed ){ + last = sqlite3VdbeCurrentAddr(v); + k = pLevel->addrBody; + pOp = sqlite3VdbeGetOp(v, k); + for(; kp1!=pLevel->iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + int x = pOp->p2; + assert( pIdx->pTable==pTab ); + if( !HasRowid(pTab) ){ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + x = pPk->aiColumn[x]; } - if( pT ) pModule->xClose(pT); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; + x = sqlite3ColumnOfIndex(pIdx, x); + if( x>=0 ){ + pOp->p2 = x; + pOp->p1 = pLevel->iIdxCur; + } + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 ); + }else if( pOp->opcode==OP_Rowid ){ + pOp->p1 = pLevel->iIdxCur; + pOp->opcode = OP_IdxRowid; } } } - - sqlite3_finalize(pStmt); } - *pbOk = (cksum1==cksum2); - return rc; + /* Final cleanup + */ + pParse->nQueryLoop = pWInfo->savedNQueryLoop; + whereInfoFree(db, pWInfo); + return; } -/* -** Run the integrity-check. If no error occurs and the current contents of -** the FTS index are correct, return SQLITE_OK. Or, if the contents of the -** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. -** -** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite -** error code. -** -** The integrity-check works as follows. For each token and indexed token -** prefix in the document set, a 64-bit checksum is calculated (by code -** in fts3ChecksumEntry()) based on the following: -** -** + The index number (0 for the main index, 1 for the first prefix -** index etc.), -** + The token (or token prefix) text itself, -** + The language-id of the row it appears in, -** + The docid of the row it appears in, -** + The column it appears in, and -** + The tokens position within that column. -** -** The checksums for all entries in the index are XORed together to create -** a single checksum for the entire index. -** -** The integrity-check code calculates the same checksum in two ways: -** -** 1. By scanning the contents of the FTS index, and -** 2. By scanning and tokenizing the content table. +/************** End of where.c ***********************************************/ +/************** Begin file parse.c *******************************************/ +/* Driver template for the LEMON parser generator. +** The author disclaims copyright to this source code. ** -** If the two checksums are identical, the integrity-check is deemed to have -** passed. +** This version of "lempar.c" is modified, slightly, for use by SQLite. +** The only modifications are the addition of a couple of NEVER() +** macros to disable tests that are needed in the case of a general +** LALR(1) grammar but which are always false in the +** specific grammar used by SQLite. */ -static int fts3DoIntegrityCheck( - Fts3Table *p /* FTS3 table handle */ -){ - int rc; - int bOk = 0; - rc = fts3IntegrityCheck(p, &bOk); - if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; - return rc; -} +/* First off, code is included that follows the "include" declaration +** in the input grammar file. */ +/* #include */ + /* -** Handle a 'special' INSERT of the form: -** -** "INSERT INTO tbl(tbl) VALUES()" -** -** Argument pVal contains the result of . Currently the only -** meaningful value to insert is the text 'optimize'. +** Disable all error recovery processing in the parser push-down +** automaton. */ -static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ - int rc; /* Return Code */ - const char *zVal = (const char *)sqlite3_value_text(pVal); - int nVal = sqlite3_value_bytes(pVal); - - if( !zVal ){ - return SQLITE_NOMEM; - }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ - rc = fts3DoOptimize(p, 0); - }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ - rc = fts3DoRebuild(p); - }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ - rc = fts3DoIntegrityCheck(p); - }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ - rc = fts3DoIncrmerge(p, &zVal[6]); - }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ - rc = fts3DoAutoincrmerge(p, &zVal[10]); -#ifdef SQLITE_TEST - }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ - p->nNodeSize = atoi(&zVal[9]); - rc = SQLITE_OK; - }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ - p->nMaxPendingData = atoi(&zVal[11]); - rc = SQLITE_OK; - }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){ - p->bNoIncrDoclist = atoi(&zVal[21]); - rc = SQLITE_OK; -#endif - }else{ - rc = SQLITE_ERROR; - } +#define YYNOERRORRECOVERY 1 - return rc; -} +/* +** Make yytestcase() the same as testcase() +*/ +#define yytestcase(X) testcase(X) -#ifndef SQLITE_DISABLE_FTS4_DEFERRED /* -** Delete all cached deferred doclists. Deferred doclists are cached -** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. +** An instance of this structure holds information about the +** LIMIT clause of a SELECT statement. */ -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ - Fts3DeferredToken *pDef; - for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ - fts3PendingListDelete(pDef->pList); - pDef->pList = 0; - } -} +struct LimitVal { + Expr *pLimit; /* The LIMIT expression. NULL if there is no limit */ + Expr *pOffset; /* The OFFSET expression. NULL if there is none */ +}; /* -** Free all entries in the pCsr->pDeffered list. Entries are added to -** this list using sqlite3Fts3DeferToken(). +** An instance of this structure is used to store the LIKE, +** GLOB, NOT LIKE, and NOT GLOB operators. */ -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ - Fts3DeferredToken *pDef; - Fts3DeferredToken *pNext; - for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ - pNext = pDef->pNext; - fts3PendingListDelete(pDef->pList); - sqlite3_free(pDef); - } - pCsr->pDeferred = 0; -} +struct LikeOp { + Token eOperator; /* "like" or "glob" or "regexp" */ + int bNot; /* True if the NOT keyword is present */ +}; /* -** Generate deferred-doclists for all tokens in the pCsr->pDeferred list -** based on the row that pCsr currently points to. +** An instance of the following structure describes the event of a +** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, +** TK_DELETE, or TK_INSTEAD. If the event is of the form ** -** A deferred-doclist is like any other doclist with position information -** included, except that it only contains entries for a single row of the -** table, not for all rows. +** UPDATE ON (a,b,c) +** +** Then the "b" IdList records the list "a,b,c". */ -SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ - int rc = SQLITE_OK; /* Return code */ - if( pCsr->pDeferred ){ - int i; /* Used to iterate through table columns */ - sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ - Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ - - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - sqlite3_tokenizer *pT = p->pTokenizer; - sqlite3_tokenizer_module const *pModule = pT->pModule; - - assert( pCsr->isRequireSeek==0 ); - iDocid = sqlite3_column_int64(pCsr->pStmt, 0); - - for(i=0; inColumn && rc==SQLITE_OK; i++){ - if( p->abNotindexed[i]==0 ){ - const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); - sqlite3_tokenizer_cursor *pTC = 0; +struct TrigEvent { int a; IdList * b; }; - rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); - while( rc==SQLITE_OK ){ - char const *zToken; /* Buffer containing token */ - int nToken = 0; /* Number of bytes in token */ - int iDum1 = 0, iDum2 = 0; /* Dummy variables */ - int iPos = 0; /* Position of token in zText */ +/* +** An instance of this structure holds the ATTACH key and the key type. +*/ +struct AttachKey { int type; Token key; }; - rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); - for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ - Fts3PhraseToken *pPT = pDef->pToken; - if( (pDef->iCol>=p->nColumn || pDef->iCol==i) - && (pPT->bFirst==0 || iPos==0) - && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) - ){ - fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); - } - } - } - if( pTC ) pModule->xClose(pTC); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; - } - } - for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ - if( pDef->pList ){ - rc = fts3PendingListAppendVarint(&pDef->pList, 0); + /* + ** For a compound SELECT statement, make sure p->pPrior->pNext==p for + ** all elements in the list. And make sure list length does not exceed + ** SQLITE_LIMIT_COMPOUND_SELECT. + */ + static void parserDoubleLinkSelect(Parse *pParse, Select *p){ + if( p->pPrior ){ + Select *pNext = 0, *pLoop; + int mxSelect, cnt = 0; + for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ + pLoop->pNext = pNext; + pLoop->selFlags |= SF_Compound; + } + if( (p->selFlags & SF_MultiValue)==0 && + (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && + cnt>mxSelect + ){ + sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); } } } - return rc; -} - -SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( - Fts3DeferredToken *p, - char **ppData, - int *pnData -){ - char *pRet; - int nSkip; - sqlite3_int64 dummy; - - *ppData = 0; - *pnData = 0; + /* This is a utility routine used to set the ExprSpan.zStart and + ** ExprSpan.zEnd values of pOut so that the span covers the complete + ** range of text beginning with pStart and going to the end of pEnd. + */ + static void spanSet(ExprSpan *pOut, Token *pStart, Token *pEnd){ + pOut->zStart = pStart->z; + pOut->zEnd = &pEnd->z[pEnd->n]; + } - if( p->pList==0 ){ - return SQLITE_OK; + /* Construct a new Expr object from a single identifier. Use the + ** new Expr to populate pOut. Set the span of pOut to be the identifier + ** that created the expression. + */ + static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token *pValue){ + pOut->pExpr = sqlite3PExpr(pParse, op, 0, 0, pValue); + pOut->zStart = pValue->z; + pOut->zEnd = &pValue->z[pValue->n]; } - pRet = (char *)sqlite3_malloc(p->pList->nData); - if( !pRet ) return SQLITE_NOMEM; + /* This routine constructs a binary expression node out of two ExprSpan + ** objects and uses the result to populate a new ExprSpan object. + */ + static void spanBinaryExpr( + ExprSpan *pOut, /* Write the result here */ + Parse *pParse, /* The parsing context. Errors accumulate here */ + int op, /* The binary operation */ + ExprSpan *pLeft, /* The left operand */ + ExprSpan *pRight /* The right operand */ + ){ + pOut->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); + pOut->zStart = pLeft->zStart; + pOut->zEnd = pRight->zEnd; + } - nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); - *pnData = p->pList->nData - nSkip; - *ppData = pRet; - - memcpy(pRet, &p->pList->aData[nSkip], *pnData); - return SQLITE_OK; -} + /* Construct an expression node for a unary postfix operator + */ + static void spanUnaryPostfix( + ExprSpan *pOut, /* Write the new expression node here */ + Parse *pParse, /* Parsing context to record errors */ + int op, /* The operator */ + ExprSpan *pOperand, /* The operand */ + Token *pPostOp /* The operand token for setting the span */ + ){ + pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); + pOut->zStart = pOperand->zStart; + pOut->zEnd = &pPostOp->z[pPostOp->n]; + } -/* -** Add an entry for token pToken to the pCsr->pDeferred list. -*/ -SQLITE_PRIVATE int sqlite3Fts3DeferToken( - Fts3Cursor *pCsr, /* Fts3 table cursor */ - Fts3PhraseToken *pToken, /* Token to defer */ - int iCol /* Column that token must appear in (or -1) */ -){ - Fts3DeferredToken *pDeferred; - pDeferred = sqlite3_malloc(sizeof(*pDeferred)); - if( !pDeferred ){ - return SQLITE_NOMEM; + /* A routine to convert a binary TK_IS or TK_ISNOT expression into a + ** unary TK_ISNULL or TK_NOTNULL expression. */ + static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ + sqlite3 *db = pParse->db; + if( pY && pA && pY->op==TK_NULL ){ + pA->op = (u8)op; + sqlite3ExprDelete(db, pA->pRight); + pA->pRight = 0; + } } - memset(pDeferred, 0, sizeof(*pDeferred)); - pDeferred->pToken = pToken; - pDeferred->pNext = pCsr->pDeferred; - pDeferred->iCol = iCol; - pCsr->pDeferred = pDeferred; - - assert( pToken->pDeferred==0 ); - pToken->pDeferred = pDeferred; - return SQLITE_OK; -} + /* Construct an expression node for a unary prefix operator + */ + static void spanUnaryPrefix( + ExprSpan *pOut, /* Write the new expression node here */ + Parse *pParse, /* Parsing context to record errors */ + int op, /* The operator */ + ExprSpan *pOperand, /* The operand */ + Token *pPreOp /* The operand token for setting the span */ + ){ + pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); + pOut->zStart = pPreOp->z; + pOut->zEnd = pOperand->zEnd; + } +/* Next is all token values, in a form suitable for use by makeheaders. +** This section will be null unless lemon is run with the -m switch. +*/ +/* +** These constants (all generated automatically by the parser generator) +** specify the various kinds of tokens (terminals) that the parser +** understands. +** +** Each symbol here is a terminal symbol in the grammar. +*/ +/* Make sure the INTERFACE macro is defined. +*/ +#ifndef INTERFACE +# define INTERFACE 1 #endif - -/* -** SQLite value pRowid contains the rowid of a row that may or may not be -** present in the FTS3 table. If it is, delete it and adjust the contents -** of subsiduary data structures accordingly. +/* The next thing included is series of defines which control +** various aspects of the generated parser. +** YYCODETYPE is the data type used for storing terminal +** and nonterminal numbers. "unsigned char" is +** used if there are fewer than 250 terminals +** and nonterminals. "int" is used otherwise. +** YYNOCODE is a number of type YYCODETYPE which corresponds +** to no legal terminal or nonterminal number. This +** number is used to fill in empty slots of the hash +** table. +** YYFALLBACK If defined, this indicates that one or more tokens +** have fall-back values which should be used if the +** original value of the token will not parse. +** YYACTIONTYPE is the data type used for storing terminal +** and nonterminal numbers. "unsigned char" is +** used if there are fewer than 250 rules and +** states combined. "int" is used otherwise. +** sqlite3ParserTOKENTYPE is the data type used for minor tokens given +** directly to the parser from the tokenizer. +** YYMINORTYPE is the data type used for all minor tokens. +** This is typically a union of many types, one of +** which is sqlite3ParserTOKENTYPE. The entry in the union +** for base tokens is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument +** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument +** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser +** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. */ -static int fts3DeleteByRowid( - Fts3Table *p, - sqlite3_value *pRowid, - int *pnChng, /* IN/OUT: Decrement if row is deleted */ - u32 *aSzDel -){ - int rc = SQLITE_OK; /* Return code */ - int bFound = 0; /* True if *pRowid really is in the table */ +#define YYCODETYPE unsigned char +#define YYNOCODE 254 +#define YYACTIONTYPE unsigned short int +#define YYWILDCARD 70 +#define sqlite3ParserTOKENTYPE Token +typedef union { + int yyinit; + sqlite3ParserTOKENTYPE yy0; + Select* yy3; + ExprList* yy14; + With* yy59; + SrcList* yy65; + struct LikeOp yy96; + Expr* yy132; + u8 yy186; + int yy328; + ExprSpan yy346; + struct TrigEvent yy378; + u16 yy381; + IdList* yy408; + struct {int value; int mask;} yy429; + TriggerStep* yy473; + struct LimitVal yy476; +} YYMINORTYPE; +#ifndef YYSTACKDEPTH +#define YYSTACKDEPTH 100 +#endif +#define sqlite3ParserARG_SDECL Parse *pParse; +#define sqlite3ParserARG_PDECL ,Parse *pParse +#define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse +#define sqlite3ParserARG_STORE yypParser->pParse = pParse +#define YYNSTATE 642 +#define YYNRULE 327 +#define YYFALLBACK 1 +#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) +#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) +#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) - fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); - if( bFound && rc==SQLITE_OK ){ - int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ - rc = fts3IsEmpty(p, pRowid, &isEmpty); - if( rc==SQLITE_OK ){ - if( isEmpty ){ - /* Deleting this row means the whole table is empty. In this case - ** delete the contents of all three tables and throw away any - ** data in the pendingTerms hash table. */ - rc = fts3DeleteAll(p, 1); - *pnChng = 0; - memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); - }else{ - *pnChng = *pnChng - 1; - if( p->zContentTbl==0 ){ - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); - } - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); - } - } - } - } +/* The yyzerominor constant is used to initialize instances of +** YYMINORTYPE objects to zero. */ +static const YYMINORTYPE yyzerominor = { 0 }; - return rc; -} +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif -/* -** This function does the work for the xUpdate method of FTS3 virtual -** tables. The schema of the virtual table being: + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. ** -** CREATE TABLE
    ( -** , -**
    HIDDEN, -** docid HIDDEN, -** HIDDEN -** ); +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N < YYNSTATE Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. +** +** N == YYNSTATE+YYNRULE A syntax error has occurred. ** +** N == YYNSTATE+YYNRULE+1 The parser accepts its input. +** +** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused +** slots in the yy_action[] table. +** +** The action table is constructed as a single large table named yy_action[]. +** Given state S and lookahead X, the action is computed as +** +** yy_action[ yy_shift_ofst[S] + X ] +** +** If the index value yy_shift_ofst[S]+X is out of range or if the value +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S] +** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table +** and that yy_default[S] should be used instead. +** +** The formula above is for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the yy_reduce_ofst[] array is used in place of +** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of +** YY_SHIFT_USE_DFLT. +** +** The following are the tables generated in this section: +** +** yy_action[] A single table containing all actions. +** yy_lookahead[] A table containing the lookahead for each entry in +** yy_action. Used to detect hash collisions. +** yy_shift_ofst[] For each state, the offset into yy_action for +** shifting terminals. +** yy_reduce_ofst[] For each state, the offset into yy_action for +** shifting non-terminals after a reduce. +** yy_default[] Default action for each state. +*/ +#define YY_ACTTAB_COUNT (1497) +static const YYACTIONTYPE yy_action[] = { + /* 0 */ 306, 212, 432, 955, 639, 191, 955, 295, 559, 88, + /* 10 */ 88, 88, 88, 81, 86, 86, 86, 86, 85, 85, + /* 20 */ 84, 84, 84, 83, 330, 185, 184, 183, 635, 635, + /* 30 */ 292, 606, 606, 88, 88, 88, 88, 683, 86, 86, + /* 40 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 16, + /* 50 */ 436, 597, 89, 90, 80, 600, 599, 601, 601, 87, + /* 60 */ 87, 88, 88, 88, 88, 684, 86, 86, 86, 86, + /* 70 */ 85, 85, 84, 84, 84, 83, 330, 306, 559, 84, + /* 80 */ 84, 84, 83, 330, 65, 86, 86, 86, 86, 85, + /* 90 */ 85, 84, 84, 84, 83, 330, 635, 635, 634, 633, + /* 100 */ 182, 682, 550, 379, 376, 375, 17, 322, 606, 606, + /* 110 */ 371, 198, 479, 91, 374, 82, 79, 165, 85, 85, + /* 120 */ 84, 84, 84, 83, 330, 598, 635, 635, 107, 89, + /* 130 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, + /* 140 */ 88, 88, 186, 86, 86, 86, 86, 85, 85, 84, + /* 150 */ 84, 84, 83, 330, 306, 594, 594, 142, 328, 327, + /* 160 */ 484, 249, 344, 238, 635, 635, 634, 633, 585, 448, + /* 170 */ 526, 525, 229, 388, 1, 394, 450, 584, 449, 635, + /* 180 */ 635, 635, 635, 319, 395, 606, 606, 199, 157, 273, + /* 190 */ 382, 268, 381, 187, 635, 635, 634, 633, 311, 555, + /* 200 */ 266, 593, 593, 266, 347, 588, 89, 90, 80, 600, + /* 210 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 478, + /* 220 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, + /* 230 */ 330, 306, 272, 536, 634, 633, 146, 610, 197, 310, + /* 240 */ 575, 182, 482, 271, 379, 376, 375, 506, 21, 634, + /* 250 */ 633, 634, 633, 635, 635, 374, 611, 574, 548, 440, + /* 260 */ 111, 563, 606, 606, 634, 633, 324, 479, 608, 608, + /* 270 */ 608, 300, 435, 573, 119, 407, 210, 162, 562, 883, + /* 280 */ 592, 592, 306, 89, 90, 80, 600, 599, 601, 601, + /* 290 */ 87, 87, 88, 88, 88, 88, 506, 86, 86, 86, + /* 300 */ 86, 85, 85, 84, 84, 84, 83, 330, 620, 111, + /* 310 */ 635, 635, 361, 606, 606, 358, 249, 349, 248, 433, + /* 320 */ 243, 479, 586, 634, 633, 195, 611, 93, 119, 221, + /* 330 */ 575, 497, 534, 534, 89, 90, 80, 600, 599, 601, + /* 340 */ 601, 87, 87, 88, 88, 88, 88, 574, 86, 86, + /* 350 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 306, + /* 360 */ 77, 429, 638, 573, 589, 530, 240, 230, 242, 105, + /* 370 */ 249, 349, 248, 515, 588, 208, 460, 529, 564, 173, + /* 380 */ 634, 633, 970, 144, 430, 2, 424, 228, 380, 557, + /* 390 */ 606, 606, 190, 153, 159, 158, 514, 51, 632, 631, + /* 400 */ 630, 71, 536, 432, 954, 196, 610, 954, 614, 45, + /* 410 */ 18, 89, 90, 80, 600, 599, 601, 601, 87, 87, + /* 420 */ 88, 88, 88, 88, 261, 86, 86, 86, 86, 85, + /* 430 */ 85, 84, 84, 84, 83, 330, 306, 608, 608, 608, + /* 440 */ 542, 424, 402, 385, 241, 506, 451, 320, 211, 543, + /* 450 */ 164, 436, 386, 293, 451, 587, 108, 496, 111, 334, + /* 460 */ 391, 591, 424, 614, 27, 452, 453, 606, 606, 72, + /* 470 */ 257, 70, 259, 452, 339, 342, 564, 582, 68, 415, + /* 480 */ 469, 328, 327, 62, 614, 45, 110, 393, 89, 90, + /* 490 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, + /* 500 */ 88, 152, 86, 86, 86, 86, 85, 85, 84, 84, + /* 510 */ 84, 83, 330, 306, 110, 499, 520, 538, 402, 389, + /* 520 */ 424, 110, 566, 500, 593, 593, 454, 82, 79, 165, + /* 530 */ 424, 591, 384, 564, 340, 615, 188, 162, 424, 350, + /* 540 */ 616, 424, 614, 44, 606, 606, 445, 582, 300, 434, + /* 550 */ 151, 19, 614, 9, 568, 580, 348, 615, 469, 567, + /* 560 */ 614, 26, 616, 614, 45, 89, 90, 80, 600, 599, + /* 570 */ 601, 601, 87, 87, 88, 88, 88, 88, 411, 86, + /* 580 */ 86, 86, 86, 85, 85, 84, 84, 84, 83, 330, + /* 590 */ 306, 579, 110, 578, 521, 282, 433, 398, 400, 255, + /* 600 */ 486, 82, 79, 165, 487, 164, 82, 79, 165, 488, + /* 610 */ 488, 364, 387, 424, 544, 544, 509, 350, 362, 155, + /* 620 */ 191, 606, 606, 559, 642, 640, 333, 82, 79, 165, + /* 630 */ 305, 564, 507, 312, 357, 614, 45, 329, 596, 595, + /* 640 */ 194, 337, 89, 90, 80, 600, 599, 601, 601, 87, + /* 650 */ 87, 88, 88, 88, 88, 424, 86, 86, 86, 86, + /* 660 */ 85, 85, 84, 84, 84, 83, 330, 306, 20, 323, + /* 670 */ 150, 263, 211, 543, 421, 596, 595, 614, 22, 424, + /* 680 */ 193, 424, 284, 424, 391, 424, 509, 424, 577, 424, + /* 690 */ 186, 335, 424, 559, 424, 313, 120, 546, 606, 606, + /* 700 */ 67, 614, 47, 614, 50, 614, 48, 614, 100, 614, + /* 710 */ 99, 614, 101, 576, 614, 102, 614, 109, 326, 89, + /* 720 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, + /* 730 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, + /* 740 */ 84, 84, 83, 330, 306, 424, 311, 424, 585, 54, + /* 750 */ 424, 516, 517, 590, 614, 112, 424, 584, 424, 572, + /* 760 */ 424, 195, 424, 571, 424, 67, 424, 614, 94, 614, + /* 770 */ 98, 424, 614, 97, 264, 606, 606, 195, 614, 46, + /* 780 */ 614, 96, 614, 30, 614, 49, 614, 115, 614, 114, + /* 790 */ 418, 229, 388, 614, 113, 306, 89, 90, 80, 600, + /* 800 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 424, + /* 810 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, + /* 820 */ 330, 119, 424, 590, 110, 372, 606, 606, 195, 53, + /* 830 */ 250, 614, 29, 195, 472, 438, 729, 190, 302, 498, + /* 840 */ 14, 523, 641, 2, 614, 43, 306, 89, 90, 80, + /* 850 */ 600, 599, 601, 601, 87, 87, 88, 88, 88, 88, + /* 860 */ 424, 86, 86, 86, 86, 85, 85, 84, 84, 84, + /* 870 */ 83, 330, 424, 613, 964, 964, 354, 606, 606, 420, + /* 880 */ 312, 64, 614, 42, 391, 355, 283, 437, 301, 255, + /* 890 */ 414, 410, 495, 492, 614, 28, 471, 306, 89, 90, + /* 900 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, + /* 910 */ 88, 424, 86, 86, 86, 86, 85, 85, 84, 84, + /* 920 */ 84, 83, 330, 424, 110, 110, 110, 110, 606, 606, + /* 930 */ 110, 254, 13, 614, 41, 532, 531, 283, 481, 531, + /* 940 */ 457, 284, 119, 561, 356, 614, 40, 284, 306, 89, + /* 950 */ 78, 80, 600, 599, 601, 601, 87, 87, 88, 88, + /* 960 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, + /* 970 */ 84, 84, 83, 330, 110, 424, 341, 220, 555, 606, + /* 980 */ 606, 351, 555, 318, 614, 95, 413, 255, 83, 330, + /* 990 */ 284, 284, 255, 640, 333, 356, 255, 614, 39, 306, + /* 1000 */ 356, 90, 80, 600, 599, 601, 601, 87, 87, 88, + /* 1010 */ 88, 88, 88, 424, 86, 86, 86, 86, 85, 85, + /* 1020 */ 84, 84, 84, 83, 330, 424, 317, 316, 141, 465, + /* 1030 */ 606, 606, 219, 619, 463, 614, 10, 417, 462, 255, + /* 1040 */ 189, 510, 553, 351, 207, 363, 161, 614, 38, 315, + /* 1050 */ 218, 255, 255, 80, 600, 599, 601, 601, 87, 87, + /* 1060 */ 88, 88, 88, 88, 424, 86, 86, 86, 86, 85, + /* 1070 */ 85, 84, 84, 84, 83, 330, 76, 419, 255, 3, + /* 1080 */ 878, 461, 424, 247, 331, 331, 614, 37, 217, 76, + /* 1090 */ 419, 390, 3, 216, 215, 422, 4, 331, 331, 424, + /* 1100 */ 547, 12, 424, 545, 614, 36, 424, 541, 422, 424, + /* 1110 */ 540, 424, 214, 424, 408, 424, 539, 403, 605, 605, + /* 1120 */ 237, 614, 25, 119, 614, 24, 588, 408, 614, 45, + /* 1130 */ 118, 614, 35, 614, 34, 614, 33, 614, 23, 588, + /* 1140 */ 60, 223, 603, 602, 513, 378, 73, 74, 140, 139, + /* 1150 */ 424, 110, 265, 75, 426, 425, 59, 424, 610, 73, + /* 1160 */ 74, 549, 402, 404, 424, 373, 75, 426, 425, 604, + /* 1170 */ 138, 610, 614, 11, 392, 76, 419, 181, 3, 614, + /* 1180 */ 32, 271, 369, 331, 331, 493, 614, 31, 149, 608, + /* 1190 */ 608, 608, 607, 15, 422, 365, 614, 8, 137, 489, + /* 1200 */ 136, 190, 608, 608, 608, 607, 15, 485, 176, 135, + /* 1210 */ 7, 252, 477, 408, 174, 133, 175, 474, 57, 56, + /* 1220 */ 132, 130, 119, 76, 419, 588, 3, 468, 245, 464, + /* 1230 */ 171, 331, 331, 125, 123, 456, 447, 122, 446, 104, + /* 1240 */ 336, 231, 422, 166, 154, 73, 74, 332, 116, 431, + /* 1250 */ 121, 309, 75, 426, 425, 222, 106, 610, 308, 637, + /* 1260 */ 204, 408, 629, 627, 628, 6, 200, 428, 427, 290, + /* 1270 */ 203, 622, 201, 588, 62, 63, 289, 66, 419, 399, + /* 1280 */ 3, 401, 288, 92, 143, 331, 331, 287, 608, 608, + /* 1290 */ 608, 607, 15, 73, 74, 227, 422, 325, 69, 416, + /* 1300 */ 75, 426, 425, 612, 412, 610, 192, 61, 569, 209, + /* 1310 */ 396, 226, 278, 225, 383, 408, 527, 558, 276, 533, + /* 1320 */ 552, 528, 321, 523, 370, 508, 180, 588, 494, 179, + /* 1330 */ 366, 117, 253, 269, 522, 503, 608, 608, 608, 607, + /* 1340 */ 15, 551, 502, 58, 274, 524, 178, 73, 74, 304, + /* 1350 */ 501, 368, 303, 206, 75, 426, 425, 491, 360, 610, + /* 1360 */ 213, 177, 483, 131, 345, 298, 297, 296, 202, 294, + /* 1370 */ 480, 490, 466, 134, 172, 129, 444, 346, 470, 128, + /* 1380 */ 314, 459, 103, 127, 126, 148, 124, 167, 443, 235, + /* 1390 */ 608, 608, 608, 607, 15, 442, 439, 623, 234, 299, + /* 1400 */ 145, 583, 291, 377, 581, 160, 119, 156, 270, 636, + /* 1410 */ 971, 169, 279, 626, 520, 625, 473, 624, 170, 621, + /* 1420 */ 618, 119, 168, 55, 409, 423, 537, 609, 286, 285, + /* 1430 */ 405, 570, 560, 556, 5, 52, 458, 554, 147, 267, + /* 1440 */ 519, 504, 518, 406, 262, 239, 260, 512, 343, 511, + /* 1450 */ 258, 353, 565, 256, 224, 251, 359, 277, 275, 476, + /* 1460 */ 475, 246, 352, 244, 467, 455, 236, 233, 232, 307, + /* 1470 */ 441, 281, 205, 163, 397, 280, 535, 505, 330, 617, + /* 1480 */ 971, 971, 971, 971, 367, 971, 971, 971, 971, 971, + /* 1490 */ 971, 971, 971, 971, 971, 971, 338, +}; +static const YYCODETYPE yy_lookahead[] = { + /* 0 */ 19, 22, 22, 23, 1, 24, 26, 15, 27, 80, + /* 10 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, + /* 20 */ 91, 92, 93, 94, 95, 108, 109, 110, 27, 28, + /* 30 */ 23, 50, 51, 80, 81, 82, 83, 122, 85, 86, + /* 40 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 22, + /* 50 */ 70, 23, 71, 72, 73, 74, 75, 76, 77, 78, + /* 60 */ 79, 80, 81, 82, 83, 122, 85, 86, 87, 88, + /* 70 */ 89, 90, 91, 92, 93, 94, 95, 19, 97, 91, + /* 80 */ 92, 93, 94, 95, 26, 85, 86, 87, 88, 89, + /* 90 */ 90, 91, 92, 93, 94, 95, 27, 28, 97, 98, + /* 100 */ 99, 122, 211, 102, 103, 104, 79, 19, 50, 51, + /* 110 */ 19, 122, 59, 55, 113, 224, 225, 226, 89, 90, + /* 120 */ 91, 92, 93, 94, 95, 23, 27, 28, 26, 71, + /* 130 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + /* 140 */ 82, 83, 51, 85, 86, 87, 88, 89, 90, 91, + /* 150 */ 92, 93, 94, 95, 19, 132, 133, 58, 89, 90, + /* 160 */ 21, 108, 109, 110, 27, 28, 97, 98, 33, 100, + /* 170 */ 7, 8, 119, 120, 22, 19, 107, 42, 109, 27, + /* 180 */ 28, 27, 28, 95, 28, 50, 51, 99, 100, 101, + /* 190 */ 102, 103, 104, 105, 27, 28, 97, 98, 107, 152, + /* 200 */ 112, 132, 133, 112, 65, 69, 71, 72, 73, 74, + /* 210 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 11, + /* 220 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, + /* 230 */ 95, 19, 101, 97, 97, 98, 24, 101, 122, 157, + /* 240 */ 12, 99, 103, 112, 102, 103, 104, 152, 22, 97, + /* 250 */ 98, 97, 98, 27, 28, 113, 27, 29, 91, 164, + /* 260 */ 165, 124, 50, 51, 97, 98, 219, 59, 132, 133, + /* 270 */ 134, 22, 23, 45, 66, 47, 212, 213, 124, 140, + /* 280 */ 132, 133, 19, 71, 72, 73, 74, 75, 76, 77, + /* 290 */ 78, 79, 80, 81, 82, 83, 152, 85, 86, 87, + /* 300 */ 88, 89, 90, 91, 92, 93, 94, 95, 164, 165, + /* 310 */ 27, 28, 230, 50, 51, 233, 108, 109, 110, 70, + /* 320 */ 16, 59, 23, 97, 98, 26, 97, 22, 66, 185, + /* 330 */ 12, 187, 27, 28, 71, 72, 73, 74, 75, 76, + /* 340 */ 77, 78, 79, 80, 81, 82, 83, 29, 85, 86, + /* 350 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 19, + /* 360 */ 22, 148, 149, 45, 23, 47, 62, 154, 64, 156, + /* 370 */ 108, 109, 110, 37, 69, 23, 163, 59, 26, 26, + /* 380 */ 97, 98, 144, 145, 146, 147, 152, 200, 52, 23, + /* 390 */ 50, 51, 26, 22, 89, 90, 60, 210, 7, 8, + /* 400 */ 9, 138, 97, 22, 23, 26, 101, 26, 174, 175, + /* 410 */ 197, 71, 72, 73, 74, 75, 76, 77, 78, 79, + /* 420 */ 80, 81, 82, 83, 16, 85, 86, 87, 88, 89, + /* 430 */ 90, 91, 92, 93, 94, 95, 19, 132, 133, 134, + /* 440 */ 23, 152, 208, 209, 140, 152, 152, 111, 195, 196, + /* 450 */ 98, 70, 163, 160, 152, 23, 22, 164, 165, 246, + /* 460 */ 207, 27, 152, 174, 175, 171, 172, 50, 51, 137, + /* 470 */ 62, 139, 64, 171, 172, 222, 124, 27, 138, 24, + /* 480 */ 163, 89, 90, 130, 174, 175, 197, 163, 71, 72, + /* 490 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + /* 500 */ 83, 22, 85, 86, 87, 88, 89, 90, 91, 92, + /* 510 */ 93, 94, 95, 19, 197, 181, 182, 23, 208, 209, + /* 520 */ 152, 197, 26, 189, 132, 133, 232, 224, 225, 226, + /* 530 */ 152, 97, 91, 26, 232, 116, 212, 213, 152, 222, + /* 540 */ 121, 152, 174, 175, 50, 51, 243, 97, 22, 23, + /* 550 */ 22, 234, 174, 175, 177, 23, 239, 116, 163, 177, + /* 560 */ 174, 175, 121, 174, 175, 71, 72, 73, 74, 75, + /* 570 */ 76, 77, 78, 79, 80, 81, 82, 83, 24, 85, + /* 580 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 590 */ 19, 23, 197, 11, 23, 227, 70, 208, 220, 152, + /* 600 */ 31, 224, 225, 226, 35, 98, 224, 225, 226, 108, + /* 610 */ 109, 110, 115, 152, 117, 118, 27, 222, 49, 123, + /* 620 */ 24, 50, 51, 27, 0, 1, 2, 224, 225, 226, + /* 630 */ 166, 124, 168, 169, 239, 174, 175, 170, 171, 172, + /* 640 */ 22, 194, 71, 72, 73, 74, 75, 76, 77, 78, + /* 650 */ 79, 80, 81, 82, 83, 152, 85, 86, 87, 88, + /* 660 */ 89, 90, 91, 92, 93, 94, 95, 19, 22, 208, + /* 670 */ 24, 23, 195, 196, 170, 171, 172, 174, 175, 152, + /* 680 */ 26, 152, 152, 152, 207, 152, 97, 152, 23, 152, + /* 690 */ 51, 244, 152, 97, 152, 247, 248, 23, 50, 51, + /* 700 */ 26, 174, 175, 174, 175, 174, 175, 174, 175, 174, + /* 710 */ 175, 174, 175, 23, 174, 175, 174, 175, 188, 71, + /* 720 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + /* 730 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, + /* 740 */ 92, 93, 94, 95, 19, 152, 107, 152, 33, 24, + /* 750 */ 152, 100, 101, 27, 174, 175, 152, 42, 152, 23, + /* 760 */ 152, 26, 152, 23, 152, 26, 152, 174, 175, 174, + /* 770 */ 175, 152, 174, 175, 23, 50, 51, 26, 174, 175, + /* 780 */ 174, 175, 174, 175, 174, 175, 174, 175, 174, 175, + /* 790 */ 163, 119, 120, 174, 175, 19, 71, 72, 73, 74, + /* 800 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 152, + /* 810 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, + /* 820 */ 95, 66, 152, 97, 197, 23, 50, 51, 26, 53, + /* 830 */ 23, 174, 175, 26, 23, 23, 23, 26, 26, 26, + /* 840 */ 36, 106, 146, 147, 174, 175, 19, 71, 72, 73, + /* 850 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + /* 860 */ 152, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 870 */ 94, 95, 152, 196, 119, 120, 19, 50, 51, 168, + /* 880 */ 169, 26, 174, 175, 207, 28, 152, 249, 250, 152, + /* 890 */ 163, 163, 163, 163, 174, 175, 163, 19, 71, 72, + /* 900 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + /* 910 */ 83, 152, 85, 86, 87, 88, 89, 90, 91, 92, + /* 920 */ 93, 94, 95, 152, 197, 197, 197, 197, 50, 51, + /* 930 */ 197, 194, 36, 174, 175, 191, 192, 152, 191, 192, + /* 940 */ 163, 152, 66, 124, 152, 174, 175, 152, 19, 71, + /* 950 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + /* 960 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, + /* 970 */ 92, 93, 94, 95, 197, 152, 100, 188, 152, 50, + /* 980 */ 51, 152, 152, 188, 174, 175, 252, 152, 94, 95, + /* 990 */ 152, 152, 152, 1, 2, 152, 152, 174, 175, 19, + /* 1000 */ 152, 72, 73, 74, 75, 76, 77, 78, 79, 80, + /* 1010 */ 81, 82, 83, 152, 85, 86, 87, 88, 89, 90, + /* 1020 */ 91, 92, 93, 94, 95, 152, 188, 188, 22, 194, + /* 1030 */ 50, 51, 240, 173, 194, 174, 175, 252, 194, 152, + /* 1040 */ 36, 181, 28, 152, 23, 219, 122, 174, 175, 219, + /* 1050 */ 221, 152, 152, 73, 74, 75, 76, 77, 78, 79, + /* 1060 */ 80, 81, 82, 83, 152, 85, 86, 87, 88, 89, + /* 1070 */ 90, 91, 92, 93, 94, 95, 19, 20, 152, 22, + /* 1080 */ 23, 194, 152, 240, 27, 28, 174, 175, 240, 19, + /* 1090 */ 20, 26, 22, 194, 194, 38, 22, 27, 28, 152, + /* 1100 */ 23, 22, 152, 116, 174, 175, 152, 23, 38, 152, + /* 1110 */ 23, 152, 221, 152, 57, 152, 23, 163, 50, 51, + /* 1120 */ 194, 174, 175, 66, 174, 175, 69, 57, 174, 175, + /* 1130 */ 40, 174, 175, 174, 175, 174, 175, 174, 175, 69, + /* 1140 */ 22, 53, 74, 75, 30, 53, 89, 90, 22, 22, + /* 1150 */ 152, 197, 23, 96, 97, 98, 22, 152, 101, 89, + /* 1160 */ 90, 91, 208, 209, 152, 53, 96, 97, 98, 101, + /* 1170 */ 22, 101, 174, 175, 152, 19, 20, 105, 22, 174, + /* 1180 */ 175, 112, 19, 27, 28, 20, 174, 175, 24, 132, + /* 1190 */ 133, 134, 135, 136, 38, 44, 174, 175, 107, 61, + /* 1200 */ 54, 26, 132, 133, 134, 135, 136, 54, 107, 22, + /* 1210 */ 5, 140, 1, 57, 36, 111, 122, 28, 79, 79, + /* 1220 */ 131, 123, 66, 19, 20, 69, 22, 1, 16, 20, + /* 1230 */ 125, 27, 28, 123, 111, 120, 23, 131, 23, 16, + /* 1240 */ 68, 142, 38, 15, 22, 89, 90, 3, 167, 4, + /* 1250 */ 248, 251, 96, 97, 98, 180, 180, 101, 251, 151, + /* 1260 */ 6, 57, 151, 13, 151, 26, 25, 151, 161, 202, + /* 1270 */ 153, 162, 153, 69, 130, 128, 203, 19, 20, 127, + /* 1280 */ 22, 126, 204, 129, 22, 27, 28, 205, 132, 133, + /* 1290 */ 134, 135, 136, 89, 90, 231, 38, 95, 137, 179, + /* 1300 */ 96, 97, 98, 206, 179, 101, 122, 107, 159, 159, + /* 1310 */ 125, 231, 216, 228, 107, 57, 184, 217, 216, 176, + /* 1320 */ 217, 176, 48, 106, 18, 184, 158, 69, 159, 158, + /* 1330 */ 46, 71, 237, 176, 176, 176, 132, 133, 134, 135, + /* 1340 */ 136, 217, 176, 137, 216, 178, 158, 89, 90, 179, + /* 1350 */ 176, 159, 179, 159, 96, 97, 98, 159, 159, 101, + /* 1360 */ 5, 158, 202, 22, 18, 10, 11, 12, 13, 14, + /* 1370 */ 190, 238, 17, 190, 158, 193, 41, 159, 202, 193, + /* 1380 */ 159, 202, 245, 193, 193, 223, 190, 32, 159, 34, + /* 1390 */ 132, 133, 134, 135, 136, 159, 39, 155, 43, 150, + /* 1400 */ 223, 177, 201, 178, 177, 186, 66, 199, 177, 152, + /* 1410 */ 253, 56, 215, 152, 182, 152, 202, 152, 63, 152, + /* 1420 */ 152, 66, 67, 242, 229, 152, 174, 152, 152, 152, + /* 1430 */ 152, 152, 152, 152, 199, 242, 202, 152, 198, 152, + /* 1440 */ 152, 152, 183, 192, 152, 215, 152, 183, 215, 183, + /* 1450 */ 152, 241, 214, 152, 211, 152, 152, 211, 211, 152, + /* 1460 */ 152, 241, 152, 152, 152, 152, 152, 152, 152, 114, + /* 1470 */ 152, 152, 235, 152, 152, 152, 174, 187, 95, 174, + /* 1480 */ 253, 253, 253, 253, 236, 253, 253, 253, 253, 253, + /* 1490 */ 253, 253, 253, 253, 253, 253, 141, +}; +#define YY_SHIFT_USE_DFLT (-86) +#define YY_SHIFT_COUNT (429) +#define YY_SHIFT_MIN (-85) +#define YY_SHIFT_MAX (1383) +static const short yy_shift_ofst[] = { + /* 0 */ 992, 1057, 1355, 1156, 1204, 1204, 1, 262, -19, 135, + /* 10 */ 135, 776, 1204, 1204, 1204, 1204, 69, 69, 53, 208, + /* 20 */ 283, 755, 58, 725, 648, 571, 494, 417, 340, 263, + /* 30 */ 212, 827, 827, 827, 827, 827, 827, 827, 827, 827, + /* 40 */ 827, 827, 827, 827, 827, 827, 878, 827, 929, 980, + /* 50 */ 980, 1070, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 60 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 70 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 80 */ 1258, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 90 */ 1204, 1204, 1204, 1204, -71, -47, -47, -47, -47, -47, + /* 100 */ 0, 29, -12, 283, 283, 139, 91, 392, 392, 894, + /* 110 */ 672, 726, 1383, -86, -86, -86, 88, 318, 318, 99, + /* 120 */ 381, -20, 283, 283, 283, 283, 283, 283, 283, 283, + /* 130 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + /* 140 */ 283, 283, 283, 283, 624, 876, 726, 672, 1340, 1340, + /* 150 */ 1340, 1340, 1340, 1340, -86, -86, -86, 305, 136, 136, + /* 160 */ 142, 167, 226, 154, 137, 152, 283, 283, 283, 283, + /* 170 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + /* 180 */ 283, 283, 283, 336, 336, 336, 283, 283, 352, 283, + /* 190 */ 283, 283, 283, 283, 228, 283, 283, 283, 283, 283, + /* 200 */ 283, 283, 283, 283, 283, 501, 569, 596, 596, 596, + /* 210 */ 507, 497, 441, 391, 353, 156, 156, 857, 353, 857, + /* 220 */ 735, 813, 639, 715, 156, 332, 715, 715, 496, 419, + /* 230 */ 646, 1357, 1184, 1184, 1335, 1335, 1184, 1341, 1260, 1144, + /* 240 */ 1346, 1346, 1346, 1346, 1184, 1306, 1144, 1341, 1260, 1260, + /* 250 */ 1144, 1184, 1306, 1206, 1284, 1184, 1184, 1306, 1184, 1306, + /* 260 */ 1184, 1306, 1262, 1207, 1207, 1207, 1274, 1262, 1207, 1217, + /* 270 */ 1207, 1274, 1207, 1207, 1185, 1200, 1185, 1200, 1185, 1200, + /* 280 */ 1184, 1184, 1161, 1262, 1202, 1202, 1262, 1154, 1155, 1147, + /* 290 */ 1152, 1144, 1241, 1239, 1250, 1250, 1254, 1254, 1254, 1254, + /* 300 */ -86, -86, -86, -86, -86, -86, 1068, 304, 526, 249, + /* 310 */ 408, -83, 434, 812, 27, 811, 807, 802, 751, 589, + /* 320 */ 651, 163, 131, 674, 366, 450, 299, 148, 23, 102, + /* 330 */ 229, -21, 1245, 1244, 1222, 1099, 1228, 1172, 1223, 1215, + /* 340 */ 1213, 1115, 1106, 1123, 1110, 1209, 1105, 1212, 1226, 1098, + /* 350 */ 1089, 1140, 1139, 1104, 1189, 1178, 1094, 1211, 1205, 1187, + /* 360 */ 1101, 1071, 1153, 1175, 1146, 1138, 1151, 1091, 1164, 1165, + /* 370 */ 1163, 1069, 1072, 1148, 1112, 1134, 1127, 1129, 1126, 1092, + /* 380 */ 1114, 1118, 1088, 1090, 1093, 1087, 1084, 987, 1079, 1077, + /* 390 */ 1074, 1065, 924, 1021, 1014, 1004, 1006, 819, 739, 896, + /* 400 */ 855, 804, 739, 740, 736, 690, 654, 665, 618, 582, + /* 410 */ 568, 528, 554, 379, 532, 479, 455, 379, 432, 371, + /* 420 */ 341, 28, 338, 116, -11, -57, -85, 7, -8, 3, +}; +#define YY_REDUCE_USE_DFLT (-110) +#define YY_REDUCE_COUNT (305) +#define YY_REDUCE_MIN (-109) +#define YY_REDUCE_MAX (1323) +static const short yy_reduce_ofst[] = { + /* 0 */ 238, 954, 213, 289, 310, 234, 144, 317, -109, 382, + /* 10 */ 377, 303, 461, 389, 378, 368, 302, 294, 253, 395, + /* 20 */ 293, 324, 403, 403, 403, 403, 403, 403, 403, 403, + /* 30 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, + /* 40 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, + /* 50 */ 403, 1022, 1012, 1005, 998, 963, 961, 959, 957, 950, + /* 60 */ 947, 930, 912, 873, 861, 823, 810, 771, 759, 720, + /* 70 */ 708, 670, 657, 619, 614, 612, 610, 608, 606, 604, + /* 80 */ 598, 595, 593, 580, 542, 540, 537, 535, 533, 531, + /* 90 */ 529, 527, 503, 386, 403, 403, 403, 403, 403, 403, + /* 100 */ 403, 403, 403, 95, 447, 82, 334, 504, 467, 403, + /* 110 */ 477, 464, 403, 403, 403, 403, 860, 747, 744, 785, + /* 120 */ 638, 638, 926, 891, 900, 899, 887, 844, 840, 835, + /* 130 */ 848, 830, 843, 829, 792, 839, 826, 737, 838, 795, + /* 140 */ 789, 47, 734, 530, 696, 777, 711, 677, 733, 730, + /* 150 */ 729, 728, 727, 627, 448, 64, 187, 1305, 1302, 1252, + /* 160 */ 1290, 1273, 1323, 1322, 1321, 1319, 1318, 1316, 1315, 1314, + /* 170 */ 1313, 1312, 1311, 1310, 1308, 1307, 1304, 1303, 1301, 1298, + /* 180 */ 1294, 1292, 1289, 1266, 1264, 1259, 1288, 1287, 1238, 1285, + /* 190 */ 1281, 1280, 1279, 1278, 1251, 1277, 1276, 1275, 1273, 1268, + /* 200 */ 1267, 1265, 1263, 1261, 1257, 1248, 1237, 1247, 1246, 1243, + /* 210 */ 1238, 1240, 1235, 1249, 1234, 1233, 1230, 1220, 1214, 1210, + /* 220 */ 1225, 1219, 1232, 1231, 1197, 1195, 1227, 1224, 1201, 1208, + /* 230 */ 1242, 1137, 1236, 1229, 1193, 1181, 1221, 1177, 1196, 1179, + /* 240 */ 1191, 1190, 1186, 1182, 1218, 1216, 1176, 1162, 1183, 1180, + /* 250 */ 1160, 1199, 1203, 1133, 1095, 1198, 1194, 1188, 1192, 1171, + /* 260 */ 1169, 1168, 1173, 1174, 1166, 1159, 1141, 1170, 1158, 1167, + /* 270 */ 1157, 1132, 1145, 1143, 1124, 1128, 1103, 1102, 1100, 1096, + /* 280 */ 1150, 1149, 1085, 1125, 1080, 1064, 1120, 1097, 1082, 1078, + /* 290 */ 1073, 1067, 1109, 1107, 1119, 1117, 1116, 1113, 1111, 1108, + /* 300 */ 1007, 1000, 1002, 1076, 1075, 1081, +}; +static const YYACTIONTYPE yy_default[] = { + /* 0 */ 647, 964, 964, 964, 878, 878, 969, 964, 774, 802, + /* 10 */ 802, 938, 969, 969, 969, 876, 969, 969, 969, 964, + /* 20 */ 969, 778, 808, 969, 969, 969, 969, 969, 969, 969, + /* 30 */ 969, 937, 939, 816, 815, 918, 789, 813, 806, 810, + /* 40 */ 879, 872, 873, 871, 875, 880, 969, 809, 841, 856, + /* 50 */ 840, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 60 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 70 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 80 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 90 */ 969, 969, 969, 969, 850, 855, 862, 854, 851, 843, + /* 100 */ 842, 844, 845, 969, 969, 673, 739, 969, 969, 846, + /* 110 */ 969, 685, 847, 859, 858, 857, 680, 969, 969, 969, + /* 120 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 130 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 140 */ 969, 969, 969, 969, 647, 964, 969, 969, 964, 964, + /* 150 */ 964, 964, 964, 964, 956, 778, 768, 969, 969, 969, + /* 160 */ 969, 969, 969, 969, 969, 969, 969, 944, 942, 969, + /* 170 */ 891, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 180 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 190 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 200 */ 969, 969, 969, 969, 653, 969, 911, 774, 774, 774, + /* 210 */ 776, 754, 766, 655, 812, 791, 791, 923, 812, 923, + /* 220 */ 710, 733, 707, 802, 791, 874, 802, 802, 775, 766, + /* 230 */ 969, 949, 782, 782, 941, 941, 782, 821, 743, 812, + /* 240 */ 750, 750, 750, 750, 782, 670, 812, 821, 743, 743, + /* 250 */ 812, 782, 670, 917, 915, 782, 782, 670, 782, 670, + /* 260 */ 782, 670, 884, 741, 741, 741, 725, 884, 741, 710, + /* 270 */ 741, 725, 741, 741, 795, 790, 795, 790, 795, 790, + /* 280 */ 782, 782, 969, 884, 888, 888, 884, 807, 796, 805, + /* 290 */ 803, 812, 676, 728, 663, 663, 652, 652, 652, 652, + /* 300 */ 961, 961, 956, 712, 712, 695, 969, 969, 969, 969, + /* 310 */ 969, 969, 687, 969, 893, 969, 969, 969, 969, 969, + /* 320 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 330 */ 969, 828, 969, 648, 951, 969, 969, 948, 969, 969, + /* 340 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 350 */ 969, 969, 969, 969, 969, 969, 921, 969, 969, 969, + /* 360 */ 969, 969, 969, 914, 913, 969, 969, 969, 969, 969, + /* 370 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, + /* 380 */ 969, 969, 969, 969, 969, 969, 969, 757, 969, 969, + /* 390 */ 969, 761, 969, 969, 969, 969, 969, 969, 804, 969, + /* 400 */ 797, 969, 877, 969, 969, 969, 969, 969, 969, 969, + /* 410 */ 969, 969, 969, 966, 969, 969, 969, 965, 969, 969, + /* 420 */ 969, 969, 969, 830, 969, 829, 833, 969, 661, 969, + /* 430 */ 644, 649, 960, 963, 962, 959, 958, 957, 952, 950, + /* 440 */ 947, 946, 945, 943, 940, 936, 897, 895, 902, 901, + /* 450 */ 900, 899, 898, 896, 894, 892, 818, 817, 814, 811, + /* 460 */ 753, 935, 890, 752, 749, 748, 669, 953, 920, 929, + /* 470 */ 928, 927, 822, 926, 925, 924, 922, 919, 906, 820, + /* 480 */ 819, 744, 882, 881, 672, 910, 909, 908, 912, 916, + /* 490 */ 907, 784, 751, 671, 668, 675, 679, 731, 732, 740, + /* 500 */ 738, 737, 736, 735, 734, 730, 681, 686, 724, 709, + /* 510 */ 708, 717, 716, 722, 721, 720, 719, 718, 715, 714, + /* 520 */ 713, 706, 705, 711, 704, 727, 726, 723, 703, 747, + /* 530 */ 746, 745, 742, 702, 701, 700, 833, 699, 698, 838, + /* 540 */ 837, 866, 826, 755, 759, 758, 762, 763, 771, 770, + /* 550 */ 769, 780, 781, 793, 792, 824, 823, 794, 779, 773, + /* 560 */ 772, 788, 787, 786, 785, 777, 767, 799, 798, 868, + /* 570 */ 783, 867, 865, 934, 933, 932, 931, 930, 870, 967, + /* 580 */ 968, 887, 889, 886, 801, 800, 885, 869, 839, 836, + /* 590 */ 690, 691, 905, 904, 903, 693, 692, 689, 688, 863, + /* 600 */ 860, 852, 864, 861, 853, 849, 848, 834, 832, 831, + /* 610 */ 827, 835, 760, 756, 825, 765, 764, 697, 696, 694, + /* 620 */ 678, 677, 674, 667, 665, 664, 666, 662, 660, 659, + /* 630 */ 658, 657, 656, 684, 683, 682, 654, 651, 650, 646, + /* 640 */ 645, 643, +}; + +/* The next table maps tokens into fallback tokens. If a construct +** like the following: ** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. */ -SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( - sqlite3_vtab *pVtab, /* FTS3 vtab object */ - int nArg, /* Size of argument array */ - sqlite3_value **apVal, /* Array of arguments */ - sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ -){ - Fts3Table *p = (Fts3Table *)pVtab; - int rc = SQLITE_OK; /* Return Code */ - int isRemove = 0; /* True for an UPDATE or DELETE */ - u32 *aSzIns = 0; /* Sizes of inserted documents */ - u32 *aSzDel = 0; /* Sizes of deleted documents */ - int nChng = 0; /* Net change in number of documents */ - int bInsertDone = 0; +#ifdef YYFALLBACK +static const YYCODETYPE yyFallback[] = { + 0, /* $ => nothing */ + 0, /* SEMI => nothing */ + 27, /* EXPLAIN => ID */ + 27, /* QUERY => ID */ + 27, /* PLAN => ID */ + 27, /* BEGIN => ID */ + 0, /* TRANSACTION => nothing */ + 27, /* DEFERRED => ID */ + 27, /* IMMEDIATE => ID */ + 27, /* EXCLUSIVE => ID */ + 0, /* COMMIT => nothing */ + 27, /* END => ID */ + 27, /* ROLLBACK => ID */ + 27, /* SAVEPOINT => ID */ + 27, /* RELEASE => ID */ + 0, /* TO => nothing */ + 0, /* TABLE => nothing */ + 0, /* CREATE => nothing */ + 27, /* IF => ID */ + 0, /* NOT => nothing */ + 0, /* EXISTS => nothing */ + 27, /* TEMP => ID */ + 0, /* LP => nothing */ + 0, /* RP => nothing */ + 0, /* AS => nothing */ + 27, /* WITHOUT => ID */ + 0, /* COMMA => nothing */ + 0, /* ID => nothing */ + 0, /* INDEXED => nothing */ + 27, /* ABORT => ID */ + 27, /* ACTION => ID */ + 27, /* AFTER => ID */ + 27, /* ANALYZE => ID */ + 27, /* ASC => ID */ + 27, /* ATTACH => ID */ + 27, /* BEFORE => ID */ + 27, /* BY => ID */ + 27, /* CASCADE => ID */ + 27, /* CAST => ID */ + 27, /* COLUMNKW => ID */ + 27, /* CONFLICT => ID */ + 27, /* DATABASE => ID */ + 27, /* DESC => ID */ + 27, /* DETACH => ID */ + 27, /* EACH => ID */ + 27, /* FAIL => ID */ + 27, /* FOR => ID */ + 27, /* IGNORE => ID */ + 27, /* INITIALLY => ID */ + 27, /* INSTEAD => ID */ + 27, /* LIKE_KW => ID */ + 27, /* MATCH => ID */ + 27, /* NO => ID */ + 27, /* KEY => ID */ + 27, /* OF => ID */ + 27, /* OFFSET => ID */ + 27, /* PRAGMA => ID */ + 27, /* RAISE => ID */ + 27, /* RECURSIVE => ID */ + 27, /* REPLACE => ID */ + 27, /* RESTRICT => ID */ + 27, /* ROW => ID */ + 27, /* TRIGGER => ID */ + 27, /* VACUUM => ID */ + 27, /* VIEW => ID */ + 27, /* VIRTUAL => ID */ + 27, /* WITH => ID */ + 27, /* REINDEX => ID */ + 27, /* RENAME => ID */ + 27, /* CTIME_KW => ID */ +}; +#endif /* YYFALLBACK */ - /* At this point it must be known if the %_stat table exists or not. - ** So bHasStat may not be 2. */ - assert( p->bHasStat==0 || p->bHasStat==1 ); +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +*/ +struct yyStackEntry { + YYACTIONTYPE stateno; /* The state-number */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct yyStackEntry yyStackEntry; - assert( p->pSegments==0 ); - assert( - nArg==1 /* DELETE operations */ - || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ - ); +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct yyParser { + int yyidx; /* Index of top element in stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyidxMax; /* Maximum value of yyidx */ +#endif + int yyerrcnt; /* Shifts left before out of the error */ + sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ +#if YYSTACKDEPTH<=0 + int yystksz; /* Current side of the stack */ + yyStackEntry *yystack; /* The parser's stack */ +#else + yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ +#endif +}; +typedef struct yyParser yyParser; - /* Check for a "special" INSERT operation. One of the form: - ** - ** INSERT INTO xyz(xyz) VALUES('command'); - */ - if( nArg>1 - && sqlite3_value_type(apVal[0])==SQLITE_NULL - && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL - ){ - rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); - goto update_out; - } +#ifndef NDEBUG +/* #include */ +static FILE *yyTraceFILE = 0; +static char *yyTracePrompt = 0; +#endif /* NDEBUG */ - if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ - rc = SQLITE_CONSTRAINT; - goto update_out; - } +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +**
      +**
    • A FILE* to which trace output should be written. +** If NULL, then tracing is turned off. +**
    • A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +**
    +** +** Outputs: +** None. +*/ +SQLITE_PRIVATE void sqlite3ParserTrace(FILE *TraceFILE, char *zTracePrompt){ + yyTraceFILE = TraceFILE; + yyTracePrompt = zTracePrompt; + if( yyTraceFILE==0 ) yyTracePrompt = 0; + else if( yyTracePrompt==0 ) yyTraceFILE = 0; +} +#endif /* NDEBUG */ - /* Allocate space to hold the change in document sizes */ - aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); - if( aSzDel==0 ){ - rc = SQLITE_NOMEM; - goto update_out; - } - aSzIns = &aSzDel[p->nColumn+1]; - memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); +#ifndef NDEBUG +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const yyTokenName[] = { + "$", "SEMI", "EXPLAIN", "QUERY", + "PLAN", "BEGIN", "TRANSACTION", "DEFERRED", + "IMMEDIATE", "EXCLUSIVE", "COMMIT", "END", + "ROLLBACK", "SAVEPOINT", "RELEASE", "TO", + "TABLE", "CREATE", "IF", "NOT", + "EXISTS", "TEMP", "LP", "RP", + "AS", "WITHOUT", "COMMA", "ID", + "INDEXED", "ABORT", "ACTION", "AFTER", + "ANALYZE", "ASC", "ATTACH", "BEFORE", + "BY", "CASCADE", "CAST", "COLUMNKW", + "CONFLICT", "DATABASE", "DESC", "DETACH", + "EACH", "FAIL", "FOR", "IGNORE", + "INITIALLY", "INSTEAD", "LIKE_KW", "MATCH", + "NO", "KEY", "OF", "OFFSET", + "PRAGMA", "RAISE", "RECURSIVE", "REPLACE", + "RESTRICT", "ROW", "TRIGGER", "VACUUM", + "VIEW", "VIRTUAL", "WITH", "REINDEX", + "RENAME", "CTIME_KW", "ANY", "OR", + "AND", "IS", "BETWEEN", "IN", + "ISNULL", "NOTNULL", "NE", "EQ", + "GT", "LE", "LT", "GE", + "ESCAPE", "BITAND", "BITOR", "LSHIFT", + "RSHIFT", "PLUS", "MINUS", "STAR", + "SLASH", "REM", "CONCAT", "COLLATE", + "BITNOT", "STRING", "JOIN_KW", "CONSTRAINT", + "DEFAULT", "NULL", "PRIMARY", "UNIQUE", + "CHECK", "REFERENCES", "AUTOINCR", "ON", + "INSERT", "DELETE", "UPDATE", "SET", + "DEFERRABLE", "FOREIGN", "DROP", "UNION", + "ALL", "EXCEPT", "INTERSECT", "SELECT", + "VALUES", "DISTINCT", "DOT", "FROM", + "JOIN", "USING", "ORDER", "GROUP", + "HAVING", "LIMIT", "WHERE", "INTO", + "INTEGER", "FLOAT", "BLOB", "VARIABLE", + "CASE", "WHEN", "THEN", "ELSE", + "INDEX", "ALTER", "ADD", "error", + "input", "cmdlist", "ecmd", "explain", + "cmdx", "cmd", "transtype", "trans_opt", + "nm", "savepoint_opt", "create_table", "create_table_args", + "createkw", "temp", "ifnotexists", "dbnm", + "columnlist", "conslist_opt", "table_options", "select", + "column", "columnid", "type", "carglist", + "typetoken", "typename", "signed", "plus_num", + "minus_num", "ccons", "term", "expr", + "onconf", "sortorder", "autoinc", "idxlist_opt", + "refargs", "defer_subclause", "refarg", "refact", + "init_deferred_pred_opt", "conslist", "tconscomma", "tcons", + "idxlist", "defer_subclause_opt", "orconf", "resolvetype", + "raisetype", "ifexists", "fullname", "selectnowith", + "oneselect", "with", "multiselect_op", "distinct", + "selcollist", "from", "where_opt", "groupby_opt", + "having_opt", "orderby_opt", "limit_opt", "values", + "nexprlist", "exprlist", "sclp", "as", + "seltablist", "stl_prefix", "joinop", "indexed_opt", + "on_opt", "using_opt", "joinop2", "idlist", + "sortlist", "setlist", "insert_cmd", "inscollist_opt", + "likeop", "between_op", "in_op", "case_operand", + "case_exprlist", "case_else", "uniqueflag", "collate", + "nmnum", "trigger_decl", "trigger_cmd_list", "trigger_time", + "trigger_event", "foreach_clause", "when_clause", "trigger_cmd", + "trnm", "tridxby", "database_kw_opt", "key_opt", + "add_column_fullname", "kwcolumn_opt", "create_vtab", "vtabarglist", + "vtabarg", "vtabargtoken", "lp", "anylist", + "wqlist", +}; +#endif /* NDEBUG */ - rc = fts3Writelock(p); - if( rc!=SQLITE_OK ) goto update_out; +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const yyRuleName[] = { + /* 0 */ "input ::= cmdlist", + /* 1 */ "cmdlist ::= cmdlist ecmd", + /* 2 */ "cmdlist ::= ecmd", + /* 3 */ "ecmd ::= SEMI", + /* 4 */ "ecmd ::= explain cmdx SEMI", + /* 5 */ "explain ::=", + /* 6 */ "explain ::= EXPLAIN", + /* 7 */ "explain ::= EXPLAIN QUERY PLAN", + /* 8 */ "cmdx ::= cmd", + /* 9 */ "cmd ::= BEGIN transtype trans_opt", + /* 10 */ "trans_opt ::=", + /* 11 */ "trans_opt ::= TRANSACTION", + /* 12 */ "trans_opt ::= TRANSACTION nm", + /* 13 */ "transtype ::=", + /* 14 */ "transtype ::= DEFERRED", + /* 15 */ "transtype ::= IMMEDIATE", + /* 16 */ "transtype ::= EXCLUSIVE", + /* 17 */ "cmd ::= COMMIT trans_opt", + /* 18 */ "cmd ::= END trans_opt", + /* 19 */ "cmd ::= ROLLBACK trans_opt", + /* 20 */ "savepoint_opt ::= SAVEPOINT", + /* 21 */ "savepoint_opt ::=", + /* 22 */ "cmd ::= SAVEPOINT nm", + /* 23 */ "cmd ::= RELEASE savepoint_opt nm", + /* 24 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", + /* 25 */ "cmd ::= create_table create_table_args", + /* 26 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", + /* 27 */ "createkw ::= CREATE", + /* 28 */ "ifnotexists ::=", + /* 29 */ "ifnotexists ::= IF NOT EXISTS", + /* 30 */ "temp ::= TEMP", + /* 31 */ "temp ::=", + /* 32 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", + /* 33 */ "create_table_args ::= AS select", + /* 34 */ "table_options ::=", + /* 35 */ "table_options ::= WITHOUT nm", + /* 36 */ "columnlist ::= columnlist COMMA column", + /* 37 */ "columnlist ::= column", + /* 38 */ "column ::= columnid type carglist", + /* 39 */ "columnid ::= nm", + /* 40 */ "nm ::= ID|INDEXED", + /* 41 */ "nm ::= STRING", + /* 42 */ "nm ::= JOIN_KW", + /* 43 */ "type ::=", + /* 44 */ "type ::= typetoken", + /* 45 */ "typetoken ::= typename", + /* 46 */ "typetoken ::= typename LP signed RP", + /* 47 */ "typetoken ::= typename LP signed COMMA signed RP", + /* 48 */ "typename ::= ID|STRING", + /* 49 */ "typename ::= typename ID|STRING", + /* 50 */ "signed ::= plus_num", + /* 51 */ "signed ::= minus_num", + /* 52 */ "carglist ::= carglist ccons", + /* 53 */ "carglist ::=", + /* 54 */ "ccons ::= CONSTRAINT nm", + /* 55 */ "ccons ::= DEFAULT term", + /* 56 */ "ccons ::= DEFAULT LP expr RP", + /* 57 */ "ccons ::= DEFAULT PLUS term", + /* 58 */ "ccons ::= DEFAULT MINUS term", + /* 59 */ "ccons ::= DEFAULT ID|INDEXED", + /* 60 */ "ccons ::= NULL onconf", + /* 61 */ "ccons ::= NOT NULL onconf", + /* 62 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", + /* 63 */ "ccons ::= UNIQUE onconf", + /* 64 */ "ccons ::= CHECK LP expr RP", + /* 65 */ "ccons ::= REFERENCES nm idxlist_opt refargs", + /* 66 */ "ccons ::= defer_subclause", + /* 67 */ "ccons ::= COLLATE ID|STRING", + /* 68 */ "autoinc ::=", + /* 69 */ "autoinc ::= AUTOINCR", + /* 70 */ "refargs ::=", + /* 71 */ "refargs ::= refargs refarg", + /* 72 */ "refarg ::= MATCH nm", + /* 73 */ "refarg ::= ON INSERT refact", + /* 74 */ "refarg ::= ON DELETE refact", + /* 75 */ "refarg ::= ON UPDATE refact", + /* 76 */ "refact ::= SET NULL", + /* 77 */ "refact ::= SET DEFAULT", + /* 78 */ "refact ::= CASCADE", + /* 79 */ "refact ::= RESTRICT", + /* 80 */ "refact ::= NO ACTION", + /* 81 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", + /* 82 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", + /* 83 */ "init_deferred_pred_opt ::=", + /* 84 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", + /* 85 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", + /* 86 */ "conslist_opt ::=", + /* 87 */ "conslist_opt ::= COMMA conslist", + /* 88 */ "conslist ::= conslist tconscomma tcons", + /* 89 */ "conslist ::= tcons", + /* 90 */ "tconscomma ::= COMMA", + /* 91 */ "tconscomma ::=", + /* 92 */ "tcons ::= CONSTRAINT nm", + /* 93 */ "tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf", + /* 94 */ "tcons ::= UNIQUE LP idxlist RP onconf", + /* 95 */ "tcons ::= CHECK LP expr RP onconf", + /* 96 */ "tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt", + /* 97 */ "defer_subclause_opt ::=", + /* 98 */ "defer_subclause_opt ::= defer_subclause", + /* 99 */ "onconf ::=", + /* 100 */ "onconf ::= ON CONFLICT resolvetype", + /* 101 */ "orconf ::=", + /* 102 */ "orconf ::= OR resolvetype", + /* 103 */ "resolvetype ::= raisetype", + /* 104 */ "resolvetype ::= IGNORE", + /* 105 */ "resolvetype ::= REPLACE", + /* 106 */ "cmd ::= DROP TABLE ifexists fullname", + /* 107 */ "ifexists ::= IF EXISTS", + /* 108 */ "ifexists ::=", + /* 109 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select", + /* 110 */ "cmd ::= DROP VIEW ifexists fullname", + /* 111 */ "cmd ::= select", + /* 112 */ "select ::= with selectnowith", + /* 113 */ "selectnowith ::= oneselect", + /* 114 */ "selectnowith ::= selectnowith multiselect_op oneselect", + /* 115 */ "multiselect_op ::= UNION", + /* 116 */ "multiselect_op ::= UNION ALL", + /* 117 */ "multiselect_op ::= EXCEPT|INTERSECT", + /* 118 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", + /* 119 */ "oneselect ::= values", + /* 120 */ "values ::= VALUES LP nexprlist RP", + /* 121 */ "values ::= values COMMA LP exprlist RP", + /* 122 */ "distinct ::= DISTINCT", + /* 123 */ "distinct ::= ALL", + /* 124 */ "distinct ::=", + /* 125 */ "sclp ::= selcollist COMMA", + /* 126 */ "sclp ::=", + /* 127 */ "selcollist ::= sclp expr as", + /* 128 */ "selcollist ::= sclp STAR", + /* 129 */ "selcollist ::= sclp nm DOT STAR", + /* 130 */ "as ::= AS nm", + /* 131 */ "as ::= ID|STRING", + /* 132 */ "as ::=", + /* 133 */ "from ::=", + /* 134 */ "from ::= FROM seltablist", + /* 135 */ "stl_prefix ::= seltablist joinop", + /* 136 */ "stl_prefix ::=", + /* 137 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", + /* 138 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 139 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 140 */ "dbnm ::=", + /* 141 */ "dbnm ::= DOT nm", + /* 142 */ "fullname ::= nm dbnm", + /* 143 */ "joinop ::= COMMA|JOIN", + /* 144 */ "joinop ::= JOIN_KW JOIN", + /* 145 */ "joinop ::= JOIN_KW nm JOIN", + /* 146 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 147 */ "on_opt ::= ON expr", + /* 148 */ "on_opt ::=", + /* 149 */ "indexed_opt ::=", + /* 150 */ "indexed_opt ::= INDEXED BY nm", + /* 151 */ "indexed_opt ::= NOT INDEXED", + /* 152 */ "using_opt ::= USING LP idlist RP", + /* 153 */ "using_opt ::=", + /* 154 */ "orderby_opt ::=", + /* 155 */ "orderby_opt ::= ORDER BY sortlist", + /* 156 */ "sortlist ::= sortlist COMMA expr sortorder", + /* 157 */ "sortlist ::= expr sortorder", + /* 158 */ "sortorder ::= ASC", + /* 159 */ "sortorder ::= DESC", + /* 160 */ "sortorder ::=", + /* 161 */ "groupby_opt ::=", + /* 162 */ "groupby_opt ::= GROUP BY nexprlist", + /* 163 */ "having_opt ::=", + /* 164 */ "having_opt ::= HAVING expr", + /* 165 */ "limit_opt ::=", + /* 166 */ "limit_opt ::= LIMIT expr", + /* 167 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 168 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 169 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", + /* 170 */ "where_opt ::=", + /* 171 */ "where_opt ::= WHERE expr", + /* 172 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", + /* 173 */ "setlist ::= setlist COMMA nm EQ expr", + /* 174 */ "setlist ::= nm EQ expr", + /* 175 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt select", + /* 176 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES", + /* 177 */ "insert_cmd ::= INSERT orconf", + /* 178 */ "insert_cmd ::= REPLACE", + /* 179 */ "inscollist_opt ::=", + /* 180 */ "inscollist_opt ::= LP idlist RP", + /* 181 */ "idlist ::= idlist COMMA nm", + /* 182 */ "idlist ::= nm", + /* 183 */ "expr ::= term", + /* 184 */ "expr ::= LP expr RP", + /* 185 */ "term ::= NULL", + /* 186 */ "expr ::= ID|INDEXED", + /* 187 */ "expr ::= JOIN_KW", + /* 188 */ "expr ::= nm DOT nm", + /* 189 */ "expr ::= nm DOT nm DOT nm", + /* 190 */ "term ::= INTEGER|FLOAT|BLOB", + /* 191 */ "term ::= STRING", + /* 192 */ "expr ::= VARIABLE", + /* 193 */ "expr ::= expr COLLATE ID|STRING", + /* 194 */ "expr ::= CAST LP expr AS typetoken RP", + /* 195 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 196 */ "expr ::= ID|INDEXED LP STAR RP", + /* 197 */ "term ::= CTIME_KW", + /* 198 */ "expr ::= expr AND expr", + /* 199 */ "expr ::= expr OR expr", + /* 200 */ "expr ::= expr LT|GT|GE|LE expr", + /* 201 */ "expr ::= expr EQ|NE expr", + /* 202 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 203 */ "expr ::= expr PLUS|MINUS expr", + /* 204 */ "expr ::= expr STAR|SLASH|REM expr", + /* 205 */ "expr ::= expr CONCAT expr", + /* 206 */ "likeop ::= LIKE_KW|MATCH", + /* 207 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 208 */ "expr ::= expr likeop expr", + /* 209 */ "expr ::= expr likeop expr ESCAPE expr", + /* 210 */ "expr ::= expr ISNULL|NOTNULL", + /* 211 */ "expr ::= expr NOT NULL", + /* 212 */ "expr ::= expr IS expr", + /* 213 */ "expr ::= expr IS NOT expr", + /* 214 */ "expr ::= NOT expr", + /* 215 */ "expr ::= BITNOT expr", + /* 216 */ "expr ::= MINUS expr", + /* 217 */ "expr ::= PLUS expr", + /* 218 */ "between_op ::= BETWEEN", + /* 219 */ "between_op ::= NOT BETWEEN", + /* 220 */ "expr ::= expr between_op expr AND expr", + /* 221 */ "in_op ::= IN", + /* 222 */ "in_op ::= NOT IN", + /* 223 */ "expr ::= expr in_op LP exprlist RP", + /* 224 */ "expr ::= LP select RP", + /* 225 */ "expr ::= expr in_op LP select RP", + /* 226 */ "expr ::= expr in_op nm dbnm", + /* 227 */ "expr ::= EXISTS LP select RP", + /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 230 */ "case_exprlist ::= WHEN expr THEN expr", + /* 231 */ "case_else ::= ELSE expr", + /* 232 */ "case_else ::=", + /* 233 */ "case_operand ::= expr", + /* 234 */ "case_operand ::=", + /* 235 */ "exprlist ::= nexprlist", + /* 236 */ "exprlist ::=", + /* 237 */ "nexprlist ::= nexprlist COMMA expr", + /* 238 */ "nexprlist ::= expr", + /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt", + /* 240 */ "uniqueflag ::= UNIQUE", + /* 241 */ "uniqueflag ::=", + /* 242 */ "idxlist_opt ::=", + /* 243 */ "idxlist_opt ::= LP idxlist RP", + /* 244 */ "idxlist ::= idxlist COMMA nm collate sortorder", + /* 245 */ "idxlist ::= nm collate sortorder", + /* 246 */ "collate ::=", + /* 247 */ "collate ::= COLLATE ID|STRING", + /* 248 */ "cmd ::= DROP INDEX ifexists fullname", + /* 249 */ "cmd ::= VACUUM", + /* 250 */ "cmd ::= VACUUM nm", + /* 251 */ "cmd ::= PRAGMA nm dbnm", + /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 256 */ "nmnum ::= plus_num", + /* 257 */ "nmnum ::= nm", + /* 258 */ "nmnum ::= ON", + /* 259 */ "nmnum ::= DELETE", + /* 260 */ "nmnum ::= DEFAULT", + /* 261 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 262 */ "plus_num ::= INTEGER|FLOAT", + /* 263 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 264 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 265 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 266 */ "trigger_time ::= BEFORE", + /* 267 */ "trigger_time ::= AFTER", + /* 268 */ "trigger_time ::= INSTEAD OF", + /* 269 */ "trigger_time ::=", + /* 270 */ "trigger_event ::= DELETE|INSERT", + /* 271 */ "trigger_event ::= UPDATE", + /* 272 */ "trigger_event ::= UPDATE OF idlist", + /* 273 */ "foreach_clause ::=", + /* 274 */ "foreach_clause ::= FOR EACH ROW", + /* 275 */ "when_clause ::=", + /* 276 */ "when_clause ::= WHEN expr", + /* 277 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 278 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 279 */ "trnm ::= nm", + /* 280 */ "trnm ::= nm DOT nm", + /* 281 */ "tridxby ::=", + /* 282 */ "tridxby ::= INDEXED BY nm", + /* 283 */ "tridxby ::= NOT INDEXED", + /* 284 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", + /* 285 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select", + /* 286 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", + /* 287 */ "trigger_cmd ::= select", + /* 288 */ "expr ::= RAISE LP IGNORE RP", + /* 289 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 290 */ "raisetype ::= ROLLBACK", + /* 291 */ "raisetype ::= ABORT", + /* 292 */ "raisetype ::= FAIL", + /* 293 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 294 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 295 */ "cmd ::= DETACH database_kw_opt expr", + /* 296 */ "key_opt ::=", + /* 297 */ "key_opt ::= KEY expr", + /* 298 */ "database_kw_opt ::= DATABASE", + /* 299 */ "database_kw_opt ::=", + /* 300 */ "cmd ::= REINDEX", + /* 301 */ "cmd ::= REINDEX nm dbnm", + /* 302 */ "cmd ::= ANALYZE", + /* 303 */ "cmd ::= ANALYZE nm dbnm", + /* 304 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 305 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", + /* 306 */ "add_column_fullname ::= fullname", + /* 307 */ "kwcolumn_opt ::=", + /* 308 */ "kwcolumn_opt ::= COLUMNKW", + /* 309 */ "cmd ::= create_vtab", + /* 310 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 311 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 312 */ "vtabarglist ::= vtabarg", + /* 313 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 314 */ "vtabarg ::=", + /* 315 */ "vtabarg ::= vtabarg vtabargtoken", + /* 316 */ "vtabargtoken ::= ANY", + /* 317 */ "vtabargtoken ::= lp anylist RP", + /* 318 */ "lp ::= LP", + /* 319 */ "anylist ::=", + /* 320 */ "anylist ::= anylist LP anylist RP", + /* 321 */ "anylist ::= anylist ANY", + /* 322 */ "with ::=", + /* 323 */ "with ::= WITH wqlist", + /* 324 */ "with ::= WITH RECURSIVE wqlist", + /* 325 */ "wqlist ::= nm idxlist_opt AS LP select RP", + /* 326 */ "wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP", +}; +#endif /* NDEBUG */ - /* If this is an INSERT operation, or an UPDATE that modifies the rowid - ** value, then this operation requires constraint handling. - ** - ** If the on-conflict mode is REPLACE, this means that the existing row - ** should be deleted from the database before inserting the new row. Or, - ** if the on-conflict mode is other than REPLACE, then this method must - ** detect the conflict and return SQLITE_CONSTRAINT before beginning to - ** modify the database file. - */ - if( nArg>1 && p->zContentTbl==0 ){ - /* Find the value object that holds the new rowid value. */ - sqlite3_value *pNewRowid = apVal[3+p->nColumn]; - if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ - pNewRowid = apVal[1]; - } - if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( - sqlite3_value_type(apVal[0])==SQLITE_NULL - || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) - )){ - /* The new rowid is not NULL (in this case the rowid will be - ** automatically assigned and there is no chance of a conflict), and - ** the statement is either an INSERT or an UPDATE that modifies the - ** rowid column. So if the conflict mode is REPLACE, then delete any - ** existing row with rowid=pNewRowid. - ** - ** Or, if the conflict mode is not REPLACE, insert the new record into - ** the %_content table. If we hit the duplicate rowid constraint (or any - ** other error) while doing so, return immediately. - ** - ** This branch may also run if pNewRowid contains a value that cannot - ** be losslessly converted to an integer. In this case, the eventual - ** call to fts3InsertData() (either just below or further on in this - ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is - ** invoked, it will delete zero rows (since no row will have - ** docid=$pNewRowid if $pNewRowid is not an integer value). - */ - if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ - rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); - }else{ - rc = fts3InsertData(p, apVal, pRowid); - bInsertDone = 1; - } - } - } - if( rc!=SQLITE_OK ){ - goto update_out; - } +#if YYSTACKDEPTH<=0 +/* +** Try to increase the size of the parser stack. +*/ +static void yyGrowStack(yyParser *p){ + int newSize; + yyStackEntry *pNew; - /* If this is a DELETE or UPDATE operation, remove the old record. */ - if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); - rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); - isRemove = 1; - } - - /* If this is an INSERT or UPDATE operation, insert the new record. */ - if( nArg>1 && rc==SQLITE_OK ){ - int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); - if( bInsertDone==0 ){ - rc = fts3InsertData(p, apVal, pRowid); - if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ - rc = FTS_CORRUPT_VTAB; - } - } - if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ - rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); - } - if( rc==SQLITE_OK ){ - assert( p->iPrevDocid==*pRowid ); - rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); - } - if( p->bHasDocsize ){ - fts3InsertDocsize(&rc, p, aSzIns); + newSize = p->yystksz*2 + 100; + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + if( pNew ){ + p->yystack = pNew; + p->yystksz = newSize; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows to %d entries!\n", + yyTracePrompt, p->yystksz); } - nChng++; - } - - if( p->bFts4 ){ - fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); +#endif } - - update_out: - sqlite3_free(aSzDel); - sqlite3Fts3SegmentsClose(p); - return rc; } +#endif /* -** Flush any data in the pending-terms hash table to disk. If successful, -** merge all segments in the database (including the new segment, if -** there was any data to flush) into a single segment. +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to sqlite3Parser and sqlite3ParserFree. */ -SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ - int rc; - rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); - if( rc==SQLITE_OK ){ - rc = fts3DoOptimize(p, 1); - if( rc==SQLITE_OK || rc==SQLITE_DONE ){ - int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); - if( rc2!=SQLITE_OK ) rc = rc2; - }else{ - sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); - sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); - } +SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(u64)){ + yyParser *pParser; + pParser = (yyParser*)(*mallocProc)( (u64)sizeof(yyParser) ); + if( pParser ){ + pParser->yyidx = -1; +#ifdef YYTRACKMAXSTACKDEPTH + pParser->yyidxMax = 0; +#endif +#if YYSTACKDEPTH<=0 + pParser->yystack = NULL; + pParser->yystksz = 0; + yyGrowStack(pParser); +#endif } - sqlite3Fts3SegmentsClose(p); - return rc; + return pParser; } -#endif +/* The following function deletes the value associated with a +** symbol. The symbol can be either a terminal or nonterminal. +** "yymajor" is the symbol code, and "yypminor" is a pointer to +** the value. +*/ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +){ + sqlite3ParserARG_FETCH; + switch( yymajor ){ + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are not used + ** inside the C code. + */ + case 163: /* select */ + case 195: /* selectnowith */ + case 196: /* oneselect */ + case 207: /* values */ +{ +sqlite3SelectDelete(pParse->db, (yypminor->yy3)); +} + break; + case 174: /* term */ + case 175: /* expr */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy346).pExpr); +} + break; + case 179: /* idxlist_opt */ + case 188: /* idxlist */ + case 200: /* selcollist */ + case 203: /* groupby_opt */ + case 205: /* orderby_opt */ + case 208: /* nexprlist */ + case 209: /* exprlist */ + case 210: /* sclp */ + case 220: /* sortlist */ + case 221: /* setlist */ + case 228: /* case_exprlist */ +{ +sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); +} + break; + case 194: /* fullname */ + case 201: /* from */ + case 212: /* seltablist */ + case 213: /* stl_prefix */ +{ +sqlite3SrcListDelete(pParse->db, (yypminor->yy65)); +} + break; + case 197: /* with */ + case 252: /* wqlist */ +{ +sqlite3WithDelete(pParse->db, (yypminor->yy59)); +} + break; + case 202: /* where_opt */ + case 204: /* having_opt */ + case 216: /* on_opt */ + case 227: /* case_operand */ + case 229: /* case_else */ + case 238: /* when_clause */ + case 243: /* key_opt */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy132)); +} + break; + case 217: /* using_opt */ + case 219: /* idlist */ + case 223: /* inscollist_opt */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy408)); +} + break; + case 234: /* trigger_cmd_list */ + case 239: /* trigger_cmd */ +{ +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy473)); +} + break; + case 236: /* trigger_event */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy378).b); +} + break; + default: break; /* If no destructor action specified: do nothing */ + } +} -/************** End of fts3_write.c ******************************************/ -/************** Begin file fts3_snippet.c ************************************/ /* -** 2009 Oct 23 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** Pop the parser's stack once. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. ** -****************************************************************************** +** Return the major token number for the symbol popped. */ +static int yy_pop_parser_stack(yyParser *pParser){ + YYCODETYPE yymajor; + yyStackEntry *yytos = &pParser->yystack[pParser->yyidx]; -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ + /* There is no mechanism by which the parser stack can be popped below + ** empty in SQLite. */ + if( NEVER(pParser->yyidx<0) ) return 0; +#ifndef NDEBUG + if( yyTraceFILE && pParser->yyidx>=0 ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + yymajor = yytos->major; + yy_destructor(pParser, yymajor, &yytos->minor); + pParser->yyidx--; + return yymajor; +} -/* -** Characters that may appear in the second argument to matchinfo(). +/* +** Deallocate and destroy a parser. Destructors are all called for +** all stack elements before shutting the parser down. +** +** Inputs: +**
      +**
    • A pointer to the parser. This should be a pointer +** obtained from sqlite3ParserAlloc. +**
    • A pointer to a function used to reclaim memory obtained +** from malloc. +**
    */ -#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ -#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ -#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ -#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ -#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ -#define FTS3_MATCHINFO_LCS 's' /* nCol values */ -#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ -#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ -#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ +SQLITE_PRIVATE void sqlite3ParserFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void*) /* Function used to reclaim memory */ +){ + yyParser *pParser = (yyParser*)p; + /* In SQLite, we never try to destroy a parser that was not successfully + ** created in the first place. */ + if( NEVER(pParser==0) ) return; + while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + free(pParser->yystack); +#endif + (*freeProc)((void*)pParser); +} /* -** The default value for the second argument to matchinfo(). +** Return the peak depth of the stack for a parser. */ -#define FTS3_MATCHINFO_DEFAULT "pcx" - +#ifdef YYTRACKMAXSTACKDEPTH +SQLITE_PRIVATE int sqlite3ParserStackPeak(void *p){ + yyParser *pParser = (yyParser*)p; + return pParser->yyidxMax; +} +#endif /* -** Used as an fts3ExprIterate() context when loading phrase doclists to -** Fts3Expr.aDoclist[]/nDoclist. +** Find the appropriate action for a parser given the terminal +** look-ahead token iLookAhead. +** +** If the look-ahead token is YYNOCODE, then check to see if the action is +** independent of the look-ahead. If it is, return the action, otherwise +** return YY_NO_ACTION. */ -typedef struct LoadDoclistCtx LoadDoclistCtx; -struct LoadDoclistCtx { - Fts3Cursor *pCsr; /* FTS3 Cursor */ - int nPhrase; /* Number of phrases seen so far */ - int nToken; /* Number of tokens seen so far */ -}; +static int yy_find_shift_action( + yyParser *pParser, /* The parser */ + YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; + int stateno = pParser->yystack[pParser->yyidx].stateno; + + if( stateno>YY_SHIFT_COUNT + || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ + return yy_default[stateno]; + } + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + if( iLookAhead>0 ){ +#ifdef YYFALLBACK + YYCODETYPE iFallback; /* Fallback token */ + if( iLookAhead %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + return yy_find_shift_action(pParser, iFallback); + } +#endif +#ifdef YYWILDCARD + { + int j = i - iLookAhead + YYWILDCARD; + if( +#if YY_SHIFT_MIN+YYWILDCARD<0 + j>=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[YYWILDCARD]); + } +#endif /* NDEBUG */ + return yy_action[j]; + } + } +#endif /* YYWILDCARD */ + } + return yy_default[stateno]; + }else{ + return yy_action[i]; + } +} /* -** The following types are used as part of the implementation of the -** fts3BestSnippet() routine. +** Find the appropriate action for a parser given the non-terminal +** look-ahead token iLookAhead. +** +** If the look-ahead token is YYNOCODE, then check to see if the action is +** independent of the look-ahead. If it is, return the action, otherwise +** return YY_NO_ACTION. */ -typedef struct SnippetIter SnippetIter; -typedef struct SnippetPhrase SnippetPhrase; -typedef struct SnippetFragment SnippetFragment; - -struct SnippetIter { - Fts3Cursor *pCsr; /* Cursor snippet is being generated from */ - int iCol; /* Extract snippet from this column */ - int nSnippet; /* Requested snippet length (in tokens) */ - int nPhrase; /* Number of phrases in query */ - SnippetPhrase *aPhrase; /* Array of size nPhrase */ - int iCurrent; /* First token of current snippet */ -}; - -struct SnippetPhrase { - int nToken; /* Number of tokens in phrase */ - char *pList; /* Pointer to start of phrase position list */ - int iHead; /* Next value in position list */ - char *pHead; /* Position list data following iHead */ - int iTail; /* Next value in trailing position list */ - char *pTail; /* Position list data following iTail */ -}; - -struct SnippetFragment { - int iCol; /* Column snippet is extracted from */ - int iPos; /* Index of first token in snippet */ - u64 covered; /* Mask of query phrases covered */ - u64 hlmask; /* Mask of snippet terms to highlight */ -}; +static int yy_find_reduce_action( + int stateno, /* Current state number */ + YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; +#ifdef YYERRORSYMBOL + if( stateno>YY_REDUCE_COUNT ){ + return yy_default[stateno]; + } +#else + assert( stateno<=YY_REDUCE_COUNT ); +#endif + i = yy_reduce_ofst[stateno]; + assert( i!=YY_REDUCE_USE_DFLT ); + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; +#ifdef YYERRORSYMBOL + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + return yy_default[stateno]; + } +#else + assert( i>=0 && iyyidx--; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); + } +#endif + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ + + UNUSED_PARAMETER(yypMinor); /* Silence some compiler warnings */ + sqlite3ErrorMsg(pParse, "parser stack overflow"); + sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument var */ +} /* -** An instance of this structure is used to manage a pair of buffers, each -** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below -** for details. +** Perform a shift action. */ -struct MatchinfoBuffer { - u8 aRef[3]; - int nElem; - int bGlobal; /* Set if global data is loaded */ - char *zMatchinfo; - u32 aMatchinfo[1]; -}; - +static void yy_shift( + yyParser *yypParser, /* The parser to be shifted */ + int yyNewState, /* The new state to shift in */ + int yyMajor, /* The major token to shift in */ + YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */ +){ + yyStackEntry *yytos; + yypParser->yyidx++; +#ifdef YYTRACKMAXSTACKDEPTH + if( yypParser->yyidx>yypParser->yyidxMax ){ + yypParser->yyidxMax = yypParser->yyidx; + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yyidx>=YYSTACKDEPTH ){ + yyStackOverflow(yypParser, yypMinor); + return; + } +#else + if( yypParser->yyidx>=yypParser->yystksz ){ + yyGrowStack(yypParser); + if( yypParser->yyidx>=yypParser->yystksz ){ + yyStackOverflow(yypParser, yypMinor); + return; + } + } +#endif + yytos = &yypParser->yystack[yypParser->yyidx]; + yytos->stateno = (YYACTIONTYPE)yyNewState; + yytos->major = (YYCODETYPE)yyMajor; + yytos->minor = *yypMinor; +#ifndef NDEBUG + if( yyTraceFILE && yypParser->yyidx>0 ){ + int i; + fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); + fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); + for(i=1; i<=yypParser->yyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + } +#endif +} -/* -** The snippet() and offsets() functions both return text values. An instance -** of the following structure is used to accumulate those values while the -** functions are running. See fts3StringAppend() for details. +/* The following table contains information about every rule that +** is used during the reduce. */ -typedef struct StrBuffer StrBuffer; -struct StrBuffer { - char *z; /* Pointer to buffer containing string */ - int n; /* Length of z in bytes (excl. nul-term) */ - int nAlloc; /* Allocated size of buffer z in bytes */ +static const struct { + YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ + unsigned char nrhs; /* Number of right-hand side symbols in the rule */ +} yyRuleInfo[] = { + { 144, 1 }, + { 145, 2 }, + { 145, 1 }, + { 146, 1 }, + { 146, 3 }, + { 147, 0 }, + { 147, 1 }, + { 147, 3 }, + { 148, 1 }, + { 149, 3 }, + { 151, 0 }, + { 151, 1 }, + { 151, 2 }, + { 150, 0 }, + { 150, 1 }, + { 150, 1 }, + { 150, 1 }, + { 149, 2 }, + { 149, 2 }, + { 149, 2 }, + { 153, 1 }, + { 153, 0 }, + { 149, 2 }, + { 149, 3 }, + { 149, 5 }, + { 149, 2 }, + { 154, 6 }, + { 156, 1 }, + { 158, 0 }, + { 158, 3 }, + { 157, 1 }, + { 157, 0 }, + { 155, 5 }, + { 155, 2 }, + { 162, 0 }, + { 162, 2 }, + { 160, 3 }, + { 160, 1 }, + { 164, 3 }, + { 165, 1 }, + { 152, 1 }, + { 152, 1 }, + { 152, 1 }, + { 166, 0 }, + { 166, 1 }, + { 168, 1 }, + { 168, 4 }, + { 168, 6 }, + { 169, 1 }, + { 169, 2 }, + { 170, 1 }, + { 170, 1 }, + { 167, 2 }, + { 167, 0 }, + { 173, 2 }, + { 173, 2 }, + { 173, 4 }, + { 173, 3 }, + { 173, 3 }, + { 173, 2 }, + { 173, 2 }, + { 173, 3 }, + { 173, 5 }, + { 173, 2 }, + { 173, 4 }, + { 173, 4 }, + { 173, 1 }, + { 173, 2 }, + { 178, 0 }, + { 178, 1 }, + { 180, 0 }, + { 180, 2 }, + { 182, 2 }, + { 182, 3 }, + { 182, 3 }, + { 182, 3 }, + { 183, 2 }, + { 183, 2 }, + { 183, 1 }, + { 183, 1 }, + { 183, 2 }, + { 181, 3 }, + { 181, 2 }, + { 184, 0 }, + { 184, 2 }, + { 184, 2 }, + { 161, 0 }, + { 161, 2 }, + { 185, 3 }, + { 185, 1 }, + { 186, 1 }, + { 186, 0 }, + { 187, 2 }, + { 187, 7 }, + { 187, 5 }, + { 187, 5 }, + { 187, 10 }, + { 189, 0 }, + { 189, 1 }, + { 176, 0 }, + { 176, 3 }, + { 190, 0 }, + { 190, 2 }, + { 191, 1 }, + { 191, 1 }, + { 191, 1 }, + { 149, 4 }, + { 193, 2 }, + { 193, 0 }, + { 149, 8 }, + { 149, 4 }, + { 149, 1 }, + { 163, 2 }, + { 195, 1 }, + { 195, 3 }, + { 198, 1 }, + { 198, 2 }, + { 198, 1 }, + { 196, 9 }, + { 196, 1 }, + { 207, 4 }, + { 207, 5 }, + { 199, 1 }, + { 199, 1 }, + { 199, 0 }, + { 210, 2 }, + { 210, 0 }, + { 200, 3 }, + { 200, 2 }, + { 200, 4 }, + { 211, 2 }, + { 211, 1 }, + { 211, 0 }, + { 201, 0 }, + { 201, 2 }, + { 213, 2 }, + { 213, 0 }, + { 212, 7 }, + { 212, 7 }, + { 212, 7 }, + { 159, 0 }, + { 159, 2 }, + { 194, 2 }, + { 214, 1 }, + { 214, 2 }, + { 214, 3 }, + { 214, 4 }, + { 216, 2 }, + { 216, 0 }, + { 215, 0 }, + { 215, 3 }, + { 215, 2 }, + { 217, 4 }, + { 217, 0 }, + { 205, 0 }, + { 205, 3 }, + { 220, 4 }, + { 220, 2 }, + { 177, 1 }, + { 177, 1 }, + { 177, 0 }, + { 203, 0 }, + { 203, 3 }, + { 204, 0 }, + { 204, 2 }, + { 206, 0 }, + { 206, 2 }, + { 206, 4 }, + { 206, 4 }, + { 149, 6 }, + { 202, 0 }, + { 202, 2 }, + { 149, 8 }, + { 221, 5 }, + { 221, 3 }, + { 149, 6 }, + { 149, 7 }, + { 222, 2 }, + { 222, 1 }, + { 223, 0 }, + { 223, 3 }, + { 219, 3 }, + { 219, 1 }, + { 175, 1 }, + { 175, 3 }, + { 174, 1 }, + { 175, 1 }, + { 175, 1 }, + { 175, 3 }, + { 175, 5 }, + { 174, 1 }, + { 174, 1 }, + { 175, 1 }, + { 175, 3 }, + { 175, 6 }, + { 175, 5 }, + { 175, 4 }, + { 174, 1 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 175, 3 }, + { 224, 1 }, + { 224, 2 }, + { 175, 3 }, + { 175, 5 }, + { 175, 2 }, + { 175, 3 }, + { 175, 3 }, + { 175, 4 }, + { 175, 2 }, + { 175, 2 }, + { 175, 2 }, + { 175, 2 }, + { 225, 1 }, + { 225, 2 }, + { 175, 5 }, + { 226, 1 }, + { 226, 2 }, + { 175, 5 }, + { 175, 3 }, + { 175, 5 }, + { 175, 4 }, + { 175, 4 }, + { 175, 5 }, + { 228, 5 }, + { 228, 4 }, + { 229, 2 }, + { 229, 0 }, + { 227, 1 }, + { 227, 0 }, + { 209, 1 }, + { 209, 0 }, + { 208, 3 }, + { 208, 1 }, + { 149, 12 }, + { 230, 1 }, + { 230, 0 }, + { 179, 0 }, + { 179, 3 }, + { 188, 5 }, + { 188, 3 }, + { 231, 0 }, + { 231, 2 }, + { 149, 4 }, + { 149, 1 }, + { 149, 2 }, + { 149, 3 }, + { 149, 5 }, + { 149, 6 }, + { 149, 5 }, + { 149, 6 }, + { 232, 1 }, + { 232, 1 }, + { 232, 1 }, + { 232, 1 }, + { 232, 1 }, + { 171, 2 }, + { 171, 1 }, + { 172, 2 }, + { 149, 5 }, + { 233, 11 }, + { 235, 1 }, + { 235, 1 }, + { 235, 2 }, + { 235, 0 }, + { 236, 1 }, + { 236, 1 }, + { 236, 3 }, + { 237, 0 }, + { 237, 3 }, + { 238, 0 }, + { 238, 2 }, + { 234, 3 }, + { 234, 2 }, + { 240, 1 }, + { 240, 3 }, + { 241, 0 }, + { 241, 3 }, + { 241, 2 }, + { 239, 7 }, + { 239, 5 }, + { 239, 5 }, + { 239, 1 }, + { 175, 4 }, + { 175, 6 }, + { 192, 1 }, + { 192, 1 }, + { 192, 1 }, + { 149, 4 }, + { 149, 6 }, + { 149, 3 }, + { 243, 0 }, + { 243, 2 }, + { 242, 1 }, + { 242, 0 }, + { 149, 1 }, + { 149, 3 }, + { 149, 1 }, + { 149, 3 }, + { 149, 6 }, + { 149, 6 }, + { 244, 1 }, + { 245, 0 }, + { 245, 1 }, + { 149, 1 }, + { 149, 4 }, + { 246, 8 }, + { 247, 1 }, + { 247, 3 }, + { 248, 0 }, + { 248, 2 }, + { 249, 1 }, + { 249, 3 }, + { 250, 1 }, + { 251, 0 }, + { 251, 4 }, + { 251, 2 }, + { 197, 0 }, + { 197, 2 }, + { 197, 3 }, + { 252, 6 }, + { 252, 8 }, }; - -/************************************************************************* -** Start of MatchinfoBuffer code. -*/ +static void yy_accept(yyParser*); /* Forward Declaration */ /* -** Allocate a two-slot MatchinfoBuffer object. +** Perform a reduce action and the shift that must immediately +** follow the reduce. */ -static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ - MatchinfoBuffer *pRet; - int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer); - int nStr = (int)strlen(zMatchinfo); - - pRet = sqlite3_malloc(nByte + nStr+1); - if( pRet ){ - memset(pRet, 0, nByte); - pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; - pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); - pRet->nElem = nElem; - pRet->zMatchinfo = ((char*)pRet) + nByte; - memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); - pRet->aRef[0] = 1; +static void yy_reduce( + yyParser *yypParser, /* The parser */ + int yyruleno /* Number of the rule by which to reduce */ +){ + int yygoto; /* The next state */ + int yyact; /* The next action */ + YYMINORTYPE yygotominor; /* The LHS of the rule reduced */ + yyStackEntry *yymsp; /* The top of the parser's stack */ + int yysize; /* Amount to pop the stack */ + sqlite3ParserARG_FETCH; + yymsp = &yypParser->yystack[yypParser->yyidx]; +#ifndef NDEBUG + if( yyTraceFILE && yyruleno>=0 + && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ + fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, + yyRuleName[yyruleno]); } +#endif /* NDEBUG */ - return pRet; -} + /* Silence complaints from purify about yygotominor being uninitialized + ** in some cases when it is copied into the stack after the following + ** switch. yygotominor is uninitialized when a rule reduces that does + ** not set the value of its left-hand side nonterminal. Leaving the + ** value of the nonterminal uninitialized is utterly harmless as long + ** as the value is never used. So really the only thing this code + ** accomplishes is to quieten purify. + ** + ** 2007-01-16: The wireshark project (www.wireshark.org) reports that + ** without this code, their parser segfaults. I'm not sure what there + ** parser is doing to make this happen. This is the second bug report + ** from wireshark this week. Clearly they are stressing Lemon in ways + ** that it has not been previously stressed... (SQLite ticket #2172) + */ + /*memset(&yygotominor, 0, sizeof(yygotominor));*/ + yygotominor = yyzerominor; -static void fts3MIBufferFree(void *p){ - MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); - assert( (u32*)p==&pBuf->aMatchinfo[1] - || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] - ); - if( (u32*)p==&pBuf->aMatchinfo[1] ){ - pBuf->aRef[1] = 0; + switch( yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line + ** { ... } // User supplied code + ** #line + ** break; + */ + case 5: /* explain ::= */ +{ sqlite3BeginParse(pParse, 0); } + break; + case 6: /* explain ::= EXPLAIN */ +{ sqlite3BeginParse(pParse, 1); } + break; + case 7: /* explain ::= EXPLAIN QUERY PLAN */ +{ sqlite3BeginParse(pParse, 2); } + break; + case 8: /* cmdx ::= cmd */ +{ sqlite3FinishCoding(pParse); } + break; + case 9: /* cmd ::= BEGIN transtype trans_opt */ +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy328);} + break; + case 13: /* transtype ::= */ +{yygotominor.yy328 = TK_DEFERRED;} + break; + case 14: /* transtype ::= DEFERRED */ + case 15: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==15); + case 16: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==16); + case 115: /* multiselect_op ::= UNION */ yytestcase(yyruleno==115); + case 117: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==117); +{yygotominor.yy328 = yymsp[0].major;} + break; + case 17: /* cmd ::= COMMIT trans_opt */ + case 18: /* cmd ::= END trans_opt */ yytestcase(yyruleno==18); +{sqlite3CommitTransaction(pParse);} + break; + case 19: /* cmd ::= ROLLBACK trans_opt */ +{sqlite3RollbackTransaction(pParse);} + break; + case 22: /* cmd ::= SAVEPOINT nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); +} + break; + case 23: /* cmd ::= RELEASE savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); +} + break; + case 24: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); +} + break; + case 26: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ +{ + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy328,0,0,yymsp[-2].minor.yy328); +} + break; + case 27: /* createkw ::= CREATE */ +{ + pParse->db->lookaside.bEnabled = 0; + yygotominor.yy0 = yymsp[0].minor.yy0; +} + break; + case 28: /* ifnotexists ::= */ + case 31: /* temp ::= */ yytestcase(yyruleno==31); + case 68: /* autoinc ::= */ yytestcase(yyruleno==68); + case 81: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ yytestcase(yyruleno==81); + case 83: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==83); + case 85: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==85); + case 97: /* defer_subclause_opt ::= */ yytestcase(yyruleno==97); + case 108: /* ifexists ::= */ yytestcase(yyruleno==108); + case 218: /* between_op ::= BETWEEN */ yytestcase(yyruleno==218); + case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); +{yygotominor.yy328 = 0;} + break; + case 29: /* ifnotexists ::= IF NOT EXISTS */ + case 30: /* temp ::= TEMP */ yytestcase(yyruleno==30); + case 69: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==69); + case 84: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==84); + case 107: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==107); + case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); + case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); +{yygotominor.yy328 = 1;} + break; + case 32: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ +{ + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy186,0); +} + break; + case 33: /* create_table_args ::= AS select */ +{ + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy3); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy3); +} + break; + case 34: /* table_options ::= */ +{yygotominor.yy186 = 0;} + break; + case 35: /* table_options ::= WITHOUT nm */ +{ + if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ + yygotominor.yy186 = TF_WithoutRowid; }else{ - pBuf->aRef[2] = 0; + yygotominor.yy186 = 0; + sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } - - if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ - sqlite3_free(pBuf); +} + break; + case 38: /* column ::= columnid type carglist */ +{ + yygotominor.yy0.z = yymsp[-2].minor.yy0.z; + yygotominor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-2].minor.yy0.z) + pParse->sLastToken.n; +} + break; + case 39: /* columnid ::= nm */ +{ + sqlite3AddColumn(pParse,&yymsp[0].minor.yy0); + yygotominor.yy0 = yymsp[0].minor.yy0; + pParse->constraintName.n = 0; +} + break; + case 40: /* nm ::= ID|INDEXED */ + case 41: /* nm ::= STRING */ yytestcase(yyruleno==41); + case 42: /* nm ::= JOIN_KW */ yytestcase(yyruleno==42); + case 45: /* typetoken ::= typename */ yytestcase(yyruleno==45); + case 48: /* typename ::= ID|STRING */ yytestcase(yyruleno==48); + case 130: /* as ::= AS nm */ yytestcase(yyruleno==130); + case 131: /* as ::= ID|STRING */ yytestcase(yyruleno==131); + case 141: /* dbnm ::= DOT nm */ yytestcase(yyruleno==141); + case 150: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==150); + case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); + case 256: /* nmnum ::= plus_num */ yytestcase(yyruleno==256); + case 257: /* nmnum ::= nm */ yytestcase(yyruleno==257); + case 258: /* nmnum ::= ON */ yytestcase(yyruleno==258); + case 259: /* nmnum ::= DELETE */ yytestcase(yyruleno==259); + case 260: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==260); + case 261: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==261); + case 262: /* plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==262); + case 263: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==263); + case 279: /* trnm ::= nm */ yytestcase(yyruleno==279); +{yygotominor.yy0 = yymsp[0].minor.yy0;} + break; + case 44: /* type ::= typetoken */ +{sqlite3AddColumnType(pParse,&yymsp[0].minor.yy0);} + break; + case 46: /* typetoken ::= typename LP signed RP */ +{ + yygotominor.yy0.z = yymsp[-3].minor.yy0.z; + yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); +} + break; + case 47: /* typetoken ::= typename LP signed COMMA signed RP */ +{ + yygotominor.yy0.z = yymsp[-5].minor.yy0.z; + yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); +} + break; + case 49: /* typename ::= typename ID|STRING */ +{yygotominor.yy0.z=yymsp[-1].minor.yy0.z; yygotominor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} + break; + case 54: /* ccons ::= CONSTRAINT nm */ + case 92: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==92); +{pParse->constraintName = yymsp[0].minor.yy0;} + break; + case 55: /* ccons ::= DEFAULT term */ + case 57: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==57); +{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy346);} + break; + case 56: /* ccons ::= DEFAULT LP expr RP */ +{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy346);} + break; + case 58: /* ccons ::= DEFAULT MINUS term */ +{ + ExprSpan v; + v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy346.pExpr, 0, 0); + v.zStart = yymsp[-1].minor.yy0.z; + v.zEnd = yymsp[0].minor.yy346.zEnd; + sqlite3AddDefaultValue(pParse,&v); +} + break; + case 59: /* ccons ::= DEFAULT ID|INDEXED */ +{ + ExprSpan v; + spanExpr(&v, pParse, TK_STRING, &yymsp[0].minor.yy0); + sqlite3AddDefaultValue(pParse,&v); +} + break; + case 61: /* ccons ::= NOT NULL onconf */ +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy328);} + break; + case 62: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy328,yymsp[0].minor.yy328,yymsp[-2].minor.yy328);} + break; + case 63: /* ccons ::= UNIQUE onconf */ +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy328,0,0,0,0);} + break; + case 64: /* ccons ::= CHECK LP expr RP */ +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy346.pExpr);} + break; + case 65: /* ccons ::= REFERENCES nm idxlist_opt refargs */ +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy328);} + break; + case 66: /* ccons ::= defer_subclause */ +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy328);} + break; + case 67: /* ccons ::= COLLATE ID|STRING */ +{sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} + break; + case 70: /* refargs ::= */ +{ yygotominor.yy328 = OE_None*0x0101; /* EV: R-19803-45884 */} + break; + case 71: /* refargs ::= refargs refarg */ +{ yygotominor.yy328 = (yymsp[-1].minor.yy328 & ~yymsp[0].minor.yy429.mask) | yymsp[0].minor.yy429.value; } + break; + case 72: /* refarg ::= MATCH nm */ + case 73: /* refarg ::= ON INSERT refact */ yytestcase(yyruleno==73); +{ yygotominor.yy429.value = 0; yygotominor.yy429.mask = 0x000000; } + break; + case 74: /* refarg ::= ON DELETE refact */ +{ yygotominor.yy429.value = yymsp[0].minor.yy328; yygotominor.yy429.mask = 0x0000ff; } + break; + case 75: /* refarg ::= ON UPDATE refact */ +{ yygotominor.yy429.value = yymsp[0].minor.yy328<<8; yygotominor.yy429.mask = 0x00ff00; } + break; + case 76: /* refact ::= SET NULL */ +{ yygotominor.yy328 = OE_SetNull; /* EV: R-33326-45252 */} + break; + case 77: /* refact ::= SET DEFAULT */ +{ yygotominor.yy328 = OE_SetDflt; /* EV: R-33326-45252 */} + break; + case 78: /* refact ::= CASCADE */ +{ yygotominor.yy328 = OE_Cascade; /* EV: R-33326-45252 */} + break; + case 79: /* refact ::= RESTRICT */ +{ yygotominor.yy328 = OE_Restrict; /* EV: R-33326-45252 */} + break; + case 80: /* refact ::= NO ACTION */ +{ yygotominor.yy328 = OE_None; /* EV: R-33326-45252 */} + break; + case 82: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 98: /* defer_subclause_opt ::= defer_subclause */ yytestcase(yyruleno==98); + case 100: /* onconf ::= ON CONFLICT resolvetype */ yytestcase(yyruleno==100); + case 103: /* resolvetype ::= raisetype */ yytestcase(yyruleno==103); +{yygotominor.yy328 = yymsp[0].minor.yy328;} + break; + case 86: /* conslist_opt ::= */ +{yygotominor.yy0.n = 0; yygotominor.yy0.z = 0;} + break; + case 87: /* conslist_opt ::= COMMA conslist */ +{yygotominor.yy0 = yymsp[-1].minor.yy0;} + break; + case 90: /* tconscomma ::= COMMA */ +{pParse->constraintName.n = 0;} + break; + case 93: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy328,yymsp[-2].minor.yy328,0);} + break; + case 94: /* tcons ::= UNIQUE LP idxlist RP onconf */ +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy328,0,0,0,0);} + break; + case 95: /* tcons ::= CHECK LP expr RP onconf */ +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy346.pExpr);} + break; + case 96: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ +{ + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy328); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy328); +} + break; + case 99: /* onconf ::= */ +{yygotominor.yy328 = OE_Default;} + break; + case 101: /* orconf ::= */ +{yygotominor.yy186 = OE_Default;} + break; + case 102: /* orconf ::= OR resolvetype */ +{yygotominor.yy186 = (u8)yymsp[0].minor.yy328;} + break; + case 104: /* resolvetype ::= IGNORE */ +{yygotominor.yy328 = OE_Ignore;} + break; + case 105: /* resolvetype ::= REPLACE */ +{yygotominor.yy328 = OE_Replace;} + break; + case 106: /* cmd ::= DROP TABLE ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy65, 0, yymsp[-1].minor.yy328); +} + break; + case 109: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select */ +{ + sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy3, yymsp[-6].minor.yy328, yymsp[-4].minor.yy328); +} + break; + case 110: /* cmd ::= DROP VIEW ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy65, 1, yymsp[-1].minor.yy328); +} + break; + case 111: /* cmd ::= select */ +{ + SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; + sqlite3Select(pParse, yymsp[0].minor.yy3, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy3); +} + break; + case 112: /* select ::= with selectnowith */ +{ + Select *p = yymsp[0].minor.yy3; + if( p ){ + p->pWith = yymsp[-1].minor.yy59; + parserDoubleLinkSelect(pParse, p); + }else{ + sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy59); } + yygotominor.yy3 = p; } - -static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ - void (*xRet)(void*) = 0; - u32 *aOut = 0; - - if( p->aRef[1]==0 ){ - p->aRef[1] = 1; - aOut = &p->aMatchinfo[1]; - xRet = fts3MIBufferFree; + break; + case 113: /* selectnowith ::= oneselect */ + case 119: /* oneselect ::= values */ yytestcase(yyruleno==119); +{yygotominor.yy3 = yymsp[0].minor.yy3;} + break; + case 114: /* selectnowith ::= selectnowith multiselect_op oneselect */ +{ + Select *pRhs = yymsp[0].minor.yy3; + if( pRhs && pRhs->pPrior ){ + SrcList *pFrom; + Token x; + x.n = 0; + parserDoubleLinkSelect(pParse, pRhs); + pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); + pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0); } - else if( p->aRef[2]==0 ){ - p->aRef[2] = 1; - aOut = &p->aMatchinfo[p->nElem+2]; - xRet = fts3MIBufferFree; + if( pRhs ){ + pRhs->op = (u8)yymsp[-1].minor.yy328; + pRhs->pPrior = yymsp[-2].minor.yy3; + pRhs->selFlags &= ~SF_MultiValue; + if( yymsp[-1].minor.yy328!=TK_ALL ) pParse->hasCompound = 1; }else{ - aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32)); - if( aOut ){ - xRet = sqlite3_free; - if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); - } + sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy3); } - - *paOut = aOut; - return xRet; + yygotominor.yy3 = pRhs; } - -static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ - p->bGlobal = 1; - memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); -} - -/* -** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() -*/ -SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ - if( p ){ - assert( p->aRef[0]==1 ); - p->aRef[0] = 0; - if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ - sqlite3_free(p); + break; + case 116: /* multiselect_op ::= UNION ALL */ +{yygotominor.yy328 = TK_ALL;} + break; + case 118: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ +{ + yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy65,yymsp[-4].minor.yy132,yymsp[-3].minor.yy14,yymsp[-2].minor.yy132,yymsp[-1].minor.yy14,yymsp[-7].minor.yy381,yymsp[0].minor.yy476.pLimit,yymsp[0].minor.yy476.pOffset); +#if SELECTTRACE_ENABLED + /* Populate the Select.zSelName[] string that is used to help with + ** query planner debugging, to differentiate between multiple Select + ** objects in a complex query. + ** + ** If the SELECT keyword is immediately followed by a C-style comment + ** then extract the first few alphanumeric characters from within that + ** comment to be the zSelName value. Otherwise, the label is #N where + ** is an integer that is incremented with each SELECT statement seen. + */ + if( yygotominor.yy3!=0 ){ + const char *z = yymsp[-8].minor.yy0.z+6; + int i; + sqlite3_snprintf(sizeof(yygotominor.yy3->zSelName), yygotominor.yy3->zSelName, "#%d", + ++pParse->nSelect); + while( z[0]==' ' ) z++; + if( z[0]=='/' && z[1]=='*' ){ + z += 2; + while( z[0]==' ' ) z++; + for(i=0; sqlite3Isalnum(z[i]); i++){} + sqlite3_snprintf(sizeof(yygotominor.yy3->zSelName), yygotominor.yy3->zSelName, "%.*s", i, z); } } +#endif /* SELECTRACE_ENABLED */ } - -/* -** End of MatchinfoBuffer code. -*************************************************************************/ - - -/* -** This function is used to help iterate through a position-list. A position -** list is a list of unique integers, sorted from smallest to largest. Each -** element of the list is represented by an FTS3 varint that takes the value -** of the difference between the current element and the previous one plus -** two. For example, to store the position-list: -** -** 4 9 113 -** -** the three varints: -** -** 6 7 106 -** -** are encoded. -** -** When this function is called, *pp points to the start of an element of -** the list. *piPos contains the value of the previous entry in the list. -** After it returns, *piPos contains the value of the next element of the -** list and *pp is advanced to the following varint. -*/ -static void fts3GetDeltaPosition(char **pp, int *piPos){ - int iVal; - *pp += fts3GetVarint32(*pp, &iVal); - *piPos += (iVal-2); + break; + case 120: /* values ::= VALUES LP nexprlist RP */ +{ + yygotominor.yy3 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0,0); } - -/* -** Helper function for fts3ExprIterate() (see below). -*/ -static int fts3ExprIterate2( - Fts3Expr *pExpr, /* Expression to iterate phrases of */ - int *piPhrase, /* Pointer to phrase counter */ - int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ - void *pCtx /* Second argument to pass to callback */ -){ - int rc; /* Return code */ - int eType = pExpr->eType; /* Type of expression node pExpr */ - - if( eType!=FTSQUERY_PHRASE ){ - assert( pExpr->pLeft && pExpr->pRight ); - rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); - if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ - rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); - } + break; + case 121: /* values ::= values COMMA LP exprlist RP */ +{ + Select *pRight, *pLeft = yymsp[-4].minor.yy3; + pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values|SF_MultiValue,0,0); + if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; + if( pRight ){ + pRight->op = TK_ALL; + pLeft = yymsp[-4].minor.yy3; + pRight->pPrior = pLeft; + yygotominor.yy3 = pRight; }else{ - rc = x(pExpr, *piPhrase, pCtx); - (*piPhrase)++; + yygotominor.yy3 = pLeft; } - return rc; } - -/* -** Iterate through all phrase nodes in an FTS3 query, except those that -** are part of a sub-tree that is the right-hand-side of a NOT operator. -** For each phrase node found, the supplied callback function is invoked. -** -** If the callback function returns anything other than SQLITE_OK, -** the iteration is abandoned and the error code returned immediately. -** Otherwise, SQLITE_OK is returned after a callback has been made for -** all eligible phrase nodes. -*/ -static int fts3ExprIterate( - Fts3Expr *pExpr, /* Expression to iterate phrases of */ - int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ - void *pCtx /* Second argument to pass to callback */ -){ - int iPhrase = 0; /* Variable used as the phrase counter */ - return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); + break; + case 122: /* distinct ::= DISTINCT */ +{yygotominor.yy381 = SF_Distinct;} + break; + case 123: /* distinct ::= ALL */ + case 124: /* distinct ::= */ yytestcase(yyruleno==124); +{yygotominor.yy381 = 0;} + break; + case 125: /* sclp ::= selcollist COMMA */ + case 243: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==243); +{yygotominor.yy14 = yymsp[-1].minor.yy14;} + break; + case 126: /* sclp ::= */ + case 154: /* orderby_opt ::= */ yytestcase(yyruleno==154); + case 161: /* groupby_opt ::= */ yytestcase(yyruleno==161); + case 236: /* exprlist ::= */ yytestcase(yyruleno==236); + case 242: /* idxlist_opt ::= */ yytestcase(yyruleno==242); +{yygotominor.yy14 = 0;} + break; + case 127: /* selcollist ::= sclp expr as */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, yymsp[-1].minor.yy346.pExpr); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yygotominor.yy14,&yymsp[-1].minor.yy346); } - - -/* -** This is an fts3ExprIterate() callback used while loading the doclists -** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also -** fts3ExprLoadDoclists(). -*/ -static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ - int rc = SQLITE_OK; - Fts3Phrase *pPhrase = pExpr->pPhrase; - LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; - - UNUSED_PARAMETER(iPhrase); - - p->nPhrase++; - p->nToken += pPhrase->nToken; - - return rc; + break; + case 128: /* selcollist ::= sclp STAR */ +{ + Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0); + yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy14, p); } - -/* -** Load the doclists for each phrase in the query associated with FTS3 cursor -** pCsr. -** -** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable -** phrases in the expression (all phrases except those directly or -** indirectly descended from the right-hand-side of a NOT operator). If -** pnToken is not NULL, then it is set to the number of tokens in all -** matchable phrases of the expression. -*/ -static int fts3ExprLoadDoclists( - Fts3Cursor *pCsr, /* Fts3 cursor for current query */ - int *pnPhrase, /* OUT: Number of phrases in query */ - int *pnToken /* OUT: Number of tokens in query */ -){ - int rc; /* Return Code */ - LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ - sCtx.pCsr = pCsr; - rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); - if( pnPhrase ) *pnPhrase = sCtx.nPhrase; - if( pnToken ) *pnToken = sCtx.nToken; - return rc; + break; + case 129: /* selcollist ::= sclp nm DOT STAR */ +{ + Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &yymsp[0].minor.yy0); + Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, pDot); } - -static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ - (*(int *)ctx)++; - pExpr->iPhrase = iPhrase; - return SQLITE_OK; + break; + case 132: /* as ::= */ +{yygotominor.yy0.n = 0;} + break; + case 133: /* from ::= */ +{yygotominor.yy65 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy65));} + break; + case 134: /* from ::= FROM seltablist */ +{ + yygotominor.yy65 = yymsp[0].minor.yy65; + sqlite3SrcListShiftJoinType(yygotominor.yy65); } -static int fts3ExprPhraseCount(Fts3Expr *pExpr){ - int nPhrase = 0; - (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); - return nPhrase; + break; + case 135: /* stl_prefix ::= seltablist joinop */ +{ + yygotominor.yy65 = yymsp[-1].minor.yy65; + if( ALWAYS(yygotominor.yy65 && yygotominor.yy65->nSrc>0) ) yygotominor.yy65->a[yygotominor.yy65->nSrc-1].jointype = (u8)yymsp[0].minor.yy328; } - -/* -** Advance the position list iterator specified by the first two -** arguments so that it points to the first element with a value greater -** than or equal to parameter iNext. -*/ -static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){ - char *pIter = *ppIter; - if( pIter ){ - int iIter = *piIter; - - while( iIternSrc==1 ){ + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + if( yygotominor.yy65 ){ + struct SrcList_item *pNew = &yygotominor.yy65->a[yygotominor.yy65->nSrc-1]; + struct SrcList_item *pOld = yymsp[-4].minor.yy65->a; + pNew->zName = pOld->zName; + pNew->zDatabase = pOld->zDatabase; + pNew->pSelect = pOld->pSelect; + pOld->zName = pOld->zDatabase = 0; + pOld->pSelect = 0; } - fts3GetDeltaPosition(&pIter, &iIter); + sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy65); + }else{ + Select *pSubquery; + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy65); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy65,0,0,0,0,SF_NestedFrom,0,0); + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); } - - *piIter = iIter; - *ppIter = pIter; } + break; + case 140: /* dbnm ::= */ + case 149: /* indexed_opt ::= */ yytestcase(yyruleno==149); +{yygotominor.yy0.z=0; yygotominor.yy0.n=0;} + break; + case 142: /* fullname ::= nm dbnm */ +{yygotominor.yy65 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} + break; + case 143: /* joinop ::= COMMA|JOIN */ +{ yygotominor.yy328 = JT_INNER; } + break; + case 144: /* joinop ::= JOIN_KW JOIN */ +{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } + break; + case 145: /* joinop ::= JOIN_KW nm JOIN */ +{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } + break; + case 146: /* joinop ::= JOIN_KW nm nm JOIN */ +{ yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } + break; + case 147: /* on_opt ::= ON expr */ + case 164: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==164); + case 171: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==171); + case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); + case 233: /* case_operand ::= expr */ yytestcase(yyruleno==233); +{yygotominor.yy132 = yymsp[0].minor.yy346.pExpr;} + break; + case 148: /* on_opt ::= */ + case 163: /* having_opt ::= */ yytestcase(yyruleno==163); + case 170: /* where_opt ::= */ yytestcase(yyruleno==170); + case 232: /* case_else ::= */ yytestcase(yyruleno==232); + case 234: /* case_operand ::= */ yytestcase(yyruleno==234); +{yygotominor.yy132 = 0;} + break; + case 151: /* indexed_opt ::= NOT INDEXED */ +{yygotominor.yy0.z=0; yygotominor.yy0.n=1;} + break; + case 152: /* using_opt ::= USING LP idlist RP */ + case 180: /* inscollist_opt ::= LP idlist RP */ yytestcase(yyruleno==180); +{yygotominor.yy408 = yymsp[-1].minor.yy408;} + break; + case 153: /* using_opt ::= */ + case 179: /* inscollist_opt ::= */ yytestcase(yyruleno==179); +{yygotominor.yy408 = 0;} + break; + case 155: /* orderby_opt ::= ORDER BY sortlist */ + case 162: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==162); + case 235: /* exprlist ::= nexprlist */ yytestcase(yyruleno==235); +{yygotominor.yy14 = yymsp[0].minor.yy14;} + break; + case 156: /* sortlist ::= sortlist COMMA expr sortorder */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14,yymsp[-1].minor.yy346.pExpr); + if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; } - -/* -** Advance the snippet iterator to the next candidate snippet. -*/ -static int fts3SnippetNextCandidate(SnippetIter *pIter){ - int i; /* Loop counter */ - - if( pIter->iCurrent<0 ){ - /* The SnippetIter object has just been initialized. The first snippet - ** candidate always starts at offset 0 (even if this candidate has a - ** score of 0.0). - */ - pIter->iCurrent = 0; - - /* Advance the 'head' iterator of each phrase to the first offset that - ** is greater than or equal to (iNext+nSnippet). - */ - for(i=0; inPhrase; i++){ - SnippetPhrase *pPhrase = &pIter->aPhrase[i]; - fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet); + break; + case 157: /* sortlist ::= expr sortorder */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy346.pExpr); + if( yygotominor.yy14 && ALWAYS(yygotominor.yy14->a) ) yygotominor.yy14->a[0].sortOrder = (u8)yymsp[0].minor.yy328; +} + break; + case 158: /* sortorder ::= ASC */ + case 160: /* sortorder ::= */ yytestcase(yyruleno==160); +{yygotominor.yy328 = SQLITE_SO_ASC;} + break; + case 159: /* sortorder ::= DESC */ +{yygotominor.yy328 = SQLITE_SO_DESC;} + break; + case 165: /* limit_opt ::= */ +{yygotominor.yy476.pLimit = 0; yygotominor.yy476.pOffset = 0;} + break; + case 166: /* limit_opt ::= LIMIT expr */ +{yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr; yygotominor.yy476.pOffset = 0;} + break; + case 167: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yygotominor.yy476.pLimit = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pOffset = yymsp[0].minor.yy346.pExpr;} + break; + case 168: /* limit_opt ::= LIMIT expr COMMA expr */ +{yygotominor.yy476.pOffset = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr;} + break; + case 169: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ +{ + sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy65, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy65,yymsp[0].minor.yy132); +} + break; + case 172: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ +{ + sqlite3WithPush(pParse, yymsp[-7].minor.yy59, 1); + sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy65, &yymsp[-3].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy14,"set list"); + sqlite3Update(pParse,yymsp[-4].minor.yy65,yymsp[-1].minor.yy14,yymsp[0].minor.yy132,yymsp[-5].minor.yy186); +} + break; + case 173: /* setlist ::= setlist COMMA nm EQ expr */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy346.pExpr); + sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); +} + break; + case 174: /* setlist ::= nm EQ expr */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy346.pExpr); + sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); +} + break; + case 175: /* cmd ::= with insert_cmd INTO fullname inscollist_opt select */ +{ + sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); + sqlite3Insert(pParse, yymsp[-2].minor.yy65, yymsp[0].minor.yy3, yymsp[-1].minor.yy408, yymsp[-4].minor.yy186); +} + break; + case 176: /* cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ +{ + sqlite3WithPush(pParse, yymsp[-6].minor.yy59, 1); + sqlite3Insert(pParse, yymsp[-3].minor.yy65, 0, yymsp[-2].minor.yy408, yymsp[-5].minor.yy186); +} + break; + case 177: /* insert_cmd ::= INSERT orconf */ +{yygotominor.yy186 = yymsp[0].minor.yy186;} + break; + case 178: /* insert_cmd ::= REPLACE */ +{yygotominor.yy186 = OE_Replace;} + break; + case 181: /* idlist ::= idlist COMMA nm */ +{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy408,&yymsp[0].minor.yy0);} + break; + case 182: /* idlist ::= nm */ +{yygotominor.yy408 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} + break; + case 183: /* expr ::= term */ +{yygotominor.yy346 = yymsp[0].minor.yy346;} + break; + case 184: /* expr ::= LP expr RP */ +{yygotominor.yy346.pExpr = yymsp[-1].minor.yy346.pExpr; spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} + break; + case 185: /* term ::= NULL */ + case 190: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==190); + case 191: /* term ::= STRING */ yytestcase(yyruleno==191); +{spanExpr(&yygotominor.yy346, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} + break; + case 186: /* expr ::= ID|INDEXED */ + case 187: /* expr ::= JOIN_KW */ yytestcase(yyruleno==187); +{spanExpr(&yygotominor.yy346, pParse, TK_ID, &yymsp[0].minor.yy0);} + break; + case 188: /* expr ::= nm DOT nm */ +{ + Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); + spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 189: /* expr ::= nm DOT nm DOT nm */ +{ + Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); + Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); + Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); + spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 192: /* expr ::= VARIABLE */ +{ + if( yymsp[0].minor.yy0.n>=2 && yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1]) ){ + /* When doing a nested parse, one can include terms in an expression + ** that look like this: #1 #2 ... These terms refer to registers + ** in the virtual machine. #N is the N-th register. */ + if( pParse->nested==0 ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = 0; + }else{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &yymsp[0].minor.yy0); + if( yygotominor.yy346.pExpr ) sqlite3GetInt32(&yymsp[0].minor.yy0.z[1], &yygotominor.yy346.pExpr->iTable); } }else{ - int iStart; - int iEnd = 0x7FFFFFFF; - - for(i=0; inPhrase; i++){ - SnippetPhrase *pPhrase = &pIter->aPhrase[i]; - if( pPhrase->pHead && pPhrase->iHeadiHead; + spanExpr(&yygotominor.yy346, pParse, TK_VARIABLE, &yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yygotominor.yy346.pExpr); + } + spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 193: /* expr ::= expr COLLATE ID|STRING */ +{ + yygotominor.yy346.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy346.pExpr, &yymsp[0].minor.yy0, 1); + yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; +} + break; + case 194: /* expr ::= CAST LP expr AS typetoken RP */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy346.pExpr, 0, &yymsp[-1].minor.yy0); + spanSet(&yygotominor.yy346,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 195: /* expr ::= ID|INDEXED LP distinct exprlist RP */ +{ + if( yymsp[-1].minor.yy14 && yymsp[-1].minor.yy14->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ + sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); + } + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); + spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + if( yymsp[-2].minor.yy381 && yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->flags |= EP_Distinct; + } +} + break; + case 196: /* expr ::= ID|INDEXED LP STAR RP */ +{ + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); + spanSet(&yygotominor.yy346,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 197: /* term ::= CTIME_KW */ +{ + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); + spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 198: /* expr ::= expr AND expr */ + case 199: /* expr ::= expr OR expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==202); + case 203: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==203); + case 204: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==204); + case 205: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==205); +{spanBinaryExpr(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);} + break; + case 206: /* likeop ::= LIKE_KW|MATCH */ +{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 0;} + break; + case 207: /* likeop ::= NOT LIKE_KW|MATCH */ +{yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 1;} + break; + case 208: /* expr ::= expr likeop expr */ +{ + ExprList *pList; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy346.pExpr); + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy96.eOperator); + if( yymsp[-1].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; + if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; +} + break; + case 209: /* expr ::= expr likeop expr ESCAPE expr */ +{ + ExprList *pList; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); + yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy96.eOperator); + if( yymsp[-3].minor.yy96.bNot ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; + if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; +} + break; + case 210: /* expr ::= expr ISNULL|NOTNULL */ +{spanUnaryPostfix(&yygotominor.yy346,pParse,yymsp[0].major,&yymsp[-1].minor.yy346,&yymsp[0].minor.yy0);} + break; + case 211: /* expr ::= expr NOT NULL */ +{spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);} + break; + case 212: /* expr ::= expr IS expr */ +{ + spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL); +} + break; + case 213: /* expr ::= expr IS NOT expr */ +{ + spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL); +} + break; + case 214: /* expr ::= NOT expr */ + case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); +{spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + break; + case 216: /* expr ::= MINUS expr */ +{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UMINUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + break; + case 217: /* expr ::= PLUS expr */ +{spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UPLUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} + break; + case 220: /* expr ::= expr between_op expr AND expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pList = pList; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } + if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; +} + break; + case 223: /* expr ::= expr in_op LP exprlist RP */ +{ + if( yymsp[-1].minor.yy14==0 ){ + /* Expressions of the form + ** + ** expr1 IN () + ** expr1 NOT IN () + ** + ** simplify to constants 0 (false) and 1 (true), respectively, + ** regardless of the value of expr1. + */ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy328]); + sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy346.pExpr); + }else if( yymsp[-1].minor.yy14->nExpr==1 ){ + /* Expressions of the form: + ** + ** expr1 IN (?1) + ** expr1 NOT IN (?2) + ** + ** with exactly one value on the RHS can be simplified to something + ** like this: + ** + ** expr1 == ?1 + ** expr1 <> ?2 + ** + ** But, the RHS of the == or <> is marked with the EP_Generic flag + ** so that it may not contribute to the computation of comparison + ** affinity or the collating sequence to use for comparison. Otherwise, + ** the semantics would be subtly different from IN or NOT IN. + */ + Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr; + yymsp[-1].minor.yy14->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + /* pRHS cannot be NULL because a malloc error would have been detected + ** before now and control would have never reached this point */ + if( ALWAYS(pRHS) ){ + pRHS->flags &= ~EP_Collate; + pRHS->flags |= EP_Generic; + } + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, yymsp[-3].minor.yy328 ? TK_NE : TK_EQ, yymsp[-4].minor.yy346.pExpr, pRHS, 0); + }else{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); } + if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); } - if( iEnd==0x7FFFFFFF ){ - return 1; + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 224: /* expr ::= LP select RP */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); } - - pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; - for(i=0; inPhrase; i++){ - SnippetPhrase *pPhrase = &pIter->aPhrase[i]; - fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1); - fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart); + yygotominor.yy346.zStart = yymsp[-2].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 225: /* expr ::= expr in_op LP select RP */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pSelect = yymsp[-1].minor.yy3; + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); } + if( yymsp[-3].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-4].minor.yy346.zStart; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } - - return 0; -} - -/* -** Retrieve information about the current candidate snippet of snippet -** iterator pIter. -*/ -static void fts3SnippetDetails( - SnippetIter *pIter, /* Snippet iterator */ - u64 mCovered, /* Bitmask of phrases already covered */ - int *piToken, /* OUT: First token of proposed snippet */ - int *piScore, /* OUT: "Score" for this snippet */ - u64 *pmCover, /* OUT: Bitmask of phrases covered */ - u64 *pmHighlight /* OUT: Bitmask of terms to highlight */ -){ - int iStart = pIter->iCurrent; /* First token of snippet */ - int iScore = 0; /* Score of this snippet */ - int i; /* Loop counter */ - u64 mCover = 0; /* Mask of phrases covered by this snippet */ - u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */ - - for(i=0; inPhrase; i++){ - SnippetPhrase *pPhrase = &pIter->aPhrase[i]; - if( pPhrase->pTail ){ - char *pCsr = pPhrase->pTail; - int iCsr = pPhrase->iTail; - - while( iCsr<(iStart+pIter->nSnippet) ){ - int j; - u64 mPhrase = (u64)1 << i; - u64 mPos = (u64)1 << (iCsr - iStart); - assert( iCsr>=iStart ); - if( (mCover|mCovered)&mPhrase ){ - iScore++; - }else{ - iScore += 1000; - } - mCover |= mPhrase; - - for(j=0; jnToken; j++){ - mHighlight |= (mPos>>j); - } - - if( 0==(*pCsr & 0x0FE) ) break; - fts3GetDeltaPosition(&pCsr, &iCsr); - } + break; + case 226: /* expr ::= expr in_op nm dbnm */ +{ + SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy346.pExpr, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); + ExprSetProperty(yygotominor.yy346.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); + }else{ + sqlite3SrcListDelete(pParse->db, pSrc); } + if( yymsp[-2].minor.yy328 ) yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy346.pExpr, 0, 0); + yygotominor.yy346.zStart = yymsp[-3].minor.yy346.zStart; + yygotominor.yy346.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; } - - /* Set the output variables before returning. */ - *piToken = iStart; - *piScore = iScore; - *pmCover = mCover; - *pmHighlight = mHighlight; -} - -/* -** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). -** Each invocation populates an element of the SnippetIter.aPhrase[] array. -*/ -static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ - SnippetIter *p = (SnippetIter *)ctx; - SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; - char *pCsr; - int rc; - - pPhrase->nToken = pExpr->pPhrase->nToken; - rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); - assert( rc==SQLITE_OK || pCsr==0 ); - if( pCsr ){ - int iFirst = 0; - pPhrase->pList = pCsr; - fts3GetDeltaPosition(&pCsr, &iFirst); - assert( iFirst>=0 ); - pPhrase->pHead = pCsr; - pPhrase->pTail = pCsr; - pPhrase->iHead = iFirst; - pPhrase->iTail = iFirst; + break; + case 227: /* expr ::= EXISTS LP select RP */ +{ + Expr *p = yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); + if( p ){ + p->x.pSelect = yymsp[-1].minor.yy3; + ExprSetProperty(p, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, p); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy3); + } + yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy132, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->x.pList = yymsp[-1].minor.yy132 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy132) : yymsp[-2].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yygotominor.yy346.pExpr); }else{ - assert( rc!=SQLITE_OK || ( - pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 - )); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy132); } - - return rc; + yygotominor.yy346.zStart = yymsp[-4].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } - -/* -** Select the fragment of text consisting of nFragment contiguous tokens -** from column iCol that represent the "best" snippet. The best snippet -** is the snippet with the highest score, where scores are calculated -** by adding: -** -** (a) +1 point for each occurrence of a matchable phrase in the snippet. -** -** (b) +1000 points for the first occurrence of each matchable phrase in -** the snippet for which the corresponding mCovered bit is not set. -** -** The selected snippet parameters are stored in structure *pFragment before -** returning. The score of the selected snippet is stored in *piScore -** before returning. -*/ -static int fts3BestSnippet( - int nSnippet, /* Desired snippet length */ - Fts3Cursor *pCsr, /* Cursor to create snippet for */ - int iCol, /* Index of column to create snippet from */ - u64 mCovered, /* Mask of phrases already covered */ - u64 *pmSeen, /* IN/OUT: Mask of phrases seen */ - SnippetFragment *pFragment, /* OUT: Best snippet found */ - int *piScore /* OUT: Score of snippet pFragment */ -){ - int rc; /* Return Code */ - int nList; /* Number of phrases in expression */ - SnippetIter sIter; /* Iterates through snippet candidates */ - int nByte; /* Number of bytes of space to allocate */ - int iBestScore = -1; /* Best snippet score found so far */ - int i; /* Loop counter */ - - memset(&sIter, 0, sizeof(sIter)); - - /* Iterate through the phrases in the expression to count them. The same - ** callback makes sure the doclists are loaded for each phrase. - */ - rc = fts3ExprLoadDoclists(pCsr, &nList, 0); - if( rc!=SQLITE_OK ){ - return rc; + break; + case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy346.pExpr); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); +} + break; + case 230: /* case_exprlist ::= WHEN expr THEN expr */ +{ + yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); +} + break; + case 237: /* nexprlist ::= nexprlist COMMA expr */ +{yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy346.pExpr);} + break; + case 238: /* nexprlist ::= expr */ +{yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy346.pExpr);} + break; + case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt */ +{ + sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, + sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy328, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy132, SQLITE_SO_ASC, yymsp[-8].minor.yy328); +} + break; + case 240: /* uniqueflag ::= UNIQUE */ + case 291: /* raisetype ::= ABORT */ yytestcase(yyruleno==291); +{yygotominor.yy328 = OE_Abort;} + break; + case 241: /* uniqueflag ::= */ +{yygotominor.yy328 = OE_None;} + break; + case 244: /* idxlist ::= idxlist COMMA nm collate sortorder */ +{ + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0, 1); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, p); + sqlite3ExprListSetName(pParse,yygotominor.yy14,&yymsp[-2].minor.yy0,1); + sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); + if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; +} + break; + case 245: /* idxlist ::= nm collate sortorder */ +{ + Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0, 1); + yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, p); + sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); + sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); + if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; +} + break; + case 246: /* collate ::= */ +{yygotominor.yy0.z = 0; yygotominor.yy0.n = 0;} + break; + case 248: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy65, yymsp[-1].minor.yy328);} + break; + case 249: /* cmd ::= VACUUM */ + case 250: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==250); +{sqlite3Vacuum(pParse);} + break; + case 251: /* cmd ::= PRAGMA nm dbnm */ +{sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} + break; + case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} + break; + case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} + break; + case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} + break; + case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} + break; + case 264: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ +{ + Token all; + all.z = yymsp[-3].minor.yy0.z; + all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy473, &all); +} + break; + case 265: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ +{ + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy328, yymsp[-4].minor.yy378.a, yymsp[-4].minor.yy378.b, yymsp[-2].minor.yy65, yymsp[0].minor.yy132, yymsp[-10].minor.yy328, yymsp[-8].minor.yy328); + yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); +} + break; + case 266: /* trigger_time ::= BEFORE */ + case 269: /* trigger_time ::= */ yytestcase(yyruleno==269); +{ yygotominor.yy328 = TK_BEFORE; } + break; + case 267: /* trigger_time ::= AFTER */ +{ yygotominor.yy328 = TK_AFTER; } + break; + case 268: /* trigger_time ::= INSTEAD OF */ +{ yygotominor.yy328 = TK_INSTEAD;} + break; + case 270: /* trigger_event ::= DELETE|INSERT */ + case 271: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==271); +{yygotominor.yy378.a = yymsp[0].major; yygotominor.yy378.b = 0;} + break; + case 272: /* trigger_event ::= UPDATE OF idlist */ +{yygotominor.yy378.a = TK_UPDATE; yygotominor.yy378.b = yymsp[0].minor.yy408;} + break; + case 275: /* when_clause ::= */ + case 296: /* key_opt ::= */ yytestcase(yyruleno==296); +{ yygotominor.yy132 = 0; } + break; + case 276: /* when_clause ::= WHEN expr */ + case 297: /* key_opt ::= KEY expr */ yytestcase(yyruleno==297); +{ yygotominor.yy132 = yymsp[0].minor.yy346.pExpr; } + break; + case 277: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ +{ + assert( yymsp[-2].minor.yy473!=0 ); + yymsp[-2].minor.yy473->pLast->pNext = yymsp[-1].minor.yy473; + yymsp[-2].minor.yy473->pLast = yymsp[-1].minor.yy473; + yygotominor.yy473 = yymsp[-2].minor.yy473; +} + break; + case 278: /* trigger_cmd_list ::= trigger_cmd SEMI */ +{ + assert( yymsp[-1].minor.yy473!=0 ); + yymsp[-1].minor.yy473->pLast = yymsp[-1].minor.yy473; + yygotominor.yy473 = yymsp[-1].minor.yy473; +} + break; + case 280: /* trnm ::= nm DOT nm */ +{ + yygotominor.yy0 = yymsp[0].minor.yy0; + sqlite3ErrorMsg(pParse, + "qualified table names are not allowed on INSERT, UPDATE, and DELETE " + "statements within triggers"); +} + break; + case 282: /* tridxby ::= INDEXED BY nm */ +{ + sqlite3ErrorMsg(pParse, + "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " + "within triggers"); +} + break; + case 283: /* tridxby ::= NOT INDEXED */ +{ + sqlite3ErrorMsg(pParse, + "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " + "within triggers"); +} + break; + case 284: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ +{ yygotominor.yy473 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy14, yymsp[0].minor.yy132, yymsp[-5].minor.yy186); } + break; + case 285: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ +{yygotominor.yy473 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy408, yymsp[0].minor.yy3, yymsp[-4].minor.yy186);} + break; + case 286: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ +{yygotominor.yy473 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy132);} + break; + case 287: /* trigger_cmd ::= select */ +{yygotominor.yy473 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy3); } + break; + case 288: /* expr ::= RAISE LP IGNORE RP */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); + if( yygotominor.yy346.pExpr ){ + yygotominor.yy346.pExpr->affinity = OE_Ignore; } - - /* Now that it is known how many phrases there are, allocate and zero - ** the required space using malloc(). - */ - nByte = sizeof(SnippetPhrase) * nList; - sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte); - if( !sIter.aPhrase ){ - return SQLITE_NOMEM; + yygotominor.yy346.zStart = yymsp[-3].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; +} + break; + case 289: /* expr ::= RAISE LP raisetype COMMA nm RP */ +{ + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); + if( yygotominor.yy346.pExpr ) { + yygotominor.yy346.pExpr->affinity = (char)yymsp[-3].minor.yy328; } - memset(sIter.aPhrase, 0, nByte); - - /* Initialize the contents of the SnippetIter object. Then iterate through - ** the set of phrases in the expression to populate the aPhrase[] array. - */ - sIter.pCsr = pCsr; - sIter.iCol = iCol; - sIter.nSnippet = nSnippet; - sIter.nPhrase = nList; - sIter.iCurrent = -1; - rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); - if( rc==SQLITE_OK ){ - - /* Set the *pmSeen output variable. */ - for(i=0; iiCol = iCol; - while( !fts3SnippetNextCandidate(&sIter) ){ - int iPos; - int iScore; - u64 mCover; - u64 mHighlite; - fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite); - assert( iScore>=0 ); - if( iScore>iBestScore ){ - pFragment->iPos = iPos; - pFragment->hlmask = mHighlite; - pFragment->covered = mCover; - iBestScore = iScore; - } + yygotominor.yy346.zStart = yymsp[-5].minor.yy0.z; + yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; +} + break; + case 290: /* raisetype ::= ROLLBACK */ +{yygotominor.yy328 = OE_Rollback;} + break; + case 292: /* raisetype ::= FAIL */ +{yygotominor.yy328 = OE_Fail;} + break; + case 293: /* cmd ::= DROP TRIGGER ifexists fullname */ +{ + sqlite3DropTrigger(pParse,yymsp[0].minor.yy65,yymsp[-1].minor.yy328); +} + break; + case 294: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ +{ + sqlite3Attach(pParse, yymsp[-3].minor.yy346.pExpr, yymsp[-1].minor.yy346.pExpr, yymsp[0].minor.yy132); +} + break; + case 295: /* cmd ::= DETACH database_kw_opt expr */ +{ + sqlite3Detach(pParse, yymsp[0].minor.yy346.pExpr); +} + break; + case 300: /* cmd ::= REINDEX */ +{sqlite3Reindex(pParse, 0, 0);} + break; + case 301: /* cmd ::= REINDEX nm dbnm */ +{sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} + break; + case 302: /* cmd ::= ANALYZE */ +{sqlite3Analyze(pParse, 0, 0);} + break; + case 303: /* cmd ::= ANALYZE nm dbnm */ +{sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} + break; + case 304: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ +{ + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy65,&yymsp[0].minor.yy0); +} + break; + case 305: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ +{ + sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy0); +} + break; + case 306: /* add_column_fullname ::= fullname */ +{ + pParse->db->lookaside.bEnabled = 0; + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy65); +} + break; + case 309: /* cmd ::= create_vtab */ +{sqlite3VtabFinishParse(pParse,0);} + break; + case 310: /* cmd ::= create_vtab LP vtabarglist RP */ +{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} + break; + case 311: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ +{ + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy328); +} + break; + case 314: /* vtabarg ::= */ +{sqlite3VtabArgInit(pParse);} + break; + case 316: /* vtabargtoken ::= ANY */ + case 317: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==317); + case 318: /* lp ::= LP */ yytestcase(yyruleno==318); +{sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} + break; + case 322: /* with ::= */ +{yygotominor.yy59 = 0;} + break; + case 323: /* with ::= WITH wqlist */ + case 324: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==324); +{ yygotominor.yy59 = yymsp[0].minor.yy59; } + break; + case 325: /* wqlist ::= nm idxlist_opt AS LP select RP */ +{ + yygotominor.yy59 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); +} + break; + case 326: /* wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP */ +{ + yygotominor.yy59 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy59, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); +} + break; + default: + /* (0) input ::= cmdlist */ yytestcase(yyruleno==0); + /* (1) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==1); + /* (2) cmdlist ::= ecmd */ yytestcase(yyruleno==2); + /* (3) ecmd ::= SEMI */ yytestcase(yyruleno==3); + /* (4) ecmd ::= explain cmdx SEMI */ yytestcase(yyruleno==4); + /* (10) trans_opt ::= */ yytestcase(yyruleno==10); + /* (11) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==11); + /* (12) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==12); + /* (20) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==20); + /* (21) savepoint_opt ::= */ yytestcase(yyruleno==21); + /* (25) cmd ::= create_table create_table_args */ yytestcase(yyruleno==25); + /* (36) columnlist ::= columnlist COMMA column */ yytestcase(yyruleno==36); + /* (37) columnlist ::= column */ yytestcase(yyruleno==37); + /* (43) type ::= */ yytestcase(yyruleno==43); + /* (50) signed ::= plus_num */ yytestcase(yyruleno==50); + /* (51) signed ::= minus_num */ yytestcase(yyruleno==51); + /* (52) carglist ::= carglist ccons */ yytestcase(yyruleno==52); + /* (53) carglist ::= */ yytestcase(yyruleno==53); + /* (60) ccons ::= NULL onconf */ yytestcase(yyruleno==60); + /* (88) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==88); + /* (89) conslist ::= tcons */ yytestcase(yyruleno==89); + /* (91) tconscomma ::= */ yytestcase(yyruleno==91); + /* (273) foreach_clause ::= */ yytestcase(yyruleno==273); + /* (274) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==274); + /* (281) tridxby ::= */ yytestcase(yyruleno==281); + /* (298) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==298); + /* (299) database_kw_opt ::= */ yytestcase(yyruleno==299); + /* (307) kwcolumn_opt ::= */ yytestcase(yyruleno==307); + /* (308) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==308); + /* (312) vtabarglist ::= vtabarg */ yytestcase(yyruleno==312); + /* (313) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==313); + /* (315) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==315); + /* (319) anylist ::= */ yytestcase(yyruleno==319); + /* (320) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==320); + /* (321) anylist ::= anylist ANY */ yytestcase(yyruleno==321); + break; + }; + assert( yyruleno>=0 && yyrulenoyyidx -= yysize; + yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); + if( yyact < YYNSTATE ){ +#ifdef NDEBUG + /* If we are not debugging and the reduce action popped at least + ** one element off the stack, then we can push the new element back + ** onto the stack here, and skip the stack overflow test in yy_shift(). + ** That gives a significant speed improvement. */ + if( yysize ){ + yypParser->yyidx++; + yymsp -= yysize-1; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yymsp->minor = yygotominor; + }else +#endif + { + yy_shift(yypParser,yyact,yygoto,&yygotominor); } - - *piScore = iBestScore; + }else{ + assert( yyact == YYNSTATE + YYNRULE + 1 ); + yy_accept(yypParser); } - sqlite3_free(sIter.aPhrase); - return rc; } - /* -** Append a string to the string-buffer passed as the first argument. -** -** If nAppend is negative, then the length of the string zAppend is -** determined using strlen(). +** The following code executes when the parse fails */ -static int fts3StringAppend( - StrBuffer *pStr, /* Buffer to append to */ - const char *zAppend, /* Pointer to data to append to buffer */ - int nAppend /* Size of zAppend in bytes (or -1) */ +#ifndef YYNOERRORRECOVERY +static void yy_parse_failed( + yyParser *yypParser /* The parser */ ){ - if( nAppend<0 ){ - nAppend = (int)strlen(zAppend); - } - - /* If there is insufficient space allocated at StrBuffer.z, use realloc() - ** to grow the buffer until so that it is big enough to accomadate the - ** appended data. - */ - if( pStr->n+nAppend+1>=pStr->nAlloc ){ - int nAlloc = pStr->nAlloc+nAppend+100; - char *zNew = sqlite3_realloc(pStr->z, nAlloc); - if( !zNew ){ - return SQLITE_NOMEM; - } - pStr->z = zNew; - pStr->nAlloc = nAlloc; + sqlite3ParserARG_FETCH; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); } - assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); - - /* Append the data to the string buffer. */ - memcpy(&pStr->z[pStr->n], zAppend, nAppend); - pStr->n += nAppend; - pStr->z[pStr->n] = '\0'; - - return SQLITE_OK; +#endif + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ + sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } +#endif /* YYNOERRORRECOVERY */ /* -** The fts3BestSnippet() function often selects snippets that end with a -** query term. That is, the final term of the snippet is always a term -** that requires highlighting. For example, if 'X' is a highlighted term -** and '.' is a non-highlighted term, BestSnippet() may select: -** -** ........X.....X -** -** This function "shifts" the beginning of the snippet forward in the -** document so that there are approximately the same number of -** non-highlighted terms to the right of the final highlighted term as there -** are to the left of the first highlighted term. For example, to this: -** -** ....X.....X.... -** -** This is done as part of extracting the snippet text, not when selecting -** the snippet. Snippet selection is done based on doclists only, so there -** is no way for fts3BestSnippet() to know whether or not the document -** actually contains terms that follow the final highlighted term. +** The following code executes when a syntax error first occurs. */ -static int fts3SnippetShift( - Fts3Table *pTab, /* FTS3 table snippet comes from */ - int iLangid, /* Language id to use in tokenizing */ - int nSnippet, /* Number of tokens desired for snippet */ - const char *zDoc, /* Document text to extract snippet from */ - int nDoc, /* Size of buffer zDoc in bytes */ - int *piPos, /* IN/OUT: First token of snippet */ - u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */ +static void yy_syntax_error( + yyParser *yypParser, /* The parser */ + int yymajor, /* The major type of the error token */ + YYMINORTYPE yyminor /* The minor type of the error token */ ){ - u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */ - - if( hlmask ){ - int nLeft; /* Tokens to the left of first highlight */ - int nRight; /* Tokens to the right of last highlight */ - int nDesired; /* Ideal number of tokens to shift forward */ - - for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); - for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); - nDesired = (nLeft-nRight)/2; - - /* Ideally, the start of the snippet should be pushed forward in the - ** document nDesired tokens. This block checks if there are actually - ** nDesired tokens to the right of the snippet. If so, *piPos and - ** *pHlMask are updated to shift the snippet nDesired tokens to the - ** right. Otherwise, the snippet is shifted by the number of tokens - ** available. - */ - if( nDesired>0 ){ - int nShift; /* Number of tokens to shift snippet by */ - int iCurrent = 0; /* Token counter */ - int rc; /* Return Code */ - sqlite3_tokenizer_module *pMod; - sqlite3_tokenizer_cursor *pC; - pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + sqlite3ParserARG_FETCH; +#define TOKEN (yyminor.yy0) - /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) - ** or more tokens in zDoc/nDoc. - */ - rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); - if( rc!=SQLITE_OK ){ - return rc; - } - while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ - const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; - rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); - } - pMod->xClose(pC); - if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } + UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ + assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} - nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; - assert( nShift<=nDesired ); - if( nShift>0 ){ - *piPos += nShift; - *pHlmask = hlmask >> nShift; - } - } +/* +** The following is executed when the parser accepts +*/ +static void yy_accept( + yyParser *yypParser /* The parser */ +){ + sqlite3ParserARG_FETCH; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); } - return SQLITE_OK; +#endif + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ + sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } -/* -** Extract the snippet text for fragment pFragment from cursor pCsr and -** append it to string buffer pOut. +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "sqlite3ParserAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +**
      +**
    • A pointer to the parser (an opaque structure.) +**
    • The major token number. +**
    • The minor token number. +**
    • An option argument of a grammar-specified type. +**
    +** +** Outputs: +** None. */ -static int fts3SnippetText( - Fts3Cursor *pCsr, /* FTS3 Cursor */ - SnippetFragment *pFragment, /* Snippet to extract */ - int iFragment, /* Fragment number */ - int isLast, /* True for final fragment in snippet */ - int nSnippet, /* Number of tokens in extracted snippet */ - const char *zOpen, /* String inserted before highlighted term */ - const char *zClose, /* String inserted after highlighted term */ - const char *zEllipsis, /* String inserted between snippets */ - StrBuffer *pOut /* Write output here */ +SQLITE_PRIVATE void sqlite3Parser( + void *yyp, /* The parser */ + int yymajor, /* The major token code number */ + sqlite3ParserTOKENTYPE yyminor /* The value for the token */ + sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc; /* Return code */ - const char *zDoc; /* Document text to extract snippet from */ - int nDoc; /* Size of zDoc in bytes */ - int iCurrent = 0; /* Current token number of document */ - int iEnd = 0; /* Byte offset of end of current token */ - int isShiftDone = 0; /* True after snippet is shifted */ - int iPos = pFragment->iPos; /* First token of snippet */ - u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */ - int iCol = pFragment->iCol+1; /* Query column to extract text from */ - sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ - sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ - - zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); - if( zDoc==0 ){ - if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ - return SQLITE_NOMEM; + YYMINORTYPE yyminorunion; + int yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef YYERRORSYMBOL + int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif + yyParser *yypParser; /* The parser */ + + /* (re)initialize the parser, if necessary */ + yypParser = (yyParser*)yyp; + if( yypParser->yyidx<0 ){ +#if YYSTACKDEPTH<=0 + if( yypParser->yystksz <=0 ){ + /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ + yyminorunion = yyzerominor; + yyStackOverflow(yypParser, &yyminorunion); + return; } - return SQLITE_OK; +#endif + yypParser->yyidx = 0; + yypParser->yyerrcnt = -1; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; } - nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol); + yyminorunion.yy0 = yyminor; +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + yyendofinput = (yymajor==0); +#endif + sqlite3ParserARG_STORE; - /* Open a token cursor on the document. */ - pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; - rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); - if( rc!=SQLITE_OK ){ - return rc; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]); } +#endif - while( rc==SQLITE_OK ){ - const char *ZDUMMY; /* Dummy argument used with tokenizer */ - int DUMMY1 = -1; /* Dummy argument used with tokenizer */ - int iBegin = 0; /* Offset in zDoc of start of token */ - int iFin = 0; /* Offset in zDoc of end of token */ - int isHighlight = 0; /* True for highlighted terms */ - - /* Variable DUMMY1 is initialized to a negative value above. Elsewhere - ** in the FTS code the variable that the third argument to xNext points to - ** is initialized to zero before the first (*but not necessarily - ** subsequent*) call to xNext(). This is done for a particular application - ** that needs to know whether or not the tokenizer is being used for - ** snippet generation or for some other purpose. - ** - ** Extreme care is required when writing code to depend on this - ** initialization. It is not a documented part of the tokenizer interface. - ** If a tokenizer is used directly by any code outside of FTS, this - ** convention might not be respected. */ - rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_DONE ){ - /* Special case - the last token of the snippet is also the last token - ** of the column. Append any punctuation that occurred between the end - ** of the previous token and the end of the document to the output. - ** Then break out of the loop. */ - rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); + do{ + yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); + if( yyactyyerrcnt--; + yymajor = YYNOCODE; + }else if( yyact < YYNSTATE + YYNRULE ){ + yy_reduce(yypParser,yyact-YYNSTATE); + }else{ + assert( yyact == YY_ERROR_ACTION ); +#ifdef YYERRORSYMBOL + int yymx; +#endif +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); } - break; - } - if( iCurrentiLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask - ); - isShiftDone = 1; - - /* Now that the shift has been done, check if the initial "..." are - ** required. They are required if (a) this is not the first fragment, - ** or (b) this fragment does not begin at position 0 of its column. +#endif +#ifdef YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** */ - if( rc==SQLITE_OK ){ - if( iPos>0 || iFragment>0 ){ - rc = fts3StringAppend(pOut, zEllipsis, -1); - }else if( iBegin ){ - rc = fts3StringAppend(pOut, zDoc, iBegin); + if( yypParser->yyerrcnt<0 ){ + yy_syntax_error(yypParser,yymajor,yyminorunion); + } + yymx = yypParser->yystack[yypParser->yyidx].major; + if( yymx==YYERRORSYMBOL || yyerrorhit ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sDiscard input token %s\n", + yyTracePrompt,yyTokenName[yymajor]); + } +#endif + yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion); + yymajor = YYNOCODE; + }else{ + while( + yypParser->yyidx >= 0 && + yymx != YYERRORSYMBOL && + (yyact = yy_find_reduce_action( + yypParser->yystack[yypParser->yyidx].stateno, + YYERRORSYMBOL)) >= YYNSTATE + ){ + yy_pop_parser_stack(yypParser); + } + if( yypParser->yyidx < 0 || yymajor==0 ){ + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yy_parse_failed(yypParser); + yymajor = YYNOCODE; + }else if( yymx!=YYERRORSYMBOL ){ + YYMINORTYPE u2; + u2.YYERRSYMDT = 0; + yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2); } } - if( rc!=SQLITE_OK || iCurrent=(iPos+nSnippet) ){ - if( isLast ){ - rc = fts3StringAppend(pOut, zEllipsis, -1); + yypParser->yyerrcnt = 3; + yyerrorhit = 1; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yymajor = YYNOCODE; + +#else /* YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( yypParser->yyerrcnt<=0 ){ + yy_syntax_error(yypParser,yymajor,yyminorunion); } - break; + yypParser->yyerrcnt = 3; + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + if( yyendofinput ){ + yy_parse_failed(yypParser); + } + yymajor = YYNOCODE; +#endif } - - /* Set isHighlight to true if this term should be highlighted. */ - isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; - - if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); - if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); - if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); - if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); - - iEnd = iFin; - } - - pMod->xClose(pC); - return rc; + }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); + return; } - +/************** End of parse.c ***********************************************/ +/************** Begin file tokenize.c ****************************************/ /* -** This function is used to count the entries in a column-list (a -** delta-encoded list of term offsets within a single column of a single -** row). When this function is called, *ppCollist should point to the -** beginning of the first varint in the column-list (the varint that -** contains the position of the first matching term in the column data). -** Before returning, *ppCollist is set to point to the first byte after -** the last varint in the column-list (either the 0x00 signifying the end -** of the position-list, or the 0x01 that precedes the column number of -** the next column in the position-list). +** 2001 September 15 ** -** The number of elements in the column-list is returned. -*/ -static int fts3ColumnlistCount(char **ppCollist){ - char *pEnd = *ppCollist; - char c = 0; - int nEntry = 0; - - /* A column-list is terminated by either a 0x01 or 0x00. */ - while( 0xFE & (*pEnd | c) ){ - c = *pEnd++ & 0x80; - if( !c ) nEntry++; - } - - *ppCollist = pEnd; - return nEntry; -} - -/* -** This function gathers 'y' or 'b' data for a single phrase. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that splits an SQL input string up into +** individual tokens and sends those tokens one-by-one over to the +** parser for analysis. */ -static void fts3ExprLHits( - Fts3Expr *pExpr, /* Phrase expression node */ - MatchInfo *p /* Matchinfo context */ -){ - Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; - int iStart; - Fts3Phrase *pPhrase = pExpr->pPhrase; - char *pIter = pPhrase->doclist.pList; - int iCol = 0; - - assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); - if( p->flag==FTS3_MATCHINFO_LHITS ){ - iStart = pExpr->iPhrase * p->nCol; - }else{ - iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); - } - - while( 1 ){ - int nHit = fts3ColumnlistCount(&pIter); - if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ - if( p->flag==FTS3_MATCHINFO_LHITS ){ - p->aMatchinfo[iStart + iCol] = (u32)nHit; - }else if( nHit ){ - p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); - } - } - assert( *pIter==0x00 || *pIter==0x01 ); - if( *pIter!=0x01 ) break; - pIter++; - pIter += fts3GetVarint32(pIter, &iCol); - } -} +/* #include */ /* -** Gather the results for matchinfo directives 'y' and 'b'. +** The charMap() macro maps alphabetic characters into their +** lower-case ASCII equivalent. On ASCII machines, this is just +** an upper-to-lower case map. On EBCDIC machines we also need +** to adjust the encoding. Only alphabetic characters and underscores +** need to be translated. */ -static void fts3ExprLHitGather( - Fts3Expr *pExpr, - MatchInfo *p -){ - assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); - if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ - if( pExpr->pLeft ){ - fts3ExprLHitGather(pExpr->pLeft, p); - fts3ExprLHitGather(pExpr->pRight, p); - }else{ - fts3ExprLHits(pExpr, p); - } - } -} +#ifdef SQLITE_ASCII +# define charMap(X) sqlite3UpperToLower[(unsigned char)X] +#endif +#ifdef SQLITE_EBCDIC +# define charMap(X) ebcdicToAscii[(unsigned char)X] +const unsigned char ebcdicToAscii[] = { +/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ +}; +#endif /* -** fts3ExprIterate() callback used to collect the "global" matchinfo stats -** for a single query. -** -** fts3ExprIterate() callback to load the 'global' elements of a -** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements -** of the matchinfo array that are constant for all rows returned by the -** current query. +** The sqlite3KeywordCode function looks up an identifier to determine if +** it is a keyword. If it is a keyword, the token code of that keyword is +** returned. If the input is not a keyword, TK_ID is returned. ** -** Argument pCtx is actually a pointer to a struct of type MatchInfo. This -** function populates Matchinfo.aMatchinfo[] as follows: +** The implementation of this routine was generated by a program, +** mkkeywordhash.h, located in the tool subdirectory of the distribution. +** The output of the mkkeywordhash.c program is written into a file +** named keywordhash.h and then included into this source file by +** the #include below. +*/ +/************** Include keywordhash.h in the middle of tokenize.c ************/ +/************** Begin file keywordhash.h *************************************/ +/***** This file contains automatically generated code ****** ** -** for(iCol=0; iColpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] - ); -} - -/* -** fts3ExprIterate() callback used to collect the "local" part of the -** FTS3_MATCHINFO_HITS array. The local stats are those elements of the -** array that are different for each row returned by the query. +** The code in this file implements a function that determines whether +** or not a given identifier is really an SQL keyword. The same thing +** might be implemented more directly using a hand-written hash table. +** But by using this automatically generated code, the size of the code +** is substantially reduced. This is important for embedded applications +** on platforms with limited memory. */ -static int fts3ExprLocalHitsCb( - Fts3Expr *pExpr, /* Phrase expression node */ - int iPhrase, /* Phrase number */ - void *pCtx /* Pointer to MatchInfo structure */ -){ - int rc = SQLITE_OK; - MatchInfo *p = (MatchInfo *)pCtx; - int iStart = iPhrase * p->nCol * 3; - int i; - - for(i=0; inCol && rc==SQLITE_OK; i++){ - char *pCsr; - rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); - if( pCsr ){ - p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); - }else{ - p->aMatchinfo[iStart+i*3] = 0; +/* Hash score: 182 */ +static int keywordCode(const char *z, int n){ + /* zText[] encodes 834 bytes of keywords in 554 bytes */ + /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ + /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ + /* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */ + /* UNIQUERYWITHOUTERELEASEATTACHAVINGROUPDATEBEGINNERECURSIVE */ + /* BETWEENOTNULLIKECASCADELETECASECOLLATECREATECURRENT_DATEDETACH */ + /* IMMEDIATEJOINSERTMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMITWHEN */ + /* WHERENAMEAFTEREPLACEANDEFAULTAUTOINCREMENTCASTCOLUMNCOMMIT */ + /* CONFLICTCROSSCURRENT_TIMESTAMPRIMARYDEFERREDISTINCTDROPFAIL */ + /* FROMFULLGLOBYIFISNULLORDERESTRICTRIGHTROLLBACKROWUNIONUSING */ + /* VACUUMVIEWINITIALLY */ + static const char zText[553] = { + 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', + 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', + 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', + 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', + 'E','R','R','A','B','L','E','L','S','E','X','C','E','P','T','R','A','N', + 'S','A','C','T','I','O','N','A','T','U','R','A','L','T','E','R','A','I', + 'S','E','X','C','L','U','S','I','V','E','X','I','S','T','S','A','V','E', + 'P','O','I','N','T','E','R','S','E','C','T','R','I','G','G','E','R','E', + 'F','E','R','E','N','C','E','S','C','O','N','S','T','R','A','I','N','T', + 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q', + 'U','E','R','Y','W','I','T','H','O','U','T','E','R','E','L','E','A','S', + 'E','A','T','T','A','C','H','A','V','I','N','G','R','O','U','P','D','A', + 'T','E','B','E','G','I','N','N','E','R','E','C','U','R','S','I','V','E', + 'B','E','T','W','E','E','N','O','T','N','U','L','L','I','K','E','C','A', + 'S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L','A', + 'T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D','A', + 'T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E','J', + 'O','I','N','S','E','R','T','M','A','T','C','H','P','L','A','N','A','L', + 'Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U','E', + 'S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','W','H', + 'E','R','E','N','A','M','E','A','F','T','E','R','E','P','L','A','C','E', + 'A','N','D','E','F','A','U','L','T','A','U','T','O','I','N','C','R','E', + 'M','E','N','T','C','A','S','T','C','O','L','U','M','N','C','O','M','M', + 'I','T','C','O','N','F','L','I','C','T','C','R','O','S','S','C','U','R', + 'R','E','N','T','_','T','I','M','E','S','T','A','M','P','R','I','M','A', + 'R','Y','D','E','F','E','R','R','E','D','I','S','T','I','N','C','T','D', + 'R','O','P','F','A','I','L','F','R','O','M','F','U','L','L','G','L','O', + 'B','Y','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S','T', + 'R','I','C','T','R','I','G','H','T','R','O','L','L','B','A','C','K','R', + 'O','W','U','N','I','O','N','U','S','I','N','G','V','A','C','U','U','M', + 'V','I','E','W','I','N','I','T','I','A','L','L','Y', + }; + static const unsigned char aHash[127] = { + 76, 105, 117, 74, 0, 45, 0, 0, 82, 0, 77, 0, 0, + 42, 12, 78, 15, 0, 116, 85, 54, 112, 0, 19, 0, 0, + 121, 0, 119, 115, 0, 22, 93, 0, 9, 0, 0, 70, 71, + 0, 69, 6, 0, 48, 90, 102, 0, 118, 101, 0, 0, 44, + 0, 103, 24, 0, 17, 0, 122, 53, 23, 0, 5, 110, 25, + 96, 0, 0, 124, 106, 60, 123, 57, 28, 55, 0, 91, 0, + 100, 26, 0, 99, 0, 0, 0, 95, 92, 97, 88, 109, 14, + 39, 108, 0, 81, 0, 18, 89, 111, 32, 0, 120, 80, 113, + 62, 46, 84, 0, 0, 94, 40, 59, 114, 0, 36, 0, 0, + 29, 0, 86, 63, 64, 0, 20, 61, 0, 56, + }; + static const unsigned char aNext[124] = { + 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, + 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 0, 0, 50, + 0, 43, 3, 47, 0, 0, 0, 0, 30, 0, 58, 0, 38, + 0, 0, 0, 1, 66, 0, 0, 67, 0, 41, 0, 0, 0, + 0, 0, 0, 49, 65, 0, 0, 0, 0, 31, 52, 16, 34, + 10, 0, 0, 0, 0, 0, 0, 0, 11, 72, 79, 0, 8, + 0, 104, 98, 0, 107, 0, 87, 0, 75, 51, 0, 27, 37, + 73, 83, 0, 35, 68, 0, 0, + }; + static const unsigned char aLen[124] = { + 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, + 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, + 11, 6, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10, + 4, 6, 2, 3, 9, 4, 2, 6, 5, 7, 4, 5, 7, + 6, 6, 5, 6, 5, 5, 9, 7, 7, 3, 2, 4, 4, + 7, 3, 6, 4, 7, 6, 12, 6, 9, 4, 6, 5, 4, + 7, 6, 5, 6, 7, 5, 4, 5, 6, 5, 7, 3, 7, + 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, 7, 8, 8, + 2, 4, 4, 4, 4, 4, 2, 2, 6, 5, 8, 5, 8, + 3, 5, 5, 6, 4, 9, 3, + }; + static const unsigned short int aOffset[124] = { + 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, + 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, + 86, 91, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152, + 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 184, 188, 192, + 199, 204, 209, 212, 218, 221, 225, 234, 240, 240, 240, 243, 246, + 250, 251, 255, 261, 265, 272, 278, 290, 296, 305, 307, 313, 318, + 320, 327, 332, 337, 343, 349, 354, 358, 361, 367, 371, 378, 380, + 387, 389, 391, 400, 404, 410, 416, 424, 429, 429, 445, 452, 459, + 460, 467, 471, 475, 479, 483, 486, 488, 490, 496, 500, 508, 513, + 521, 524, 529, 534, 540, 544, 549, + }; + static const unsigned char aCode[124] = { + TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, + TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, + TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, + TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, + TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, + TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW, + TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_SAVEPOINT, + TK_INTERSECT, TK_TRIGGER, TK_REFERENCES, TK_CONSTRAINT, TK_INTO, + TK_OFFSET, TK_OF, TK_SET, TK_TEMP, TK_TEMP, + TK_OR, TK_UNIQUE, TK_QUERY, TK_WITHOUT, TK_WITH, + TK_JOIN_KW, TK_RELEASE, TK_ATTACH, TK_HAVING, TK_GROUP, + TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RECURSIVE, TK_BETWEEN, + TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, TK_LIKE_KW, + TK_CASCADE, TK_ASC, TK_DELETE, TK_CASE, TK_COLLATE, + TK_CREATE, TK_CTIME_KW, TK_DETACH, TK_IMMEDIATE, TK_JOIN, + TK_INSERT, TK_MATCH, TK_PLAN, TK_ANALYZE, TK_PRAGMA, + TK_ABORT, TK_VALUES, TK_VIRTUAL, TK_LIMIT, TK_WHEN, + TK_WHERE, TK_RENAME, TK_AFTER, TK_REPLACE, TK_AND, + TK_DEFAULT, TK_AUTOINCR, TK_TO, TK_IN, TK_CAST, + TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, + TK_CTIME_KW, TK_PRIMARY, TK_DEFERRED, TK_DISTINCT, TK_IS, + TK_DROP, TK_FAIL, TK_FROM, TK_JOIN_KW, TK_LIKE_KW, + TK_BY, TK_IF, TK_ISNULL, TK_ORDER, TK_RESTRICT, + TK_JOIN_KW, TK_ROLLBACK, TK_ROW, TK_UNION, TK_USING, + TK_VACUUM, TK_VIEW, TK_INITIALLY, TK_ALL, + }; + int h, i; + if( n<2 ) return TK_ID; + h = ((charMap(z[0])*4) ^ + (charMap(z[n-1])*3) ^ + n) % 127; + for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){ + if( aLen[i]==n && sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){ + testcase( i==0 ); /* REINDEX */ + testcase( i==1 ); /* INDEXED */ + testcase( i==2 ); /* INDEX */ + testcase( i==3 ); /* DESC */ + testcase( i==4 ); /* ESCAPE */ + testcase( i==5 ); /* EACH */ + testcase( i==6 ); /* CHECK */ + testcase( i==7 ); /* KEY */ + testcase( i==8 ); /* BEFORE */ + testcase( i==9 ); /* FOREIGN */ + testcase( i==10 ); /* FOR */ + testcase( i==11 ); /* IGNORE */ + testcase( i==12 ); /* REGEXP */ + testcase( i==13 ); /* EXPLAIN */ + testcase( i==14 ); /* INSTEAD */ + testcase( i==15 ); /* ADD */ + testcase( i==16 ); /* DATABASE */ + testcase( i==17 ); /* AS */ + testcase( i==18 ); /* SELECT */ + testcase( i==19 ); /* TABLE */ + testcase( i==20 ); /* LEFT */ + testcase( i==21 ); /* THEN */ + testcase( i==22 ); /* END */ + testcase( i==23 ); /* DEFERRABLE */ + testcase( i==24 ); /* ELSE */ + testcase( i==25 ); /* EXCEPT */ + testcase( i==26 ); /* TRANSACTION */ + testcase( i==27 ); /* ACTION */ + testcase( i==28 ); /* ON */ + testcase( i==29 ); /* NATURAL */ + testcase( i==30 ); /* ALTER */ + testcase( i==31 ); /* RAISE */ + testcase( i==32 ); /* EXCLUSIVE */ + testcase( i==33 ); /* EXISTS */ + testcase( i==34 ); /* SAVEPOINT */ + testcase( i==35 ); /* INTERSECT */ + testcase( i==36 ); /* TRIGGER */ + testcase( i==37 ); /* REFERENCES */ + testcase( i==38 ); /* CONSTRAINT */ + testcase( i==39 ); /* INTO */ + testcase( i==40 ); /* OFFSET */ + testcase( i==41 ); /* OF */ + testcase( i==42 ); /* SET */ + testcase( i==43 ); /* TEMPORARY */ + testcase( i==44 ); /* TEMP */ + testcase( i==45 ); /* OR */ + testcase( i==46 ); /* UNIQUE */ + testcase( i==47 ); /* QUERY */ + testcase( i==48 ); /* WITHOUT */ + testcase( i==49 ); /* WITH */ + testcase( i==50 ); /* OUTER */ + testcase( i==51 ); /* RELEASE */ + testcase( i==52 ); /* ATTACH */ + testcase( i==53 ); /* HAVING */ + testcase( i==54 ); /* GROUP */ + testcase( i==55 ); /* UPDATE */ + testcase( i==56 ); /* BEGIN */ + testcase( i==57 ); /* INNER */ + testcase( i==58 ); /* RECURSIVE */ + testcase( i==59 ); /* BETWEEN */ + testcase( i==60 ); /* NOTNULL */ + testcase( i==61 ); /* NOT */ + testcase( i==62 ); /* NO */ + testcase( i==63 ); /* NULL */ + testcase( i==64 ); /* LIKE */ + testcase( i==65 ); /* CASCADE */ + testcase( i==66 ); /* ASC */ + testcase( i==67 ); /* DELETE */ + testcase( i==68 ); /* CASE */ + testcase( i==69 ); /* COLLATE */ + testcase( i==70 ); /* CREATE */ + testcase( i==71 ); /* CURRENT_DATE */ + testcase( i==72 ); /* DETACH */ + testcase( i==73 ); /* IMMEDIATE */ + testcase( i==74 ); /* JOIN */ + testcase( i==75 ); /* INSERT */ + testcase( i==76 ); /* MATCH */ + testcase( i==77 ); /* PLAN */ + testcase( i==78 ); /* ANALYZE */ + testcase( i==79 ); /* PRAGMA */ + testcase( i==80 ); /* ABORT */ + testcase( i==81 ); /* VALUES */ + testcase( i==82 ); /* VIRTUAL */ + testcase( i==83 ); /* LIMIT */ + testcase( i==84 ); /* WHEN */ + testcase( i==85 ); /* WHERE */ + testcase( i==86 ); /* RENAME */ + testcase( i==87 ); /* AFTER */ + testcase( i==88 ); /* REPLACE */ + testcase( i==89 ); /* AND */ + testcase( i==90 ); /* DEFAULT */ + testcase( i==91 ); /* AUTOINCREMENT */ + testcase( i==92 ); /* TO */ + testcase( i==93 ); /* IN */ + testcase( i==94 ); /* CAST */ + testcase( i==95 ); /* COLUMN */ + testcase( i==96 ); /* COMMIT */ + testcase( i==97 ); /* CONFLICT */ + testcase( i==98 ); /* CROSS */ + testcase( i==99 ); /* CURRENT_TIMESTAMP */ + testcase( i==100 ); /* CURRENT_TIME */ + testcase( i==101 ); /* PRIMARY */ + testcase( i==102 ); /* DEFERRED */ + testcase( i==103 ); /* DISTINCT */ + testcase( i==104 ); /* IS */ + testcase( i==105 ); /* DROP */ + testcase( i==106 ); /* FAIL */ + testcase( i==107 ); /* FROM */ + testcase( i==108 ); /* FULL */ + testcase( i==109 ); /* GLOB */ + testcase( i==110 ); /* BY */ + testcase( i==111 ); /* IF */ + testcase( i==112 ); /* ISNULL */ + testcase( i==113 ); /* ORDER */ + testcase( i==114 ); /* RESTRICT */ + testcase( i==115 ); /* RIGHT */ + testcase( i==116 ); /* ROLLBACK */ + testcase( i==117 ); /* ROW */ + testcase( i==118 ); /* UNION */ + testcase( i==119 ); /* USING */ + testcase( i==120 ); /* VACUUM */ + testcase( i==121 ); /* VIEW */ + testcase( i==122 ); /* INITIALLY */ + testcase( i==123 ); /* ALL */ + return aCode[i]; } } - - return rc; -} - -static int fts3MatchinfoCheck( - Fts3Table *pTab, - char cArg, - char **pzErr -){ - if( (cArg==FTS3_MATCHINFO_NPHRASE) - || (cArg==FTS3_MATCHINFO_NCOL) - || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) - || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) - || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) - || (cArg==FTS3_MATCHINFO_LCS) - || (cArg==FTS3_MATCHINFO_HITS) - || (cArg==FTS3_MATCHINFO_LHITS) - || (cArg==FTS3_MATCHINFO_LHITS_BM) - ){ - return SQLITE_OK; - } - sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); - return SQLITE_ERROR; + return TK_ID; } - -static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ - int nVal; /* Number of integers output by cArg */ - - switch( cArg ){ - case FTS3_MATCHINFO_NDOC: - case FTS3_MATCHINFO_NPHRASE: - case FTS3_MATCHINFO_NCOL: - nVal = 1; - break; - - case FTS3_MATCHINFO_AVGLENGTH: - case FTS3_MATCHINFO_LENGTH: - case FTS3_MATCHINFO_LCS: - nVal = pInfo->nCol; - break; - - case FTS3_MATCHINFO_LHITS: - nVal = pInfo->nCol * pInfo->nPhrase; - break; - - case FTS3_MATCHINFO_LHITS_BM: - nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); - break; - - default: - assert( cArg==FTS3_MATCHINFO_HITS ); - nVal = pInfo->nCol * pInfo->nPhrase * 3; - break; - } - - return nVal; +SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){ + return keywordCode((char*)z, n); } +#define SQLITE_N_KEYWORD 124 -static int fts3MatchinfoSelectDoctotal( - Fts3Table *pTab, - sqlite3_stmt **ppStmt, - sqlite3_int64 *pnDoc, - const char **paLen -){ - sqlite3_stmt *pStmt; - const char *a; - sqlite3_int64 nDoc; - - if( !*ppStmt ){ - int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); - if( rc!=SQLITE_OK ) return rc; - } - pStmt = *ppStmt; - assert( sqlite3_data_count(pStmt)==1 ); - - a = sqlite3_column_blob(pStmt, 0); - a += sqlite3Fts3GetVarint(a, &nDoc); - if( nDoc==0 ) return FTS_CORRUPT_VTAB; - *pnDoc = (u32)nDoc; +/************** End of keywordhash.h *****************************************/ +/************** Continuing where we left off in tokenize.c *******************/ - if( paLen ) *paLen = a; - return SQLITE_OK; -} /* -** An instance of the following structure is used to store state while -** iterating through a multi-column position-list corresponding to the -** hits for a single phrase on a single row in order to calculate the -** values for a matchinfo() FTS3_MATCHINFO_LCS request. +** If X is a character that can be used in an identifier then +** IdChar(X) will be true. Otherwise it is false. +** +** For ASCII, any character with the high-order bit set is +** allowed in an identifier. For 7-bit characters, +** sqlite3IsIdChar[X] must be 1. +** +** For EBCDIC, the rules are more complex but have the same +** end result. +** +** Ticket #1066. the SQL standard does not allow '$' in the +** middle of identifiers. But many SQL implementations do. +** SQLite will allow '$' in identifiers for compatibility. +** But the feature is undocumented. */ -typedef struct LcsIterator LcsIterator; -struct LcsIterator { - Fts3Expr *pExpr; /* Pointer to phrase expression */ - int iPosOffset; /* Tokens count up to end of this phrase */ - char *pRead; /* Cursor used to iterate through aDoclist */ - int iPos; /* Current position */ +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */ + 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */ }; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif +SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } -/* -** If LcsIterator.iCol is set to the following value, the iterator has -** finished iterating through all offsets for all columns. -*/ -#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; - -static int fts3MatchinfoLcsCb( - Fts3Expr *pExpr, /* Phrase expression node */ - int iPhrase, /* Phrase number (numbered from zero) */ - void *pCtx /* Pointer to MatchInfo structure */ -){ - LcsIterator *aIter = (LcsIterator *)pCtx; - aIter[iPhrase].pExpr = pExpr; - return SQLITE_OK; -} - -/* -** Advance the iterator passed as an argument to the next position. Return -** 1 if the iterator is at EOF or if it now points to the start of the -** position list for the next column. -*/ -static int fts3LcsIteratorAdvance(LcsIterator *pIter){ - char *pRead = pIter->pRead; - sqlite3_int64 iRead; - int rc = 0; - - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - if( iRead==0 || iRead==1 ){ - pRead = 0; - rc = 1; - }else{ - pIter->iPos += (int)(iRead-2); - } - pIter->pRead = pRead; - return rc; -} - /* -** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. -** -** If the call is successful, the longest-common-substring lengths for each -** column are written into the first nCol elements of the pInfo->aMatchinfo[] -** array before returning. SQLITE_OK is returned in this case. -** -** Otherwise, if an error occurs, an SQLite error code is returned and the -** data written to the first nCol elements of pInfo->aMatchinfo[] is -** undefined. +** Return the length of the token that begins at z[0]. +** Store the token type in *tokenType before returning. */ -static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ - LcsIterator *aIter; - int i; - int iCol; - int nToken = 0; - - /* Allocate and populate the array of LcsIterator objects. The array - ** contains one element for each matchable phrase in the query. - **/ - aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase); - if( !aIter ) return SQLITE_NOMEM; - memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); - (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); - - for(i=0; inPhrase; i++){ - LcsIterator *pIter = &aIter[i]; - nToken -= pIter->pExpr->pPhrase->nToken; - pIter->iPosOffset = nToken; - } - - for(iCol=0; iColnCol; iCol++){ - int nLcs = 0; /* LCS value for this column */ - int nLive = 0; /* Number of iterators in aIter not at EOF */ - - for(i=0; inPhrase; i++){ - int rc; - LcsIterator *pIt = &aIter[i]; - rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); - if( rc!=SQLITE_OK ) return rc; - if( pIt->pRead ){ - pIt->iPos = pIt->iPosOffset; - fts3LcsIteratorAdvance(&aIter[i]); - nLive++; +SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ + int i, c; + switch( *z ){ + case ' ': case '\t': case '\n': case '\f': case '\r': { + testcase( z[0]==' ' ); + testcase( z[0]=='\t' ); + testcase( z[0]=='\n' ); + testcase( z[0]=='\f' ); + testcase( z[0]=='\r' ); + for(i=1; sqlite3Isspace(z[i]); i++){} + *tokenType = TK_SPACE; + return i; + } + case '-': { + if( z[1]=='-' ){ + for(i=2; (c=z[i])!=0 && c!='\n'; i++){} + *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + return i; } + *tokenType = TK_MINUS; + return 1; } - - while( nLive>0 ){ - LcsIterator *pAdv = 0; /* The iterator to advance by one position */ - int nThisLcs = 0; /* LCS for the current iterator positions */ - - for(i=0; inPhrase; i++){ - LcsIterator *pIter = &aIter[i]; - if( pIter->pRead==0 ){ - /* This iterator is already at EOF for this column. */ - nThisLcs = 0; - }else{ - if( pAdv==0 || pIter->iPosiPos ){ - pAdv = pIter; - } - if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ - nThisLcs++; + case '(': { + *tokenType = TK_LP; + return 1; + } + case ')': { + *tokenType = TK_RP; + return 1; + } + case ';': { + *tokenType = TK_SEMI; + return 1; + } + case '+': { + *tokenType = TK_PLUS; + return 1; + } + case '*': { + *tokenType = TK_STAR; + return 1; + } + case '/': { + if( z[1]!='*' || z[2]==0 ){ + *tokenType = TK_SLASH; + return 1; + } + for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} + if( c ) i++; + *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + return i; + } + case '%': { + *tokenType = TK_REM; + return 1; + } + case '=': { + *tokenType = TK_EQ; + return 1 + (z[1]=='='); + } + case '<': { + if( (c=z[1])=='=' ){ + *tokenType = TK_LE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_NE; + return 2; + }else if( c=='<' ){ + *tokenType = TK_LSHIFT; + return 2; + }else{ + *tokenType = TK_LT; + return 1; + } + } + case '>': { + if( (c=z[1])=='=' ){ + *tokenType = TK_GE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_RSHIFT; + return 2; + }else{ + *tokenType = TK_GT; + return 1; + } + } + case '!': { + if( z[1]!='=' ){ + *tokenType = TK_ILLEGAL; + return 2; + }else{ + *tokenType = TK_NE; + return 2; + } + } + case '|': { + if( z[1]!='|' ){ + *tokenType = TK_BITOR; + return 1; + }else{ + *tokenType = TK_CONCAT; + return 2; + } + } + case ',': { + *tokenType = TK_COMMA; + return 1; + } + case '&': { + *tokenType = TK_BITAND; + return 1; + } + case '~': { + *tokenType = TK_BITNOT; + return 1; + } + case '`': + case '\'': + case '"': { + int delim = z[0]; + testcase( delim=='`' ); + testcase( delim=='\'' ); + testcase( delim=='"' ); + for(i=1; (c=z[i])!=0; i++){ + if( c==delim ){ + if( z[i+1]==delim ){ + i++; }else{ - nThisLcs = 1; + break; } - if( nThisLcs>nLcs ) nLcs = nThisLcs; } } - if( fts3LcsIteratorAdvance(pAdv) ) nLive--; + if( c=='\'' ){ + *tokenType = TK_STRING; + return i+1; + }else if( c!=0 ){ + *tokenType = TK_ID; + return i+1; + }else{ + *tokenType = TK_ILLEGAL; + return i; + } } - - pInfo->aMatchinfo[iCol] = nLcs; - } - - sqlite3_free(aIter); - return SQLITE_OK; -} - -/* -** Populate the buffer pInfo->aMatchinfo[] with an array of integers to -** be returned by the matchinfo() function. Argument zArg contains the -** format string passed as the second argument to matchinfo (or the -** default value "pcx" if no second argument was specified). The format -** string has already been validated and the pInfo->aMatchinfo[] array -** is guaranteed to be large enough for the output. -** -** If bGlobal is true, then populate all fields of the matchinfo() output. -** If it is false, then assume that those fields that do not change between -** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) -** have already been populated. -** -** Return SQLITE_OK if successful, or an SQLite error code if an error -** occurs. If a value other than SQLITE_OK is returned, the state the -** pInfo->aMatchinfo[] buffer is left in is undefined. -*/ -static int fts3MatchinfoValues( - Fts3Cursor *pCsr, /* FTS3 cursor object */ - int bGlobal, /* True to grab the global stats */ - MatchInfo *pInfo, /* Matchinfo context object */ - const char *zArg /* Matchinfo format string */ -){ - int rc = SQLITE_OK; - int i; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - sqlite3_stmt *pSelect = 0; - - for(i=0; rc==SQLITE_OK && zArg[i]; i++){ - pInfo->flag = zArg[i]; - switch( zArg[i] ){ - case FTS3_MATCHINFO_NPHRASE: - if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; - break; - - case FTS3_MATCHINFO_NCOL: - if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; - break; - - case FTS3_MATCHINFO_NDOC: - if( bGlobal ){ - sqlite3_int64 nDoc = 0; - rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); - pInfo->aMatchinfo[0] = (u32)nDoc; - } - break; - - case FTS3_MATCHINFO_AVGLENGTH: - if( bGlobal ){ - sqlite3_int64 nDoc; /* Number of rows in table */ - const char *a; /* Aggregate column length array */ - - rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); - if( rc==SQLITE_OK ){ - int iCol; - for(iCol=0; iColnCol; iCol++){ - u32 iVal; - sqlite3_int64 nToken; - a += sqlite3Fts3GetVarint(a, &nToken); - iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); - pInfo->aMatchinfo[iCol] = iVal; - } - } - } - break; - - case FTS3_MATCHINFO_LENGTH: { - sqlite3_stmt *pSelectDocsize = 0; - rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); - if( rc==SQLITE_OK ){ - int iCol; - const char *a = sqlite3_column_blob(pSelectDocsize, 0); - for(iCol=0; iColnCol; iCol++){ - sqlite3_int64 nToken; - a += sqlite3Fts3GetVarint(a, &nToken); - pInfo->aMatchinfo[iCol] = (u32)nToken; + case '.': { +#ifndef SQLITE_OMIT_FLOATING_POINT + if( !sqlite3Isdigit(z[1]) ) +#endif + { + *tokenType = TK_DOT; + return 1; + } + /* If the next character is a digit, this is a floating point + ** number that begins with ".". Fall thru into the next case */ + } + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': { + testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); + testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); + testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); + testcase( z[0]=='9' ); + *tokenType = TK_INTEGER; +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } +#endif + for(i=0; sqlite3Isdigit(z[i]); i++){} +#ifndef SQLITE_OMIT_FLOATING_POINT + if( z[i]=='.' ){ + i++; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + i += 2; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } +#endif + while( IdChar(z[i]) ){ + *tokenType = TK_ILLEGAL; + i++; + } + return i; + } + case '[': { + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} + *tokenType = c==']' ? TK_ID : TK_ILLEGAL; + return i; + } + case '?': { + *tokenType = TK_VARIABLE; + for(i=1; sqlite3Isdigit(z[i]); i++){} + return i; + } +#ifndef SQLITE_OMIT_TCL_VARIABLE + case '$': +#endif + case '@': /* For compatibility with MS SQL Server */ + case '#': + case ':': { + int n = 0; + testcase( z[0]=='$' ); testcase( z[0]=='@' ); + testcase( z[0]==':' ); testcase( z[0]=='#' ); + *tokenType = TK_VARIABLE; + for(i=1; (c=z[i])!=0; i++){ + if( IdChar(c) ){ + n++; +#ifndef SQLITE_OMIT_TCL_VARIABLE + }else if( c=='(' && n>0 ){ + do{ + i++; + }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); + if( c==')' ){ + i++; + }else{ + *tokenType = TK_ILLEGAL; } + break; + }else if( c==':' && z[i+1]==':' ){ + i++; +#endif + }else{ + break; } - sqlite3_reset(pSelectDocsize); - break; } - - case FTS3_MATCHINFO_LCS: - rc = fts3ExprLoadDoclists(pCsr, 0, 0); - if( rc==SQLITE_OK ){ - rc = fts3MatchinfoLcs(pCsr, pInfo); + if( n==0 ) *tokenType = TK_ILLEGAL; + return i; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case 'x': case 'X': { + testcase( z[0]=='x' ); testcase( z[0]=='X' ); + if( z[1]=='\'' ){ + *tokenType = TK_BLOB; + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } } - break; - - case FTS3_MATCHINFO_LHITS_BM: - case FTS3_MATCHINFO_LHITS: { - int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); - memset(pInfo->aMatchinfo, 0, nZero); - fts3ExprLHitGather(pCsr->pExpr, pInfo); - break; + if( z[i] ) i++; + return i; } - - default: { - Fts3Expr *pExpr; - assert( zArg[i]==FTS3_MATCHINFO_HITS ); - pExpr = pCsr->pExpr; - rc = fts3ExprLoadDoclists(pCsr, 0, 0); - if( rc!=SQLITE_OK ) break; - if( bGlobal ){ - if( pCsr->pDeferred ){ - rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); - if( rc!=SQLITE_OK ) break; - } - rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); - sqlite3Fts3EvalTestDeferred(pCsr, &rc); - if( rc!=SQLITE_OK ) break; - } - (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + /* Otherwise fall through to the next case */ + } +#endif + default: { + if( !IdChar(*z) ){ break; } + for(i=1; IdChar(z[i]); i++){} + *tokenType = keywordCode((char*)z, i); + return i; } - - pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); } - - sqlite3_reset(pSelect); - return rc; + *tokenType = TK_ILLEGAL; + return 1; } - /* -** Populate pCsr->aMatchinfo[] with data for the current row. The -** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). +** Run the parser on the given SQL string. The parser structure is +** passed in. An SQLITE_ status code is returned. If an error occurs +** then an and attempt is made to write an error message into +** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that +** error message. */ -static void fts3GetMatchinfo( - sqlite3_context *pCtx, /* Return results here */ - Fts3Cursor *pCsr, /* FTS3 Cursor object */ - const char *zArg /* Second argument to matchinfo() function */ -){ - MatchInfo sInfo; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; - int bGlobal = 0; /* Collect 'global' stats as well as local */ - - u32 *aOut = 0; - void (*xDestroyOut)(void*) = 0; - - memset(&sInfo, 0, sizeof(MatchInfo)); - sInfo.pCursor = pCsr; - sInfo.nCol = pTab->nColumn; +SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ + int nErr = 0; /* Number of errors encountered */ + int i; /* Loop counter */ + void *pEngine; /* The LEMON-generated LALR(1) parser */ + int tokenType; /* type of the next token */ + int lastTokenParsed = -1; /* type of the previous token */ + u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */ + sqlite3 *db = pParse->db; /* The database connection */ + int mxSqlLen; /* Max length of an SQL string */ - /* If there is cached matchinfo() data, but the format string for the - ** cache does not match the format string for this request, discard - ** the cached data. */ - if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ - sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); - pCsr->pMIBuffer = 0; + assert( zSql!=0 ); + mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + if( db->nVdbeActive==0 ){ + db->u1.isInterrupted = 0; } - - /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the - ** matchinfo function has been called for this query. In this case - ** allocate the array used to accumulate the matchinfo data and - ** initialize those elements that are constant for every row. - */ - if( pCsr->pMIBuffer==0 ){ - int nMatchinfo = 0; /* Number of u32 elements in match-info */ - int i; /* Used to iterate through zArg */ - - /* Determine the number of phrases in the query */ - pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); - sInfo.nPhrase = pCsr->nPhrase; - - /* Determine the number of integers in the buffer returned by this call. */ - for(i=0; zArg[i]; i++){ - char *zErr = 0; - if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ - sqlite3_result_error(pCtx, zErr, -1); - sqlite3_free(zErr); - return; + pParse->rc = SQLITE_OK; + pParse->zTail = zSql; + i = 0; + assert( pzErrMsg!=0 ); + pEngine = sqlite3ParserAlloc(sqlite3Malloc); + if( pEngine==0 ){ + db->mallocFailed = 1; + return SQLITE_NOMEM; + } + assert( pParse->pNewTable==0 ); + assert( pParse->pNewTrigger==0 ); + assert( pParse->nVar==0 ); + assert( pParse->nzVar==0 ); + assert( pParse->azVar==0 ); + enableLookaside = db->lookaside.bEnabled; + if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; + while( !db->mallocFailed && zSql[i]!=0 ){ + assert( i>=0 ); + pParse->sLastToken.z = &zSql[i]; + pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); + i += pParse->sLastToken.n; + if( i>mxSqlLen ){ + pParse->rc = SQLITE_TOOBIG; + break; + } + switch( tokenType ){ + case TK_SPACE: { + if( db->u1.isInterrupted ){ + sqlite3ErrorMsg(pParse, "interrupt"); + pParse->rc = SQLITE_INTERRUPT; + goto abort_parse; + } + break; + } + case TK_ILLEGAL: { + sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", + &pParse->sLastToken); + goto abort_parse; + } + case TK_SEMI: { + pParse->zTail = &zSql[i]; + /* Fall thru into the default case */ + } + default: { + sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); + lastTokenParsed = tokenType; + if( pParse->rc!=SQLITE_OK ){ + goto abort_parse; + } + break; } - nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); } - - /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ - pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); - if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; - - pCsr->isMatchinfoNeeded = 1; - bGlobal = 1; } - - if( rc==SQLITE_OK ){ - xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); - if( xDestroyOut==0 ){ - rc = SQLITE_NOMEM; +abort_parse: + assert( nErr==0 ); + if( zSql[i]==0 && pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ + if( lastTokenParsed!=TK_SEMI ){ + sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); + pParse->zTail = &zSql[i]; } - } - - if( rc==SQLITE_OK ){ - sInfo.aMatchinfo = aOut; - sInfo.nPhrase = pCsr->nPhrase; - rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); - if( bGlobal ){ - fts3MIBufferSetGlobal(pCsr->pMIBuffer); + if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ + sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); } } - - if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pCtx, rc); - if( xDestroyOut ) xDestroyOut(aOut); - }else{ - int n = pCsr->pMIBuffer->nElem * sizeof(u32); - sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); +#ifdef YYTRACKMAXSTACKDEPTH + sqlite3_mutex_enter(sqlite3MallocMutex()); + sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, + sqlite3ParserStackPeak(pEngine) + ); + sqlite3_mutex_leave(sqlite3MallocMutex()); +#endif /* YYDEBUG */ + sqlite3ParserFree(pEngine, sqlite3_free); + db->lookaside.bEnabled = enableLookaside; + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM; } -} - -/* -** Implementation of snippet() function. -*/ -SQLITE_PRIVATE void sqlite3Fts3Snippet( - sqlite3_context *pCtx, /* SQLite function call context */ - Fts3Cursor *pCsr, /* Cursor object */ - const char *zStart, /* Snippet start text - "" */ - const char *zEnd, /* Snippet end text - "" */ - const char *zEllipsis, /* Snippet ellipsis text - "..." */ - int iCol, /* Extract snippet from this column */ - int nToken /* Approximate number of tokens in snippet */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; - int i; - StrBuffer res = {0, 0, 0}; - - /* The returned text includes up to four fragments of text extracted from - ** the data in the current row. The first iteration of the for(...) loop - ** below attempts to locate a single fragment of text nToken tokens in - ** size that contains at least one instance of all phrases in the query - ** expression that appear in the current row. If such a fragment of text - ** cannot be found, the second iteration of the loop attempts to locate - ** a pair of fragments, and so on. - */ - int nSnippet = 0; /* Number of fragments in this snippet */ - SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */ - int nFToken = -1; /* Number of tokens in each fragment */ - - if( !pCsr->pExpr ){ - sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); - return; + if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ + sqlite3SetString(&pParse->zErrMsg, db, "%s", sqlite3ErrStr(pParse->rc)); } + assert( pzErrMsg!=0 ); + if( pParse->zErrMsg ){ + *pzErrMsg = pParse->zErrMsg; + sqlite3_log(pParse->rc, "%s", *pzErrMsg); + pParse->zErrMsg = 0; + nErr++; + } + if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ + sqlite3VdbeDelete(pParse->pVdbe); + pParse->pVdbe = 0; + } +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pParse->nested==0 ){ + sqlite3DbFree(db, pParse->aTableLock); + pParse->aTableLock = 0; + pParse->nTableLock = 0; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_free(pParse->apVtabLock); +#endif - for(nSnippet=1; 1; nSnippet++){ - - int iSnip; /* Loop counter 0..nSnippet-1 */ - u64 mCovered = 0; /* Bitmask of phrases covered by snippet */ - u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */ - - if( nToken>=0 ){ - nFToken = (nToken+nSnippet-1) / nSnippet; - }else{ - nFToken = -1 * nToken; - } - - for(iSnip=0; iSnipnColumn; iRead++){ - SnippetFragment sF = {0, 0, 0, 0}; - int iS = 0; - if( iCol>=0 && iRead!=iCol ) continue; - - /* Find the best snippet of nFToken tokens in column iRead. */ - rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); - if( rc!=SQLITE_OK ){ - goto snippet_out; - } - if( iS>iBestScore ){ - *pFragment = sF; - iBestScore = iS; - } - } - - mCovered |= pFragment->covered; - } - - /* If all query phrases seen by fts3BestSnippet() are present in at least - ** one of the nSnippet snippet fragments, break out of the loop. + if( !IN_DECLARE_VTAB ){ + /* If the pParse->declareVtab flag is set, do not delete any table + ** structure built up in pParse->pNewTable. The calling code (see vtab.c) + ** will take responsibility for freeing the Table structure. */ - assert( (mCovered&mSeen)==mCovered ); - if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break; + sqlite3DeleteTable(db, pParse->pNewTable); } - assert( nFToken>0 ); - - for(i=0; ibFreeWith ) sqlite3WithDelete(db, pParse->pWith); + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); + sqlite3DbFree(db, pParse->azVar); + while( pParse->pAinc ){ + AutoincInfo *p = pParse->pAinc; + pParse->pAinc = p->pNext; + sqlite3DbFree(db, p); } - - snippet_out: - sqlite3Fts3SegmentsClose(pTab); - if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pCtx, rc); - sqlite3_free(res.z); - }else{ - sqlite3_result_text(pCtx, res.z, -1, sqlite3_free); + while( pParse->pZombieTab ){ + Table *p = pParse->pZombieTab; + pParse->pZombieTab = p->pNextZombie; + sqlite3DeleteTable(db, p); } + assert( nErr==0 || pParse->rc!=SQLITE_OK ); + return nErr; } - -typedef struct TermOffset TermOffset; -typedef struct TermOffsetCtx TermOffsetCtx; - -struct TermOffset { - char *pList; /* Position-list */ - int iPos; /* Position just read from pList */ - int iOff; /* Offset of this term from read positions */ -}; - -struct TermOffsetCtx { - Fts3Cursor *pCsr; - int iCol; /* Column of table to populate aTerm for */ - int iTerm; - sqlite3_int64 iDocid; - TermOffset *aTerm; -}; - +/************** End of tokenize.c ********************************************/ +/************** Begin file complete.c ****************************************/ /* -** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets(). +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that implements the sqlite3_complete() API. +** This code used to be part of the tokenizer.c source file. But by +** separating it out, the code will be automatically omitted from +** static links that do not use it. */ -static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ - TermOffsetCtx *p = (TermOffsetCtx *)ctx; - int nTerm; /* Number of tokens in phrase */ - int iTerm; /* For looping through nTerm phrase terms */ - char *pList; /* Pointer to position list for phrase */ - int iPos = 0; /* First position in position-list */ - int rc; - - UNUSED_PARAMETER(iPhrase); - rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pList); - nTerm = pExpr->pPhrase->nToken; - if( pList ){ - fts3GetDeltaPosition(&pList, &iPos); - assert( iPos>=0 ); - } - - for(iTerm=0; iTermaTerm[p->iTerm++]; - pT->iOff = nTerm-iTerm-1; - pT->pList = pList; - pT->iPos = iPos; - } - - return rc; -} +#ifndef SQLITE_OMIT_COMPLETE /* -** Implementation of offsets() function. +** This is defined in tokenize.c. We just have to import the definition. */ -SQLITE_PRIVATE void sqlite3Fts3Offsets( - sqlite3_context *pCtx, /* SQLite function call context */ - Fts3Cursor *pCsr /* Cursor object */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; - int rc; /* Return Code */ - int nToken; /* Number of tokens in query */ - int iCol; /* Column currently being processed */ - StrBuffer res = {0, 0, 0}; /* Result string */ - TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */ - - if( !pCsr->pExpr ){ - sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); - return; - } +#ifndef SQLITE_AMALGAMATION +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif +#endif /* SQLITE_AMALGAMATION */ - memset(&sCtx, 0, sizeof(sCtx)); - assert( pCsr->isRequireSeek==0 ); - /* Count the number of terms in the query */ - rc = fts3ExprLoadDoclists(pCsr, 0, &nToken); - if( rc!=SQLITE_OK ) goto offsets_out; +/* +** Token types used by the sqlite3_complete() routine. See the header +** comments on that procedure for additional information. +*/ +#define tkSEMI 0 +#define tkWS 1 +#define tkOTHER 2 +#ifndef SQLITE_OMIT_TRIGGER +#define tkEXPLAIN 3 +#define tkCREATE 4 +#define tkTEMP 5 +#define tkTRIGGER 6 +#define tkEND 7 +#endif - /* Allocate the array of TermOffset iterators. */ - sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken); - if( 0==sCtx.aTerm ){ - rc = SQLITE_NOMEM; - goto offsets_out; - } - sCtx.iDocid = pCsr->iPrevId; - sCtx.pCsr = pCsr; +/* +** Return TRUE if the given SQL string ends in a semicolon. +** +** Special handling is require for CREATE TRIGGER statements. +** Whenever the CREATE TRIGGER keywords are seen, the statement +** must end with ";END;". +** +** This implementation uses a state machine with 8 states: +** +** (0) INVALID We have not yet seen a non-whitespace character. +** +** (1) START At the beginning or end of an SQL statement. This routine +** returns 1 if it ends in the START state and 0 if it ends +** in any other state. +** +** (2) NORMAL We are in the middle of statement which ends with a single +** semicolon. +** +** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of +** a statement. +** +** (4) CREATE The keyword CREATE has been seen at the beginning of a +** statement, possibly preceded by EXPLAIN and/or followed by +** TEMP or TEMPORARY +** +** (5) TRIGGER We are in the middle of a trigger definition that must be +** ended by a semicolon, the keyword END, and another semicolon. +** +** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at +** the end of a trigger definition. +** +** (7) END We've seen the ";END" of the ";END;" that occurs at the end +** of a trigger definition. +** +** Transitions between states above are determined by tokens extracted +** from the input. The following tokens are significant: +** +** (0) tkSEMI A semicolon. +** (1) tkWS Whitespace. +** (2) tkOTHER Any other SQL token. +** (3) tkEXPLAIN The "explain" keyword. +** (4) tkCREATE The "create" keyword. +** (5) tkTEMP The "temp" or "temporary" keyword. +** (6) tkTRIGGER The "trigger" keyword. +** (7) tkEND The "end" keyword. +** +** Whitespace never causes a state transition and is always ignored. +** This means that a SQL string of all whitespace is invalid. +** +** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed +** to recognize the end of a trigger can be omitted. All we have to do +** is look for a semicolon that is not part of an string or comment. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_complete(const char *zSql){ + u8 state = 0; /* Current state, using numbers defined in header comment */ + u8 token; /* Value of the next token */ - /* Loop through the table columns, appending offset information to - ** string-buffer res for each column. +#ifndef SQLITE_OMIT_TRIGGER + /* A complex statement machine used to detect the end of a CREATE TRIGGER + ** statement. This is the normal case. */ - for(iCol=0; iColnColumn; iCol++){ - sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ - const char *ZDUMMY; /* Dummy argument used with xNext() */ - int NDUMMY = 0; /* Dummy argument used with xNext() */ - int iStart = 0; - int iEnd = 0; - int iCurrent = 0; - const char *zDoc; - int nDoc; + static const u8 trans[8][8] = { + /* Token: */ + /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ + /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, + /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, + /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, + /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, + /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, + /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, + /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, + }; +#else + /* If triggers are not supported by this compile then the statement machine + ** used to detect the end of a statement is much simpler + */ + static const u8 trans[3][3] = { + /* Token: */ + /* State: ** SEMI WS OTHER */ + /* 0 INVALID: */ { 1, 0, 2, }, + /* 1 START: */ { 1, 1, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, }, + }; +#endif /* SQLITE_OMIT_TRIGGER */ - /* Initialize the contents of sCtx.aTerm[] for column iCol. There is - ** no way that this operation can fail, so the return code from - ** fts3ExprIterate() can be discarded. - */ - sCtx.iCol = iCol; - sCtx.iTerm = 0; - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif - /* Retreive the text stored in column iCol. If an SQL NULL is stored - ** in column iCol, jump immediately to the next iteration of the loop. - ** If an OOM occurs while retrieving the data (this can happen if SQLite - ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM - ** to the caller. - */ - zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); - nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); - if( zDoc==0 ){ - if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){ - continue; + while( *zSql ){ + switch( *zSql ){ + case ';': { /* A semicolon */ + token = tkSEMI; + break; } - rc = SQLITE_NOMEM; - goto offsets_out; - } - - /* Initialize a tokenizer iterator to iterate through column iCol. */ - rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, - zDoc, nDoc, &pC - ); - if( rc!=SQLITE_OK ) goto offsets_out; - - rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); - while( rc==SQLITE_OK ){ - int i; /* Used to loop through terms */ - int iMinPos = 0x7FFFFFFF; /* Position of next token */ - TermOffset *pTerm = 0; /* TermOffset associated with next token */ - - for(i=0; ipList && (pT->iPos-pT->iOff)iPos-pT->iOff; - pTerm = pT; - } + case ' ': + case '\r': + case '\t': + case '\n': + case '\f': { /* White space is ignored */ + token = tkWS; + break; } - - if( !pTerm ){ - /* All offsets for this column have been gathered. */ - rc = SQLITE_DONE; - }else{ - assert( iCurrent<=iMinPos ); - if( 0==(0xFE&*pTerm->pList) ){ - pTerm->pList = 0; - }else{ - fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos); + case '/': { /* C-style comments */ + if( zSql[1]!='*' ){ + token = tkOTHER; + break; } - while( rc==SQLITE_OK && iCurrentxNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + zSql += 2; + while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } + if( zSql[0]==0 ) return 0; + zSql++; + token = tkWS; + break; + } + case '-': { /* SQL-style comments from "--" to end of line */ + if( zSql[1]!='-' ){ + token = tkOTHER; + break; } - if( rc==SQLITE_OK ){ - char aBuffer[64]; - sqlite3_snprintf(sizeof(aBuffer), aBuffer, - "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart - ); - rc = fts3StringAppend(&res, aBuffer, -1); - }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ - rc = FTS_CORRUPT_VTAB; + while( *zSql && *zSql!='\n' ){ zSql++; } + if( *zSql==0 ) return state==1; + token = tkWS; + break; + } + case '[': { /* Microsoft-style identifiers in [...] */ + zSql++; + while( *zSql && *zSql!=']' ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + case '`': /* Grave-accent quoted symbols used by MySQL */ + case '"': /* single- and double-quoted strings */ + case '\'': { + int c = *zSql; + zSql++; + while( *zSql && *zSql!=c ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + default: { +#ifdef SQLITE_EBCDIC + unsigned char c; +#endif + if( IdChar((u8)*zSql) ){ + /* Keywords and unquoted identifiers */ + int nId; + for(nId=1; IdChar(zSql[nId]); nId++){} +#ifdef SQLITE_OMIT_TRIGGER + token = tkOTHER; +#else + switch( *zSql ){ + case 'c': case 'C': { + if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ + token = tkCREATE; + }else{ + token = tkOTHER; + } + break; + } + case 't': case 'T': { + if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ + token = tkTRIGGER; + }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ + token = tkTEMP; + }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ + token = tkTEMP; + }else{ + token = tkOTHER; + } + break; + } + case 'e': case 'E': { + if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ + token = tkEND; + }else +#ifndef SQLITE_OMIT_EXPLAIN + if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ + token = tkEXPLAIN; + }else +#endif + { + token = tkOTHER; + } + break; + } + default: { + token = tkOTHER; + break; + } + } +#endif /* SQLITE_OMIT_TRIGGER */ + zSql += nId-1; + }else{ + /* Operators and special symbols */ + token = tkOTHER; } + break; } } - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - } - - pMod->xClose(pC); - if( rc!=SQLITE_OK ) goto offsets_out; - } - - offsets_out: - sqlite3_free(sCtx.aTerm); - assert( rc!=SQLITE_DONE ); - sqlite3Fts3SegmentsClose(pTab); - if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pCtx, rc); - sqlite3_free(res.z); - }else{ - sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); + state = trans[state][token]; + zSql++; } - return; + return state==1; } +#ifndef SQLITE_OMIT_UTF16 /* -** Implementation of matchinfo() function. +** This routine is the same as the sqlite3_complete() routine described +** above, except that the parameter is required to be UTF-16 encoded, not +** UTF-8. */ -SQLITE_PRIVATE void sqlite3Fts3Matchinfo( - sqlite3_context *pContext, /* Function call context */ - Fts3Cursor *pCsr, /* FTS3 table cursor */ - const char *zArg /* Second arg to matchinfo() function */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - const char *zFormat; - - if( zArg ){ - zFormat = zArg; - }else{ - zFormat = FTS3_MATCHINFO_DEFAULT; - } +SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *zSql){ + sqlite3_value *pVal; + char const *zSql8; + int rc; - if( !pCsr->pExpr ){ - sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); - return; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zSql8 ){ + rc = sqlite3_complete(zSql8); }else{ - /* Retrieve matchinfo() data. */ - fts3GetMatchinfo(pContext, pCsr, zFormat); - sqlite3Fts3SegmentsClose(pTab); + rc = SQLITE_NOMEM; } + sqlite3ValueFree(pVal); + return sqlite3ApiExit(0, rc); } +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_OMIT_COMPLETE */ -#endif +/************** End of complete.c ********************************************/ +/************** Begin file main.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. +*/ -/************** End of fts3_snippet.c ****************************************/ -/************** Begin file fts3_unicode.c ************************************/ +#ifdef SQLITE_ENABLE_FTS3 +/************** Include fts3.h in the middle of main.c ***********************/ +/************** Begin file fts3.h ********************************************/ /* -** 2012 May 24 +** 2006 Oct 10 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -163692,15811 +130941,12798 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo( ** ****************************************************************************** ** -** Implementation of the "unicode" full-text-search tokenizer. +** This header file is used by programs that want to link against the +** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. */ -#ifndef SQLITE_DISABLE_FTS3_UNICODE - -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#if 0 +extern "C" { +#endif /* __cplusplus */ -/* #include */ -/* #include */ -/* #include */ -/* #include */ +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); -/* #include "fts3_tokenizer.h" */ +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ +/************** End of fts3.h ************************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_RTREE +/************** Include rtree.h in the middle of main.c **********************/ +/************** Begin file rtree.h *******************************************/ /* -** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied -** from the sqlite3 source file utf.c. If this file is compiled as part -** of the amalgamation, they are not required. +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** RTREE library. All it does is declare the sqlite3RtreeInit() interface. */ -#ifndef SQLITE_AMALGAMATION - -static const unsigned char sqlite3Utf8Trans1[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, -}; - -#define READ_UTF8(zIn, zTerm, c) \ - c = *(zIn++); \ - if( c>=0xc0 ){ \ - c = sqlite3Utf8Trans1[c-0xc0]; \ - while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ - c = (c<<6) + (0x3f & *(zIn++)); \ - } \ - if( c<0x80 \ - || (c&0xFFFFF800)==0xD800 \ - || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ - } - -#define WRITE_UTF8(zOut, c) { \ - if( c<0x00080 ){ \ - *zOut++ = (u8)(c&0xFF); \ - } \ - else if( c<0x00800 ){ \ - *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - } \ - else if( c<0x10000 ){ \ - *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ - *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - }else{ \ - *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ - *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ - *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - } \ -} - -#endif /* ifndef SQLITE_AMALGAMATION */ - -typedef struct unicode_tokenizer unicode_tokenizer; -typedef struct unicode_cursor unicode_cursor; - -struct unicode_tokenizer { - sqlite3_tokenizer base; - int bRemoveDiacritic; - int nException; - int *aiException; -}; -struct unicode_cursor { - sqlite3_tokenizer_cursor base; - const unsigned char *aInput; /* Input text being tokenized */ - int nInput; /* Size of aInput[] in bytes */ - int iOff; /* Current offset within aInput[] */ - int iToken; /* Index of next token to be returned */ - char *zToken; /* storage for current token */ - int nAlloc; /* space allocated at zToken */ -}; +#if 0 +extern "C" { +#endif /* __cplusplus */ +SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); -/* -** Destroy a tokenizer allocated by unicodeCreate(). -*/ -static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ - if( pTokenizer ){ - unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer; - sqlite3_free(p->aiException); - sqlite3_free(p); - } - return SQLITE_OK; -} +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ +/************** End of rtree.h ***********************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_ICU +/************** Include sqliteicu.h in the middle of main.c ******************/ +/************** Begin file sqliteicu.h ***************************************/ /* -** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE -** statement has specified that the tokenizer for this table shall consider -** all characters in string zIn/nIn to be separators (if bAlnum==0) or -** token characters (if bAlnum==1). +** 2008 May 26 ** -** For each codepoint in the zIn/nIn string, this function checks if the -** sqlite3FtsUnicodeIsalnum() function already returns the desired result. -** If so, no action is taken. Otherwise, the codepoint is added to the -** unicode_tokenizer.aiException[] array. For the purposes of tokenization, -** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all -** codepoints in the aiException[] array. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() -** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. -** It is not possible to change the behavior of the tokenizer with respect -** to these codepoints. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** ICU extension. All it does is declare the sqlite3IcuInit() interface. */ -static int unicodeAddExceptions( - unicode_tokenizer *p, /* Tokenizer to add exceptions to */ - int bAlnum, /* Replace Isalnum() return value with this */ - const char *zIn, /* Array of characters to make exceptions */ - int nIn /* Length of z in bytes */ -){ - const unsigned char *z = (const unsigned char *)zIn; - const unsigned char *zTerm = &z[nIn]; - int iCode; - int nEntry = 0; - - assert( bAlnum==0 || bAlnum==1 ); - while( zaiException, (p->nException+nEntry)*sizeof(int)); - if( aNew==0 ) return SQLITE_NOMEM; - nNew = p->nException; +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ - z = (const unsigned char *)zIn; - while( zi; j--) aNew[j] = aNew[j-1]; - aNew[i] = iCode; - nNew++; - } - } - p->aiException = aNew; - p->nException = nNew; - } - return SQLITE_OK; -} +/************** End of sqliteicu.h *******************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif -/* -** Return true if the p->aiException[] array contains the value iCode. +#ifndef SQLITE_AMALGAMATION +/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant +** contains the text of SQLITE_VERSION macro. */ -static int unicodeIsException(unicode_tokenizer *p, int iCode){ - if( p->nException>0 ){ - int *a = p->aiException; - int iLo = 0; - int iHi = p->nException-1; - - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - if( iCode==a[iTest] ){ - return 1; - }else if( iCode>a[iTest] ){ - iLo = iTest+1; - }else{ - iHi = iTest-1; - } - } - } - - return 0; -} +SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; +#endif -/* -** Return true if, for the purposes of tokenization, codepoint iCode is -** considered a token character (not a separator). +/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns +** a pointer to the to the sqlite3_version[] string constant. */ -static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){ - assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); - return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode); -} +SQLITE_API const char *SQLITE_STDCALL sqlite3_libversion(void){ return sqlite3_version; } -/* -** Create a new tokenizer instance. +/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a +** pointer to a string constant whose value is the same as the +** SQLITE_SOURCE_ID C preprocessor macro. */ -static int unicodeCreate( - int nArg, /* Size of array argv[] */ - const char * const *azArg, /* Tokenizer creation arguments */ - sqlite3_tokenizer **pp /* OUT: New tokenizer handle */ -){ - unicode_tokenizer *pNew; /* New tokenizer object */ - int i; - int rc = SQLITE_OK; - - pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); - if( pNew==NULL ) return SQLITE_NOMEM; - memset(pNew, 0, sizeof(unicode_tokenizer)); - pNew->bRemoveDiacritic = 1; +SQLITE_API const char *SQLITE_STDCALL sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } - for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; - } - else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ - pNew->bRemoveDiacritic = 0; - } - else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ - rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); - } - else if( n>=11 && memcmp("separators=", z, 11)==0 ){ - rc = unicodeAddExceptions(pNew, 0, &z[11], n-11); - } - else{ - /* Unrecognized argument */ - rc = SQLITE_ERROR; - } - } +/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function +** returns an integer equal to SQLITE_VERSION_NUMBER. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } - if( rc!=SQLITE_OK ){ - unicodeDestroy((sqlite3_tokenizer *)pNew); - pNew = 0; - } - *pp = (sqlite3_tokenizer *)pNew; - return rc; -} +/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled with mutexing code omitted due to +** the SQLITE_THREADSAFE compile-time option being set to 0. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } /* -** Prepare to begin tokenizing a particular string. The input -** string to be tokenized is pInput[0..nBytes-1]. A cursor -** used to incrementally tokenize this string is returned in -** *ppCursor. +** When compiling the test fixture or with debugging enabled (on Win32), +** this variable being set to non-zero will cause OSTRACE macros to emit +** extra diagnostic information. */ -static int unicodeOpen( - sqlite3_tokenizer *p, /* The tokenizer */ - const char *aInput, /* Input string */ - int nInput, /* Size of string aInput in bytes */ - sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */ -){ - unicode_cursor *pCsr; - - pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor)); - if( pCsr==0 ){ - return SQLITE_NOMEM; - } - memset(pCsr, 0, sizeof(unicode_cursor)); - - pCsr->aInput = (const unsigned char *)aInput; - if( aInput==0 ){ - pCsr->nInput = 0; - }else if( nInput<0 ){ - pCsr->nInput = (int)strlen(aInput); - }else{ - pCsr->nInput = nInput; - } - - *pp = &pCsr->base; - UNUSED_PARAMETER(p); - return SQLITE_OK; -} +#ifdef SQLITE_HAVE_OS_TRACE +# ifndef SQLITE_DEBUG_OS_TRACE +# define SQLITE_DEBUG_OS_TRACE 0 +# endif + int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +#endif +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) /* -** Close a tokenization cursor previously opened by a call to -** simpleOpen() above. +** If the following function pointer is not NULL and if +** SQLITE_ENABLE_IOTRACE is enabled, then messages describing +** I/O active are written using this function. These messages +** are intended for debugging activity only. */ -static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){ - unicode_cursor *pCsr = (unicode_cursor *) pCursor; - sqlite3_free(pCsr->zToken); - sqlite3_free(pCsr); - return SQLITE_OK; -} +SQLITE_API void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...) = 0; +#endif /* -** Extract the next token from a tokenization cursor. The cursor must -** have been opened by a prior call to simpleOpen(). +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** temporary files. +** +** See also the "PRAGMA temp_store_directory" SQL command. */ -static int unicodeNext( - sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */ - const char **paToken, /* OUT: Token text */ - int *pnToken, /* OUT: Number of bytes at *paToken */ - int *piStart, /* OUT: Starting offset of token */ - int *piEnd, /* OUT: Ending offset of token */ - int *piPos /* OUT: Position integer of token */ -){ - unicode_cursor *pCsr = (unicode_cursor *)pC; - unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); - int iCode = 0; - char *zOut; - const unsigned char *z = &pCsr->aInput[pCsr->iOff]; - const unsigned char *zStart = z; - const unsigned char *zEnd; - const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; - - /* Scan past any delimiter characters before the start of the next token. - ** Return SQLITE_DONE early if this takes us all the way to the end of - ** the input. */ - while( z=zTerm ) return SQLITE_DONE; - - zOut = pCsr->zToken; - do { - int iOut; - - /* Grow the output buffer if required. */ - if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ - char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64); - if( !zNew ) return SQLITE_NOMEM; - zOut = &zNew[zOut - pCsr->zToken]; - pCsr->zToken = zNew; - pCsr->nAlloc += 64; - } - - /* Write the folded case of the last character read to the output */ - zEnd = z; - iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic); - if( iOut ){ - WRITE_UTF8(zOut, iOut); - } - - /* If the cursor is not at EOF, read the next character */ - if( z>=zTerm ) break; - READ_UTF8(z, zTerm, iCode); - }while( unicodeIsAlnum(p, iCode) - || sqlite3FtsUnicodeIsdiacritic(iCode) - ); - - /* Set the output variables and return. */ - pCsr->iOff = (int)(z - pCsr->aInput); - *paToken = pCsr->zToken; - *pnToken = (int)(zOut - pCsr->zToken); - *piStart = (int)(zStart - pCsr->aInput); - *piEnd = (int)(zEnd - pCsr->aInput); - *piPos = pCsr->iToken++; - return SQLITE_OK; -} +SQLITE_API char *sqlite3_temp_directory = 0; /* -** Set *ppModule to a pointer to the sqlite3_tokenizer_module -** structure for the unicode tokenizer. +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** all database files specified with a relative pathname. +** +** See also the "PRAGMA data_store_directory" SQL command. */ -SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ - static const sqlite3_tokenizer_module module = { - 0, - unicodeCreate, - unicodeDestroy, - unicodeOpen, - unicodeClose, - unicodeNext, - 0, - }; - *ppModule = &module; -} - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ -#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ +SQLITE_API char *sqlite3_data_directory = 0; -/************** End of fts3_unicode.c ****************************************/ -/************** Begin file fts3_unicode2.c ***********************************/ /* -** 2012 May 25 +** Initialize SQLite. ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** This routine must be called to initialize the memory allocation, +** VFS, and mutex subsystems prior to doing any serious work with +** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT +** this routine will be called automatically by key routines such as +** sqlite3_open(). ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** This routine is a no-op except on its very first call for the process, +** or for the first call after a call to sqlite3_shutdown. ** -****************************************************************************** +** The first thread to call this routine runs the initialization to +** completion. If subsequent threads call this routine before the first +** thread has finished the initialization process, then the subsequent +** threads must block until the first thread finishes with the initialization. +** +** The first thread might call this routine recursively. Recursive +** calls to this routine should not block, of course. Otherwise the +** initialization process would never complete. +** +** Let X be the first thread to enter this routine. Let Y be some other +** thread. Then while the initial invocation of this routine by X is +** incomplete, it is required that: +** +** * Calls to this routine from Y must block until the outer-most +** call by X completes. +** +** * Recursive calls to this routine from thread X return immediately +** without blocking. */ +SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ + MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ + int rc; /* Result code */ +#ifdef SQLITE_EXTRA_INIT + int bRunExtraInit = 0; /* Extra initialization needed */ +#endif -/* -** DO NOT EDIT THIS MACHINE GENERATED FILE. -*/ +#ifdef SQLITE_OMIT_WSD + rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif -#ifndef SQLITE_DISABLE_FTS3_UNICODE -#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + /* If the following assert() fails on some obscure processor/compiler + ** combination, the work-around is to set the correct pointer + ** size at compile-time using -DSQLITE_PTRSIZE=n compile-time option */ + assert( SQLITE_PTRSIZE==sizeof(char*) ); -/* #include */ + /* If SQLite is already completely initialized, then this call + ** to sqlite3_initialize() should be a no-op. But the initialization + ** must be complete. So isInit must not be set until the very end + ** of this routine. + */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; -/* -** Return true if the argument corresponds to a unicode codepoint -** classified as either a letter or a number. Otherwise false. -** -** The results are undefined if the value passed to this function -** is less than zero. -*/ -SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ - /* Each unsigned integer in the following array corresponds to a contiguous - ** range of unicode codepoints that are not either letters or numbers (i.e. - ** codepoints for which this function should return 0). + /* Make sure the mutex subsystem is initialized. If unable to + ** initialize the mutex subsystem, return early with the error. + ** If the system is so sick that we are unable to allocate a mutex, + ** there is not much SQLite is going to be able to do. ** - ** The most significant 22 bits in each 32-bit value contain the first - ** codepoint in the range. The least significant 10 bits are used to store - ** the size of the range (always at least 1). In other words, the value - ** ((C<<22) + N) represents a range of N codepoints starting with codepoint - ** C. It is not possible to represent a range larger than 1023 codepoints - ** using this format. + ** The mutex subsystem must take care of serializing its own + ** initialization. */ - static const unsigned int aEntry[] = { - 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, - 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, - 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, - 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, - 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, - 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, - 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, - 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, - 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, - 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, - 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, - 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, - 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, - 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, - 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, - 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, - 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, - 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, - 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, - 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, - 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, - 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, - 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, - 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, - 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, - 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, - 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, - 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, - 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, - 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, - 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, - 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, - 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, - 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, - 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, - 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, - 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, - 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, - 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, - 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, - 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, - 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, - 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, - 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, - 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, - 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, - 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, - 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, - 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, - 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, - 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, - 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, - 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, - 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, - 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, - 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, - 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, - 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, - 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, - 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, - 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, - 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, - 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, - 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, - 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, - 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, - 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, - 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, - 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, - 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, - 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, - 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, - 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, - 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, - 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, - 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, - 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, - 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, - 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, - 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, - 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, - 0x380400F0, - }; - static const unsigned int aAscii[4] = { - 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, - }; + rc = sqlite3MutexInit(); + if( rc ) return rc; - if( c<128 ){ - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); - }else if( c<(1<<22) ){ - unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; - int iRes = 0; - int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; - int iLo = 0; - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - if( key >= aEntry[iTest] ){ - iRes = iTest; - iLo = iTest+1; - }else{ - iHi = iTest-1; + /* Initialize the malloc() system and the recursive pInitMutex mutex. + ** This operation is protected by the STATIC_MASTER mutex. Note that + ** MutexAlloc() is called for a static mutex prior to initializing the + ** malloc subsystem - this implies that the allocation of a static + ** mutex must not require support from the malloc subsystem. + */ + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.isMutexInit = 1; + if( !sqlite3GlobalConfig.isMallocInit ){ + rc = sqlite3MallocInit(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isMallocInit = 1; + if( !sqlite3GlobalConfig.pInitMutex ){ + sqlite3GlobalConfig.pInitMutex = + sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ + rc = SQLITE_NOMEM; } } - assert( aEntry[0]=aEntry[iRes] ); - return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); } - return 1; -} - - -/* -** If the argument is a codepoint corresponding to a lowercase letter -** in the ASCII range with a diacritic added, return the codepoint -** of the ASCII letter only. For example, if passed 235 - "LATIN -** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER -** E"). The resuls of passing a codepoint that corresponds to an -** uppercase letter are undefined. -*/ -static int remove_diacritic(int c){ - unsigned short aDia[] = { - 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, - 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, - 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, - 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, - 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, - 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, - 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, - 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, - 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, - 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, - 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, - 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, - 62924, 63050, 63082, 63274, 63390, - }; - char aChar[] = { - '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', - 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', - 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', - 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', - 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', - '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', - 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', - 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', - 'e', 'i', 'o', 'u', 'y', - }; - - unsigned int key = (((unsigned int)c)<<3) | 0x00000007; - int iRes = 0; - int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; - int iLo = 0; - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - if( key >= aDia[iTest] ){ - iRes = iTest; - iLo = iTest+1; - }else{ - iHi = iTest-1; - } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.nRefInitMutex++; } - assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); -} - - -/* -** Return true if the argument interpreted as a unicode codepoint -** is a diacritical modifier character. -*/ -SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){ - unsigned int mask0 = 0x08029FDF; - unsigned int mask1 = 0x000361F8; - if( c<768 || c>817 ) return 0; - return (c < 768+32) ? - (mask0 & (1 << (c-768))) : - (mask1 & (1 << (c-768-32))); -} + sqlite3_mutex_leave(pMaster); + /* If rc is not SQLITE_OK at this point, then either the malloc + ** subsystem could not be initialized or the system failed to allocate + ** the pInitMutex mutex. Return an error in either case. */ + if( rc!=SQLITE_OK ){ + return rc; + } -/* -** Interpret the argument as a unicode codepoint. If the codepoint -** is an upper case character that has a lower case equivalent, -** return the codepoint corresponding to the lower case version. -** Otherwise, return a copy of the argument. -** -** The results are undefined if the value passed to this function -** is less than zero. -*/ -SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ - /* Each entry in the following array defines a rule for folding a range - ** of codepoints to lower case. The rule applies to a range of nRange - ** codepoints starting at codepoint iCode. - ** - ** If the least significant bit in flags is clear, then the rule applies - ** to all nRange codepoints (i.e. all nRange codepoints are upper case and - ** need to be folded). Or, if it is set, then the rule only applies to - ** every second codepoint in the range, starting with codepoint C. + /* Do the rest of the initialization under the recursive mutex so + ** that we will be able to handle recursive calls into + ** sqlite3_initialize(). The recursive calls normally come through + ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other + ** recursive calls might also be possible. ** - ** The 7 most significant bits in flags are an index into the aiOff[] - ** array. If a specific codepoint C does require folding, then its lower - ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls + ** to the xInit method, so the xInit method need not be threadsafe. ** - ** The contents of this array are generated by parsing the CaseFolding.txt - ** file distributed as part of the "Unicode Character Database". See - ** http://www.unicode.org for details. + ** The following mutex is what serializes access to the appdef pcache xInit + ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the + ** call to sqlite3PcacheInitialize(). */ - static const struct TableEntry { - unsigned short iCode; - unsigned char flags; - unsigned char nRange; - } aEntry[] = { - {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, - {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, - {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, - {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, - {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, - {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, - {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, - {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, - {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, - {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, - {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, - {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, - {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, - {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, - {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, - {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, - {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, - {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, - {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, - {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, - {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, - {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, - {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, - {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, - {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, - {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, - {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, - {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, - {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, - {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, - {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, - {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, - {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, - {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, - {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, - {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, - {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, - {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, - {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, - {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, - {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, - {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, - {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, - {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, - {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, - {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, - {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, - {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, - {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, - {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, - {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, - {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, - {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, - {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, - {65313, 14, 26}, - }; - static const unsigned short aiOff[] = { - 1, 2, 8, 15, 16, 26, 28, 32, - 37, 38, 40, 48, 63, 64, 69, 71, - 79, 80, 116, 202, 203, 205, 206, 207, - 209, 210, 211, 213, 214, 217, 218, 219, - 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, - 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, - 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, - 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, - 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, - 65514, 65521, 65527, 65528, 65529, - }; - - int ret = c; - - assert( c>=0 ); - assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); - - if( c<128 ){ - if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); - }else if( c<65536 ){ - int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; - int iLo = 0; - int iRes = -1; - - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - int cmp = (c - aEntry[iTest].iCode); - if( cmp>=0 ){ - iRes = iTest; - iLo = iTest+1; - }else{ - iHi = iTest-1; - } + sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); + if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + sqlite3GlobalConfig.inProgress = 1; + memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); + sqlite3RegisterGlobalFunctions(); + if( sqlite3GlobalConfig.isPCacheInit==0 ){ + rc = sqlite3PcacheInitialize(); } - assert( iRes<0 || c>=aEntry[iRes].iCode ); - - if( iRes>=0 ){ - const struct TableEntry *p = &aEntry[iRes]; - if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ - ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; - assert( ret>0 ); - } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isPCacheInit = 1; + rc = sqlite3OsInit(); } - - if( bRemoveDiacritic ) ret = remove_diacritic(ret); - } - - else if( c>=66560 && c<66600 ){ - ret = c + 40; + if( rc==SQLITE_OK ){ + sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, + sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); + sqlite3GlobalConfig.isInit = 1; +#ifdef SQLITE_EXTRA_INIT + bRunExtraInit = 1; +#endif + } + sqlite3GlobalConfig.inProgress = 0; } + sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); - return ret; -} -#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ -#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ - -/************** End of fts3_unicode2.c ***************************************/ -/************** Begin file rtree.c *******************************************/ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains code for implementations of the r-tree and r*-tree -** algorithms packaged as an SQLite virtual table module. -*/ - -/* -** Database Format of R-Tree Tables -** -------------------------------- -** -** The data structure for a single virtual r-tree table is stored in three -** native SQLite tables declared as follows. In each case, the '%' character -** in the table name is replaced with the user-supplied name of the r-tree -** table. -** -** CREATE TABLE %_node(nodeno INTEGER PRIMARY KEY, data BLOB) -** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) -** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER) -** -** The data for each node of the r-tree structure is stored in the %_node -** table. For each node that is not the root node of the r-tree, there is -** an entry in the %_parent table associating the node with its parent. -** And for each row of data in the table, there is an entry in the %_rowid -** table that maps from the entries rowid to the id of the node that it -** is stored on. -** -** The root node of an r-tree always exists, even if the r-tree table is -** empty. The nodeno of the root node is always 1. All other nodes in the -** table must be the same size as the root node. The content of each node -** is formatted as follows: -** -** 1. If the node is the root node (node 1), then the first 2 bytes -** of the node contain the tree depth as a big-endian integer. -** For non-root nodes, the first 2 bytes are left unused. -** -** 2. The next 2 bytes contain the number of entries currently -** stored in the node. -** -** 3. The remainder of the node contains the node entries. Each entry -** consists of a single 8-byte integer followed by an even number -** of 4-byte coordinates. For leaf nodes the integer is the rowid -** of a record. For internal nodes it is the node number of a -** child page. -*/ - -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) + /* Go back under the static mutex and clean up the recursive + ** mutex to prevent a resource leak. + */ + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.nRefInitMutex--; + if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ + assert( sqlite3GlobalConfig.nRefInitMutex==0 ); + sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); + sqlite3GlobalConfig.pInitMutex = 0; + } + sqlite3_mutex_leave(pMaster); -#ifndef SQLITE_CORE -/* #include "sqlite3ext.h" */ - SQLITE_EXTENSION_INIT1 -#else -/* #include "sqlite3.h" */ + /* The following is just a sanity check to make sure SQLite has + ** been compiled correctly. It is important to run this code, but + ** we don't want to run it too often and soak up CPU cycles for no + ** reason. So we run it once during initialization. + */ +#ifndef NDEBUG +#ifndef SQLITE_OMIT_FLOATING_POINT + /* This section of code's only "output" is via assert() statements. */ + if ( rc==SQLITE_OK ){ + u64 x = (((u64)1)<<63)-1; + double y; + assert(sizeof(x)==8); + assert(sizeof(x)==sizeof(y)); + memcpy(&y, &x, 8); + assert( sqlite3IsNaN(y) ); + } +#endif #endif -/* #include */ -/* #include */ -/* #include */ - -#ifndef SQLITE_AMALGAMATION -#include "sqlite3rtree.h" -typedef sqlite3_int64 i64; -typedef unsigned char u8; -typedef unsigned short u16; -typedef unsigned int u32; + /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT + ** compile-time option. + */ +#ifdef SQLITE_EXTRA_INIT + if( bRunExtraInit ){ + int SQLITE_EXTRA_INIT(const char*); + rc = SQLITE_EXTRA_INIT(0); + } #endif -/* The following macro is used to suppress compiler warnings. + return rc; +} + +/* +** Undo the effects of sqlite3_initialize(). Must not be called while +** there are outstanding database connections or memory allocations or +** while any part of SQLite is otherwise in use in any thread. This +** routine is not threadsafe. But it is safe to invoke this routine +** on when SQLite is already shut down. If SQLite is already shut down +** when this routine is invoked, then this routine is a harmless no-op. */ -#ifndef UNUSED_PARAMETER -# define UNUSED_PARAMETER(x) (void)(x) +SQLITE_API int SQLITE_STDCALL sqlite3_shutdown(void){ +#ifdef SQLITE_OMIT_WSD + int rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } #endif -typedef struct Rtree Rtree; -typedef struct RtreeCursor RtreeCursor; -typedef struct RtreeNode RtreeNode; -typedef struct RtreeCell RtreeCell; -typedef struct RtreeConstraint RtreeConstraint; -typedef struct RtreeMatchArg RtreeMatchArg; -typedef struct RtreeGeomCallback RtreeGeomCallback; -typedef union RtreeCoord RtreeCoord; -typedef struct RtreeSearchPoint RtreeSearchPoint; + if( sqlite3GlobalConfig.isInit ){ +#ifdef SQLITE_EXTRA_SHUTDOWN + void SQLITE_EXTRA_SHUTDOWN(void); + SQLITE_EXTRA_SHUTDOWN(); +#endif + sqlite3_os_end(); + sqlite3_reset_auto_extension(); + sqlite3GlobalConfig.isInit = 0; + } + if( sqlite3GlobalConfig.isPCacheInit ){ + sqlite3PcacheShutdown(); + sqlite3GlobalConfig.isPCacheInit = 0; + } + if( sqlite3GlobalConfig.isMallocInit ){ + sqlite3MallocEnd(); + sqlite3GlobalConfig.isMallocInit = 0; -/* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ -#define RTREE_MAX_DIMENSIONS 5 +#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES + /* The heap subsystem has now been shutdown and these values are supposed + ** to be NULL or point to memory that was obtained from sqlite3_malloc(), + ** which would rely on that heap subsystem; therefore, make sure these + ** values cannot refer to heap memory that was just invalidated when the + ** heap subsystem was shutdown. This is only done if the current call to + ** this function resulted in the heap subsystem actually being shutdown. + */ + sqlite3_data_directory = 0; + sqlite3_temp_directory = 0; +#endif + } + if( sqlite3GlobalConfig.isMutexInit ){ + sqlite3MutexEnd(); + sqlite3GlobalConfig.isMutexInit = 0; + } -/* Size of hash table Rtree.aHash. This hash table is not expected to -** ever contain very many entries, so a fixed number of buckets is -** used. -*/ -#define HASHSIZE 97 + return SQLITE_OK; +} -/* The xBestIndex method of this virtual table requires an estimate of -** the number of rows in the virtual table to calculate the costs of -** various strategies. If possible, this estimate is loaded from the -** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum). -** Otherwise, if no sqlite_stat1 entry is available, use -** RTREE_DEFAULT_ROWEST. +/* +** This API allows applications to modify the global configuration of +** the SQLite library at run-time. +** +** This routine should only be called when there are no outstanding +** database connections or memory allocations. This routine is not +** threadsafe. Failure to heed these warnings can lead to unpredictable +** behavior. */ -#define RTREE_DEFAULT_ROWEST 1048576 -#define RTREE_MIN_ROWEST 100 +SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ + va_list ap; + int rc = SQLITE_OK; -/* -** An rtree virtual-table object. -*/ -struct Rtree { - sqlite3_vtab base; /* Base class. Must be first */ - sqlite3 *db; /* Host database connection */ - int iNodeSize; /* Size in bytes of each node in the node table */ - u8 nDim; /* Number of dimensions */ - u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ - u8 nBytesPerCell; /* Bytes consumed per cell */ - int iDepth; /* Current depth of the r-tree structure */ - char *zDb; /* Name of database containing r-tree table */ - char *zName; /* Name of r-tree table */ - int nBusy; /* Current number of users of this structure */ - i64 nRowEst; /* Estimated number of rows in this table */ + /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; - /* List of nodes removed during a CondenseTree operation. List is - ** linked together via the pointer normally used for hash chains - - ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree - ** headed by the node (leaf nodes have RtreeNode.iNode==0). - */ - RtreeNode *pDeleted; - int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ + va_start(ap, op); + switch( op ){ - /* Statements to read/write/delete a record from xxx_node */ - sqlite3_stmt *pReadNode; - sqlite3_stmt *pWriteNode; - sqlite3_stmt *pDeleteNode; + /* Mutex configuration options are only available in a threadsafe + ** compile. + */ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ + case SQLITE_CONFIG_SINGLETHREAD: { + /* EVIDENCE-OF: R-02748-19096 This option sets the threading mode to + ** Single-thread. */ + sqlite3GlobalConfig.bCoreMutex = 0; /* Disable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ + case SQLITE_CONFIG_MULTITHREAD: { + /* EVIDENCE-OF: R-14374-42468 This option sets the threading mode to + ** Multi-thread. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ + case SQLITE_CONFIG_SERIALIZED: { + /* EVIDENCE-OF: R-41220-51800 This option sets the threading mode to + ** Serialized. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 1; /* Enable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ + case SQLITE_CONFIG_MUTEX: { + /* Specify an alternative mutex implementation */ + sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ + case SQLITE_CONFIG_GETMUTEX: { + /* Retrieve the current mutex implementation */ + *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; + break; + } +#endif - /* Statements to read/write/delete a record from xxx_rowid */ - sqlite3_stmt *pReadRowid; - sqlite3_stmt *pWriteRowid; - sqlite3_stmt *pDeleteRowid; + case SQLITE_CONFIG_MALLOC: { + /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The argument specifies alternative + ** low-level memory allocation routines to be used in place of the memory + ** allocation routines built into SQLite. */ + sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); + break; + } + case SQLITE_CONFIG_GETMALLOC: { + /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is + ** filled with the currently defined memory allocation routines. */ + if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); + *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; + break; + } + case SQLITE_CONFIG_MEMSTATUS: { + /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes + ** single argument of type int, interpreted as a boolean, which enables + ** or disables the collection of memory allocation statistics. */ + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_SCRATCH: { + /* EVIDENCE-OF: R-08404-60887 There are three arguments to + ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from + ** which the scratch allocations will be drawn, the size of each scratch + ** allocation (sz), and the maximum number of scratch allocations (N). */ + sqlite3GlobalConfig.pScratch = va_arg(ap, void*); + sqlite3GlobalConfig.szScratch = va_arg(ap, int); + sqlite3GlobalConfig.nScratch = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PAGECACHE: { + /* EVIDENCE-OF: R-31408-40510 There are three arguments to + ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory, the size + ** of each page buffer (sz), and the number of pages (N). */ + sqlite3GlobalConfig.pPage = va_arg(ap, void*); + sqlite3GlobalConfig.szPage = va_arg(ap, int); + sqlite3GlobalConfig.nPage = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PCACHE_HDRSZ: { + /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes + ** a single parameter which is a pointer to an integer and writes into + ** that integer the number of extra bytes per page required for each page + ** in SQLITE_CONFIG_PAGECACHE. */ + *va_arg(ap, int*) = + sqlite3HeaderSizeBtree() + + sqlite3HeaderSizePcache() + + sqlite3HeaderSizePcache1(); + break; + } - /* Statements to read/write/delete a record from xxx_parent */ - sqlite3_stmt *pReadParent; - sqlite3_stmt *pWriteParent; - sqlite3_stmt *pDeleteParent; + case SQLITE_CONFIG_PCACHE: { + /* no-op */ + break; + } + case SQLITE_CONFIG_GETPCACHE: { + /* now an error */ + rc = SQLITE_ERROR; + break; + } - RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ -}; + case SQLITE_CONFIG_PCACHE2: { + /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. This object specifies the interface to a custom page cache + ** implementation. */ + sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); + break; + } + case SQLITE_CONFIG_GETPCACHE2: { + /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. SQLite copies of the current page cache implementation into + ** that object. */ + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ + sqlite3PCacheSetDefault(); + } + *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; + break; + } -/* Possible values for Rtree.eCoordType: */ -#define RTREE_COORD_REAL32 0 -#define RTREE_COORD_INT32 1 +/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only +** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or +** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */ +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + case SQLITE_CONFIG_HEAP: { + /* EVIDENCE-OF: R-19854-42126 There are three arguments to + ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the + ** number of bytes in the memory buffer, and the minimum allocation size. + */ + sqlite3GlobalConfig.pHeap = va_arg(ap, void*); + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + sqlite3GlobalConfig.mnReq = va_arg(ap, int); -/* -** If SQLITE_RTREE_INT_ONLY is defined, then this virtual table will -** only deal with integer coordinates. No floating point operations -** will be done. -*/ -#ifdef SQLITE_RTREE_INT_ONLY - typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ - typedef int RtreeValue; /* Low accuracy coordinate */ -# define RTREE_ZERO 0 -#else - typedef double RtreeDValue; /* High accuracy coordinate */ - typedef float RtreeValue; /* Low accuracy coordinate */ -# define RTREE_ZERO 0.0 + if( sqlite3GlobalConfig.mnReq<1 ){ + sqlite3GlobalConfig.mnReq = 1; + }else if( sqlite3GlobalConfig.mnReq>(1<<12) ){ + /* cap min request size at 2^12 */ + sqlite3GlobalConfig.mnReq = (1<<12); + } + + if( sqlite3GlobalConfig.pHeap==0 ){ + /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) + ** is NULL, then SQLite reverts to using its default memory allocator + ** (the system malloc() implementation), undoing any prior invocation of + ** SQLITE_CONFIG_MALLOC. + ** + ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to + ** revert to its default implementation when sqlite3_initialize() is run + */ + memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); + }else{ + /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the + ** alternative memory allocator is engaged to handle all of SQLites + ** memory allocation needs. */ +#ifdef SQLITE_ENABLE_MEMSYS3 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); +#endif + } + break; + } #endif -/* -** When doing a search of an r-tree, instances of the following structure -** record intermediate results from the tree walk. -** -** The id is always a node-id. For iLevel>=1 the id is the node-id of -** the node that the RtreeSearchPoint represents. When iLevel==0, however, -** the id is of the parent node and the cell that RtreeSearchPoint -** represents is the iCell-th entry in the parent node. -*/ -struct RtreeSearchPoint { - RtreeDValue rScore; /* The score for this node. Smallest goes first. */ - sqlite3_int64 id; /* Node ID */ - u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ - u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ - u8 iCell; /* Cell index within the node */ -}; + case SQLITE_CONFIG_LOOKASIDE: { + sqlite3GlobalConfig.szLookaside = va_arg(ap, int); + sqlite3GlobalConfig.nLookaside = va_arg(ap, int); + break; + } + + /* Record a pointer to the logger function and its first argument. + ** The default is NULL. Logging is disabled if the function pointer is + ** NULL. + */ + case SQLITE_CONFIG_LOG: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); + */ + typedef void(*LOGFUNC_t)(void*,int,const char*); + sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); + sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + break; + } -/* -** The minimum number of cells allowed for a node is a third of the -** maximum. In Gutman's notation: -** -** m = M/3 -** -** If an R*-tree "Reinsert" operation is required, the same number of -** cells are removed from the overfull node and reinserted into the tree. -*/ -#define RTREE_MINCELLS(p) ((((p)->iNodeSize-4)/(p)->nBytesPerCell)/3) -#define RTREE_REINSERT(p) RTREE_MINCELLS(p) -#define RTREE_MAXCELLS 51 + /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames + ** can be changed at start-time using the + ** sqlite3_config(SQLITE_CONFIG_URI,1) or + ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. + */ + case SQLITE_CONFIG_URI: { + /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single + ** argument of type int. If non-zero, then URI handling is globally + ** enabled. If the parameter is zero, then URI handling is globally + ** disabled. */ + sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + break; + } -/* -** The smallest possible node-size is (512-64)==448 bytes. And the largest -** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). -** Therefore all non-root nodes must contain at least 3 entries. Since -** 2^40 is greater than 2^64, an r-tree structure always has a depth of -** 40 or less. -*/ -#define RTREE_MAX_DEPTH 40 + case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN + ** option takes a single integer argument which is interpreted as a + ** boolean in order to enable or disable the use of covering indices for + ** full table scans in the query optimizer. */ + sqlite3GlobalConfig.bUseCis = va_arg(ap, int); + break; + } +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif -/* -** Number of entries in the cursor RtreeNode cache. The first entry is -** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining -** entries cache the RtreeNode for the first elements of the priority queue. -*/ -#define RTREE_CACHE_SZ 5 + case SQLITE_CONFIG_MMAP_SIZE: { + /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit + ** integer (sqlite3_int64) values that are the default mmap size limit + ** (the default setting for PRAGMA mmap_size) and the maximum allowed + ** mmap size limit. */ + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + /* EVIDENCE-OF: R-53367-43190 If either argument to this option is + ** negative, then that argument is changed to its compile-time default. + ** + ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be + ** silently truncated if necessary so that it does not exceed the + ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE + ** compile-time option. + */ + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.mxMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } -/* -** An rtree cursor object. -*/ -struct RtreeCursor { - sqlite3_vtab_cursor base; /* Base class. Must be first */ - u8 atEOF; /* True if at end of search */ - u8 bPoint; /* True if sPoint is valid */ - int iStrategy; /* Copy of idxNum search parameter */ - int nConstraint; /* Number of entries in aConstraint */ - RtreeConstraint *aConstraint; /* Search constraints. */ - int nPointAlloc; /* Number of slots allocated for aPoint[] */ - int nPoint; /* Number of slots used in aPoint[] */ - int mxLevel; /* iLevel value for root of the tree */ - RtreeSearchPoint *aPoint; /* Priority queue for search points */ - RtreeSearchPoint sPoint; /* Cached next search point */ - RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ - u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ -}; +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ + case SQLITE_CONFIG_WIN32_HEAPSIZE: { + /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit + ** unsigned integer value that specifies the maximum size of the created + ** heap. */ + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + break; + } +#endif -/* Return the Rtree of a RtreeCursor */ -#define RTREE_OF_CURSOR(X) ((Rtree*)((X)->base.pVtab)) + case SQLITE_CONFIG_PMASZ: { + sqlite3GlobalConfig.szPma = va_arg(ap, unsigned int); + break; + } + + default: { + rc = SQLITE_ERROR; + break; + } + } + va_end(ap); + return rc; +} /* -** A coordinate can be either a floating point number or a integer. All -** coordinates within a single R-Tree are always of the same time. +** Set up the lookaside buffers for a database connection. +** Return SQLITE_OK on success. +** If lookaside is already active, return SQLITE_BUSY. +** +** The sz parameter is the number of bytes in each lookaside slot. +** The cnt parameter is the number of slots. If pStart is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc(). +** If pStart is not NULL then it is sz*cnt bytes of memory to use for +** the lookaside memory. */ -union RtreeCoord { - RtreeValue f; /* Floating point value */ - int i; /* Integer value */ - u32 u; /* Unsigned for byte-order conversions */ -}; +static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ + void *pStart; + if( db->lookaside.nOut ){ + return SQLITE_BUSY; + } + /* Free any existing lookaside buffer for this handle before + ** allocating a new one so we don't have to have space for + ** both at the same time. + */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger + ** than a pointer to be useful. + */ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; + if( cnt<0 ) cnt = 0; + if( sz==0 || cnt==0 ){ + sz = 0; + pStart = 0; + }else if( pBuf==0 ){ + sqlite3BeginBenignMalloc(); + pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ + sqlite3EndBenignMalloc(); + if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; + }else{ + pStart = pBuf; + } + db->lookaside.pStart = pStart; + db->lookaside.pFree = 0; + db->lookaside.sz = (u16)sz; + if( pStart ){ + int i; + LookasideSlot *p; + assert( sz > (int)sizeof(LookasideSlot*) ); + p = (LookasideSlot*)pStart; + for(i=cnt-1; i>=0; i--){ + p->pNext = db->lookaside.pFree; + db->lookaside.pFree = p; + p = (LookasideSlot*)&((u8*)p)[sz]; + } + db->lookaside.pEnd = p; + db->lookaside.bEnabled = 1; + db->lookaside.bMalloced = pBuf==0 ?1:0; + }else{ + db->lookaside.pStart = db; + db->lookaside.pEnd = db; + db->lookaside.bEnabled = 0; + db->lookaside.bMalloced = 0; + } + return SQLITE_OK; +} /* -** The argument is an RtreeCoord. Return the value stored within the RtreeCoord -** formatted as a RtreeDValue (double or int64). This macro assumes that local -** variable pRtree points to the Rtree structure associated with the -** RtreeCoord. +** Return the mutex associated with a database connection. */ -#ifdef SQLITE_RTREE_INT_ONLY -# define DCOORD(coord) ((RtreeDValue)coord.i) -#else -# define DCOORD(coord) ( \ - (pRtree->eCoordType==RTREE_COORD_REAL32) ? \ - ((double)coord.f) : \ - ((double)coord.i) \ - ) +SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } #endif + return db->mutex; +} /* -** A search constraint. +** Free up as much memory as we can from the given database +** connection. */ -struct RtreeConstraint { - int iCoord; /* Index of constrained coordinate */ - int op; /* Constraining operation */ - union { - RtreeDValue rValue; /* Constraint value. */ - int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); - int (*xQueryFunc)(sqlite3_rtree_query_info*); - } u; - sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ -}; - -/* Possible values for RtreeConstraint.op */ -#define RTREE_EQ 0x41 /* A */ -#define RTREE_LE 0x42 /* B */ -#define RTREE_LT 0x43 /* C */ -#define RTREE_GE 0x44 /* D */ -#define RTREE_GT 0x45 /* E */ -#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ -#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ +SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3 *db){ + int i; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerShrink(pPager); + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} -/* -** An rtree structure node. +/* +** Configuration settings for an individual database connection */ -struct RtreeNode { - RtreeNode *pParent; /* Parent node */ - i64 iNode; /* The node number */ - int nRef; /* Number of references to this node */ - int isDirty; /* True if the node needs to be written to disk */ - u8 *zData; /* Content of the node, as should be on disk */ - RtreeNode *pNext; /* Next node in this hash collision chain */ -}; +SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc; + va_start(ap, op); + switch( op ){ + case SQLITE_DBCONFIG_LOOKASIDE: { + void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ + int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ + int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ + rc = setupLookaside(db, pBuf, sz, cnt); + break; + } + default: { + static const struct { + int op; /* The opcode */ + u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ + } aFlagOp[] = { + { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, + { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + }; + unsigned int i; + rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ + for(i=0; iflags; + if( onoff>0 ){ + db->flags |= aFlagOp[i].mask; + }else if( onoff==0 ){ + db->flags &= ~aFlagOp[i].mask; + } + if( oldFlags!=db->flags ){ + sqlite3ExpirePreparedStatements(db); + } + if( pRes ){ + *pRes = (db->flags & aFlagOp[i].mask)!=0; + } + rc = SQLITE_OK; + break; + } + } + break; + } + } + va_end(ap); + return rc; +} -/* Return the number of cells in a node */ -#define NCELL(pNode) readInt16(&(pNode)->zData[2]) -/* -** A single cell from a node, deserialized +/* +** Return true if the buffer z[0..n-1] contains all spaces. */ -struct RtreeCell { - i64 iRowid; /* Node or entry ID */ - RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; /* Bounding box coordinates */ -}; - +static int allSpaces(const char *z, int n){ + while( n>0 && z[n-1]==' ' ){ n--; } + return n==0; +} /* -** This object becomes the sqlite3_user_data() for the SQL functions -** that are created by sqlite3_rtree_geometry_callback() and -** sqlite3_rtree_query_callback() and which appear on the right of MATCH -** operators in order to constrain a search. +** This is the default collating function named "BINARY" which is always +** available. ** -** xGeom and xQueryFunc are the callback functions. Exactly one of -** xGeom and xQueryFunc fields is non-NULL, depending on whether the -** SQL function was created using sqlite3_rtree_geometry_callback() or -** sqlite3_rtree_query_callback(). -** -** This object is deleted automatically by the destructor mechanism in -** sqlite3_create_function_v2(). +** If the padFlag argument is not NULL then space padding at the end +** of strings is ignored. This implements the RTRIM collation. */ -struct RtreeGeomCallback { - int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); - int (*xQueryFunc)(sqlite3_rtree_query_info*); - void (*xDestructor)(void*); - void *pContext; -}; - +static int binCollFunc( + void *padFlag, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int rc, n; + n = nKey1 (y) ? (y) : (x)) +SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_last_insert_rowid(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } #endif - -/* -** Functions to deserialize a 16 bit integer, 32 bit real number and -** 64 bit integer. The deserialized value is returned. -*/ -static int readInt16(u8 *p){ - return (p[0]<<8) + p[1]; -} -static void readCoord(u8 *p, RtreeCoord *pCoord){ - pCoord->u = ( - (((u32)p[0]) << 24) + - (((u32)p[1]) << 16) + - (((u32)p[2]) << 8) + - (((u32)p[3]) << 0) - ); -} -static i64 readInt64(u8 *p){ - return ( - (((i64)p[0]) << 56) + - (((i64)p[1]) << 48) + - (((i64)p[2]) << 40) + - (((i64)p[3]) << 32) + - (((i64)p[4]) << 24) + - (((i64)p[5]) << 16) + - (((i64)p[6]) << 8) + - (((i64)p[7]) << 0) - ); + return db->lastRowid; } /* -** Functions to serialize a 16 bit integer, 32 bit real number and -** 64 bit integer. The value returned is the number of bytes written -** to the argument buffer (always 2, 4 and 8 respectively). +** Return the number of changes in the most recent call to sqlite3_exec(). */ -static int writeInt16(u8 *p, int i){ - p[0] = (i>> 8)&0xFF; - p[1] = (i>> 0)&0xFF; - return 2; -} -static int writeCoord(u8 *p, RtreeCoord *pCoord){ - u32 i; - assert( sizeof(RtreeCoord)==4 ); - assert( sizeof(u32)==4 ); - i = pCoord->u; - p[0] = (i>>24)&0xFF; - p[1] = (i>>16)&0xFF; - p[2] = (i>> 8)&0xFF; - p[3] = (i>> 0)&0xFF; - return 4; -} -static int writeInt64(u8 *p, i64 i){ - p[0] = (i>>56)&0xFF; - p[1] = (i>>48)&0xFF; - p[2] = (i>>40)&0xFF; - p[3] = (i>>32)&0xFF; - p[4] = (i>>24)&0xFF; - p[5] = (i>>16)&0xFF; - p[6] = (i>> 8)&0xFF; - p[7] = (i>> 0)&0xFF; - return 8; +SQLITE_API int SQLITE_STDCALL sqlite3_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->nChange; } /* -** Increment the reference count of node p. +** Return the number of changes since the database handle was opened. */ -static void nodeReference(RtreeNode *p){ - if( p ){ - p->nRef++; +SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; } +#endif + return db->nTotalChange; } /* -** Clear the content of node p (set all bytes to 0x00). +** Close all open savepoints. This function only manipulates fields of the +** database handle object, it does not close any savepoints that may be open +** at the b-tree/pager level. */ -static void nodeZero(Rtree *pRtree, RtreeNode *p){ - memset(&p->zData[2], 0, pRtree->iNodeSize-2); - p->isDirty = 1; +SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ + while( db->pSavepoint ){ + Savepoint *pTmp = db->pSavepoint; + db->pSavepoint = pTmp->pNext; + sqlite3DbFree(db, pTmp); + } + db->nSavepoint = 0; + db->nStatement = 0; + db->isTransactionSavepoint = 0; } /* -** Given a node number iNode, return the corresponding key to use -** in the Rtree.aHash table. +** Invoke the destructor function associated with FuncDef p, if any. Except, +** if this is not the last copy of the function, do not invoke it. Multiple +** copies of a single function are created when create_function() is called +** with SQLITE_ANY as the encoding. */ -static int nodeHash(i64 iNode){ - return iNode % HASHSIZE; +static void functionDestroy(sqlite3 *db, FuncDef *p){ + FuncDestructor *pDestructor = p->pDestructor; + if( pDestructor ){ + pDestructor->nRef--; + if( pDestructor->nRef==0 ){ + pDestructor->xDestroy(pDestructor->pUserData); + sqlite3DbFree(db, pDestructor); + } + } } /* -** Search the node hash table for node iNode. If found, return a pointer -** to it. Otherwise, return 0. +** Disconnect all sqlite3_vtab objects that belong to database connection +** db. This is called when db is being closed. */ -static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ - RtreeNode *p; - for(p=pRtree->aHash[nodeHash(iNode)]; p && p->iNode!=iNode; p=p->pNext); - return p; +static void disconnectAllVtab(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int i; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( db->aDb[i].pSchema ){ + HashElem *p; + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + Table *pTab = (Table *)sqliteHashData(p); + if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); + } + } + } + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); +#else + UNUSED_PARAMETER(db); +#endif } /* -** Add node pNode to the node hash table. +** Return TRUE if database connection db has unfinalized prepared +** statements or unfinished sqlite3_backup objects. */ -static void nodeHashInsert(Rtree *pRtree, RtreeNode *pNode){ - int iHash; - assert( pNode->pNext==0 ); - iHash = nodeHash(pNode->iNode); - pNode->pNext = pRtree->aHash[iHash]; - pRtree->aHash[iHash] = pNode; +static int connectionIsBusy(sqlite3 *db){ + int j; + assert( sqlite3_mutex_held(db->mutex) ); + if( db->pVdbe ) return 1; + for(j=0; jnDb; j++){ + Btree *pBt = db->aDb[j].pBt; + if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; + } + return 0; } /* -** Remove node pNode from the node hash table. +** Close an existing SQLite database */ -static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){ - RtreeNode **pp; - if( pNode->iNode!=0 ){ - pp = &pRtree->aHash[nodeHash(pNode->iNode)]; - for( ; (*pp)!=pNode; pp = &(*pp)->pNext){ assert(*pp); } - *pp = pNode->pNext; - pNode->pNext = 0; +static int sqlite3Close(sqlite3 *db, int forceZombie){ + if( !db ){ + /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or + ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ + return SQLITE_OK; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; } + sqlite3_mutex_enter(db->mutex); + + /* Force xDisconnect calls on all virtual tables */ + disconnectAllVtab(db); + + /* If a transaction is open, the disconnectAllVtab() call above + ** will not have called the xDisconnect() method on any virtual + ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() + ** call will do so. We need to do this before the check for active + ** SQL statements below, as the v-table implementation may be storing + ** some prepared statements internally. + */ + sqlite3VtabRollback(db); + + /* Legacy behavior (sqlite3_close() behavior) is to return + ** SQLITE_BUSY if the connection can not be closed immediately. + */ + if( !forceZombie && connectionIsBusy(db) ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " + "statements or unfinished backups"); + sqlite3_mutex_leave(db->mutex); + return SQLITE_BUSY; + } + +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + } +#endif + + /* Convert the connection into a zombie and then close it. + */ + db->magic = SQLITE_MAGIC_ZOMBIE; + sqlite3LeaveMutexAndCloseZombie(db); + return SQLITE_OK; } /* -** Allocate and return new r-tree node. Initially, (RtreeNode.iNode==0), -** indicating that node has not yet been assigned a node number. It is -** assigned a node number when nodeWrite() is called to write the -** node contents out to the database. +** Two variations on the public interface for closing a database +** connection. The sqlite3_close() version returns SQLITE_BUSY and +** leaves the connection option if there are unfinalized prepared +** statements or unfinished sqlite3_backups. The sqlite3_close_v2() +** version forces the connection to become a zombie if there are +** unclosed resources, and arranges for deallocation when the last +** prepare statement or sqlite3_backup closes. */ -static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ - RtreeNode *pNode; - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); - if( pNode ){ - memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); - pNode->zData = (u8 *)&pNode[1]; - pNode->nRef = 1; - pNode->pParent = pParent; - pNode->isDirty = 1; - nodeReference(pParent); - } - return pNode; -} +SQLITE_API int SQLITE_STDCALL sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } +SQLITE_API int SQLITE_STDCALL sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } + /* -** Obtain a reference to an r-tree node. +** Close the mutex on database connection db. +** +** Furthermore, if database connection db is a zombie (meaning that there +** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and +** every sqlite3_stmt has now been finalized and every sqlite3_backup has +** finished, then free all resources. */ -static int nodeAcquire( - Rtree *pRtree, /* R-tree structure */ - i64 iNode, /* Node number to load */ - RtreeNode *pParent, /* Either the parent node or NULL */ - RtreeNode **ppNode /* OUT: Acquired node */ -){ - int rc; - int rc2 = SQLITE_OK; - RtreeNode *pNode; +SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ + HashElem *i; /* Hash table iterator */ + int j; - /* Check if the requested node is already in the hash table. If so, - ** increase its reference count and return it. + /* If there are outstanding sqlite3_stmt or sqlite3_backup objects + ** or if the connection has not yet been closed by sqlite3_close_v2(), + ** then just leave the mutex and return. */ - if( (pNode = nodeHashLookup(pRtree, iNode)) ){ - assert( !pParent || !pNode->pParent || pNode->pParent==pParent ); - if( pParent && !pNode->pParent ){ - nodeReference(pParent); - pNode->pParent = pParent; - } - pNode->nRef++; - *ppNode = pNode; - return SQLITE_OK; + if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ + sqlite3_mutex_leave(db->mutex); + return; } - sqlite3_bind_int64(pRtree->pReadNode, 1, iNode); - rc = sqlite3_step(pRtree->pReadNode); - if( rc==SQLITE_ROW ){ - const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); - if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){ - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); - if( !pNode ){ - rc2 = SQLITE_NOMEM; - }else{ - pNode->pParent = pParent; - pNode->zData = (u8 *)&pNode[1]; - pNode->nRef = 1; - pNode->iNode = iNode; - pNode->isDirty = 0; - pNode->pNext = 0; - memcpy(pNode->zData, zBlob, pRtree->iNodeSize); - nodeReference(pParent); + /* If we reach this point, it means that the database connection has + ** closed all sqlite3_stmt and sqlite3_backup objects and has been + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, + ** go ahead and free all resources. + */ + + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); + + /* Free any outstanding Savepoint structures. */ + sqlite3CloseSavepoints(db); + + /* Close all database connections */ + for(j=0; jnDb; j++){ + struct Db *pDb = &db->aDb[j]; + if( pDb->pBt ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + if( j!=1 ){ + pDb->pSchema = 0; } } } - rc = sqlite3_reset(pRtree->pReadNode); - if( rc==SQLITE_OK ) rc = rc2; + /* Clear the TEMP schema separately and last */ + if( db->aDb[1].pSchema ){ + sqlite3SchemaClear(db->aDb[1].pSchema); + } + sqlite3VtabUnlockList(db); - /* If the root node was just loaded, set pRtree->iDepth to the height - ** of the r-tree structure. A height of zero means all data is stored on - ** the root node. A height of one means the children of the root node - ** are the leaves, and so on. If the depth as specified on the root node - ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt. + /* Free up the array of auxiliary databases */ + sqlite3CollapseDatabaseArray(db); + assert( db->nDb<=2 ); + assert( db->aDb==db->aDbStatic ); + + /* Tell the code in notify.c that the connection no longer holds any + ** locks and does not require any further unlock-notify callbacks. */ - if( pNode && iNode==1 ){ - pRtree->iDepth = readInt16(pNode->zData); - if( pRtree->iDepth>RTREE_MAX_DEPTH ){ - rc = SQLITE_CORRUPT_VTAB; + sqlite3ConnectionClosed(db); + + for(j=0; jaFunc.a); j++){ + FuncDef *pNext, *pHash, *p; + for(p=db->aFunc.a[j]; p; p=pHash){ + pHash = p->pHash; + while( p ){ + functionDestroy(db, p); + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + } } } - - /* If no error has occurred so far, check if the "number of entries" - ** field on the node is too large. If so, set the return code to - ** SQLITE_CORRUPT_VTAB. - */ - if( pNode && rc==SQLITE_OK ){ - if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ - rc = SQLITE_CORRUPT_VTAB; + for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(i); + /* Invoke any destructors registered for collation sequence user data. */ + for(j=0; j<3; j++){ + if( pColl[j].xDel ){ + pColl[j].xDel(pColl[j].pUser); + } } + sqlite3DbFree(db, pColl); } - - if( rc==SQLITE_OK ){ - if( pNode!=0 ){ - nodeHashInsert(pRtree, pNode); - }else{ - rc = SQLITE_CORRUPT_VTAB; + sqlite3HashClear(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ + Module *pMod = (Module *)sqliteHashData(i); + if( pMod->xDestroy ){ + pMod->xDestroy(pMod->pAux); } - *ppNode = pNode; - }else{ - sqlite3_free(pNode); - *ppNode = 0; + sqlite3DbFree(db, pMod); } + sqlite3HashClear(&db->aModule); +#endif - return rc; -} + sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ + sqlite3ValueFree(db->pErr); + sqlite3CloseExtensions(db); +#if SQLITE_USER_AUTHENTICATION + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); +#endif -/* -** Overwrite cell iCell of node pNode with the contents of pCell. -*/ -static void nodeOverwriteCell( - Rtree *pRtree, /* The overall R-Tree */ - RtreeNode *pNode, /* The node into which the cell is to be written */ - RtreeCell *pCell, /* The cell to write */ - int iCell /* Index into pNode into which pCell is written */ -){ - int ii; - u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; - p += writeInt64(p, pCell->iRowid); - for(ii=0; ii<(pRtree->nDim*2); ii++){ - p += writeCoord(p, &pCell->aCoord[ii]); + db->magic = SQLITE_MAGIC_ERROR; + + /* The temp-database schema is allocated differently from the other schema + ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). + ** So it needs to be freed here. Todo: Why not roll the temp schema into + ** the same sqliteMalloc() as the one that allocates the database + ** structure? + */ + sqlite3DbFree(db, db->aDb[1].pSchema); + sqlite3_mutex_leave(db->mutex); + db->magic = SQLITE_MAGIC_CLOSED; + sqlite3_mutex_free(db->mutex); + assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); } - pNode->isDirty = 1; + sqlite3_free(db); } /* -** Remove the cell with index iCell from node pNode. +** Rollback all database files. If tripCode is not SQLITE_OK, then +** any write cursors are invalidated ("tripped" - as in "tripping a circuit +** breaker") and made to return tripCode if there are any further +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. */ -static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ - u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; - u8 *pSrc = &pDst[pRtree->nBytesPerCell]; - int nByte = (NCELL(pNode) - iCell - 1) * pRtree->nBytesPerCell; - memmove(pDst, pSrc, nByte); - writeInt16(&pNode->zData[2], NCELL(pNode)-1); - pNode->isDirty = 1; -} +SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ + int i; + int inTrans = 0; + int schemaChange; + assert( sqlite3_mutex_held(db->mutex) ); + sqlite3BeginBenignMalloc(); -/* -** Insert the contents of cell pCell into node pNode. If the insert -** is successful, return SQLITE_OK. -** -** If there is not enough free space in pNode, return SQLITE_FULL. -*/ -static int nodeInsertCell( - Rtree *pRtree, /* The overall R-Tree */ - RtreeNode *pNode, /* Write new cell into this node */ - RtreeCell *pCell /* The cell to be inserted */ -){ - int nCell; /* Current number of cells in pNode */ - int nMaxCell; /* Maximum number of cells for pNode */ + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; - nMaxCell = (pRtree->iNodeSize-4)/pRtree->nBytesPerCell; - nCell = NCELL(pNode); + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p ){ + if( sqlite3BtreeIsInTrans(p) ){ + inTrans = 1; + } + sqlite3BtreeRollback(p, tripCode, !schemaChange); + } + } + sqlite3VtabRollback(db); + sqlite3EndBenignMalloc(); - assert( nCell<=nMaxCell ); - if( nCellzData[2], nCell+1); - pNode->isDirty = 1; + if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ + sqlite3ExpirePreparedStatements(db); + sqlite3ResetAllSchemasOfConnection(db); } + sqlite3BtreeLeaveAll(db); - return (nCell==nMaxCell); + /* Any deferred constraint violations have now been resolved. */ + db->nDeferredCons = 0; + db->nDeferredImmCons = 0; + db->flags &= ~SQLITE_DeferFKs; + + /* If one has been configured, invoke the rollback-hook callback */ + if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ + db->xRollbackCallback(db->pRollbackArg); + } } /* -** If the node is dirty, write it out to the database. +** Return a static string containing the name corresponding to the error code +** specified in the argument. */ -static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ - int rc = SQLITE_OK; - if( pNode->isDirty ){ - sqlite3_stmt *p = pRtree->pWriteNode; - if( pNode->iNode ){ - sqlite3_bind_int64(p, 1, pNode->iNode); - }else{ - sqlite3_bind_null(p, 1); - } - sqlite3_bind_blob(p, 2, pNode->zData, pRtree->iNodeSize, SQLITE_STATIC); - sqlite3_step(p); - pNode->isDirty = 0; - rc = sqlite3_reset(p); - if( pNode->iNode==0 && rc==SQLITE_OK ){ - pNode->iNode = sqlite3_last_insert_rowid(pRtree->db); - nodeHashInsert(pRtree, pNode); +#if defined(SQLITE_NEED_ERR_NAME) +SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; + case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; } } - return rc; + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; } +#endif /* -** Release a reference to a node. If the node is dirty and the reference -** count drops to zero, the node data is written to the database. +** Return a static string that describes the kind of error specified in the +** argument. */ -static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ - int rc = SQLITE_OK; - if( pNode ){ - assert( pNode->nRef>0 ); - pNode->nRef--; - if( pNode->nRef==0 ){ - if( pNode->iNode==1 ){ - pRtree->iDepth = -1; - } - if( pNode->pParent ){ - rc = nodeRelease(pRtree, pNode->pParent); - } - if( rc==SQLITE_OK ){ - rc = nodeWrite(pRtree, pNode); +SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ + static const char* const aMsg[] = { + /* SQLITE_OK */ "not an error", + /* SQLITE_ERROR */ "SQL logic error or missing database", + /* SQLITE_INTERNAL */ 0, + /* SQLITE_PERM */ "access permission denied", + /* SQLITE_ABORT */ "callback requested query abort", + /* SQLITE_BUSY */ "database is locked", + /* SQLITE_LOCKED */ "database table is locked", + /* SQLITE_NOMEM */ "out of memory", + /* SQLITE_READONLY */ "attempt to write a readonly database", + /* SQLITE_INTERRUPT */ "interrupted", + /* SQLITE_IOERR */ "disk I/O error", + /* SQLITE_CORRUPT */ "database disk image is malformed", + /* SQLITE_NOTFOUND */ "unknown operation", + /* SQLITE_FULL */ "database or disk is full", + /* SQLITE_CANTOPEN */ "unable to open database file", + /* SQLITE_PROTOCOL */ "locking protocol", + /* SQLITE_EMPTY */ "table contains no data", + /* SQLITE_SCHEMA */ "database schema has changed", + /* SQLITE_TOOBIG */ "string or blob too big", + /* SQLITE_CONSTRAINT */ "constraint failed", + /* SQLITE_MISMATCH */ "datatype mismatch", + /* SQLITE_MISUSE */ "library routine called out of sequence", + /* SQLITE_NOLFS */ "large file support is disabled", + /* SQLITE_AUTH */ "authorization denied", + /* SQLITE_FORMAT */ "auxiliary database format error", + /* SQLITE_RANGE */ "bind or column index out of range", + /* SQLITE_NOTADB */ "file is encrypted or is not a database", + }; + const char *zErr = "unknown error"; + switch( rc ){ + case SQLITE_ABORT_ROLLBACK: { + zErr = "abort due to ROLLBACK"; + break; + } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rczData[4 + pRtree->nBytesPerCell*iCell]); -} +#if SQLITE_OS_WIN || HAVE_USLEEP + static const u8 delays[] = + { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; + static const u8 totals[] = + { 0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228 }; +# define NDELAY ArraySize(delays) + sqlite3 *db = (sqlite3 *)ptr; + int timeout = db->busyTimeout; + int delay, prior; -/* -** Return coordinate iCoord from cell iCell in node pNode. -*/ -static void nodeGetCoord( - Rtree *pRtree, /* The overall R-Tree */ - RtreeNode *pNode, /* The node from which to extract a coordinate */ - int iCell, /* The index of the cell within the node */ - int iCoord, /* Which coordinate to extract */ - RtreeCoord *pCoord /* OUT: Space to write result to */ -){ - readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); + assert( count>=0 ); + if( count < NDELAY ){ + delay = delays[count]; + prior = totals[count]; + }else{ + delay = delays[NDELAY-1]; + prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); + } + if( prior + delay > timeout ){ + delay = timeout - prior; + if( delay<=0 ) return 0; + } + sqlite3OsSleep(db->pVfs, delay*1000); + return 1; +#else + sqlite3 *db = (sqlite3 *)ptr; + int timeout = ((sqlite3 *)ptr)->busyTimeout; + if( (count+1)*1000 > timeout ){ + return 0; + } + sqlite3OsSleep(db->pVfs, 1000000); + return 1; +#endif } /* -** Deserialize cell iCell of node pNode. Populate the structure pointed -** to by pCell with the results. +** Invoke the given busy handler. +** +** This routine is called when an operation failed with a lock. +** If this routine returns non-zero, the lock is retried. If it +** returns 0, the operation aborts with an SQLITE_BUSY error. */ -static void nodeGetCell( - Rtree *pRtree, /* The overall R-Tree */ - RtreeNode *pNode, /* The node containing the cell to be read */ - int iCell, /* Index of the cell within the node */ - RtreeCell *pCell /* OUT: Write the cell contents here */ -){ - u8 *pData; - RtreeCoord *pCoord; - int ii; - pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); - pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); - pCoord = pCell->aCoord; - for(ii=0; iinDim*2; ii++){ - readCoord(&pData[ii*4], &pCoord[ii]); +SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ + int rc; + if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0; + rc = p->xFunc(p->pArg, p->nBusy); + if( rc==0 ){ + p->nBusy = -1; + }else{ + p->nBusy++; } + return rc; } - -/* Forward declaration for the function that does the work of -** the virtual table module xCreate() and xConnect() methods. -*/ -static int rtreeInit( - sqlite3 *, void *, int, const char *const*, sqlite3_vtab **, char **, int -); - -/* -** Rtree virtual table module xCreate method. +/* +** This routine sets the busy callback for an Sqlite database to the +** given callback function with the given argument. */ -static int rtreeCreate( +SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler( sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr + int (*xBusy)(void*,int), + void *pArg ){ - return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 1); +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->busyHandler.xFunc = xBusy; + db->busyHandler.pArg = pArg; + db->busyHandler.nBusy = 0; + db->busyTimeout = 0; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; } -/* -** Rtree virtual table module xConnect method. +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** This routine sets the progress callback for an Sqlite database to the +** given callback function with the given argument. The progress callback will +** be invoked every nOps opcodes. */ -static int rtreeConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr +SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler( + sqlite3 *db, + int nOps, + int (*xProgress)(void*), + void *pArg ){ - return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 0); +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( nOps>0 ){ + db->xProgress = xProgress; + db->nProgressOps = (unsigned)nOps; + db->pProgressArg = pArg; + }else{ + db->xProgress = 0; + db->nProgressOps = 0; + db->pProgressArg = 0; + } + sqlite3_mutex_leave(db->mutex); } +#endif + /* -** Increment the r-tree reference count. +** This routine installs a default busy handler that waits for the +** specified number of milliseconds before returning 0. */ -static void rtreeReference(Rtree *pRtree){ - pRtree->nBusy++; +SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3 *db, int ms){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( ms>0 ){ + sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); + db->busyTimeout = ms; + }else{ + sqlite3_busy_handler(db, 0, 0); + } + return SQLITE_OK; } /* -** Decrement the r-tree reference count. When the reference count reaches -** zero the structure is deleted. +** Cause any pending operation to stop at its earliest opportunity. */ -static void rtreeRelease(Rtree *pRtree){ - pRtree->nBusy--; - if( pRtree->nBusy==0 ){ - sqlite3_finalize(pRtree->pReadNode); - sqlite3_finalize(pRtree->pWriteNode); - sqlite3_finalize(pRtree->pDeleteNode); - sqlite3_finalize(pRtree->pReadRowid); - sqlite3_finalize(pRtree->pWriteRowid); - sqlite3_finalize(pRtree->pDeleteRowid); - sqlite3_finalize(pRtree->pReadParent); - sqlite3_finalize(pRtree->pWriteParent); - sqlite3_finalize(pRtree->pDeleteParent); - sqlite3_free(pRtree); +SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; } +#endif + db->u1.isInterrupted = 1; } -/* -** Rtree virtual table module xDisconnect method. + +/* +** This function is exactly the same as sqlite3_create_function(), except +** that it is designed to be called by internal code. The difference is +** that if a malloc() fails in sqlite3_create_function(), an error code +** is returned and the mallocFailed flag cleared. */ -static int rtreeDisconnect(sqlite3_vtab *pVtab){ - rtreeRelease((Rtree *)pVtab); +SQLITE_PRIVATE int sqlite3CreateFunc( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int enc, + void *pUserData, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + FuncDestructor *pDestructor +){ + FuncDef *p; + int nName; + int extraFlags; + + assert( sqlite3_mutex_held(db->mutex) ); + if( zFunctionName==0 || + (xFunc && (xFinal || xStep)) || + (!xFunc && (xFinal && !xStep)) || + (!xFunc && (!xFinal && xStep)) || + (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || + (255<(nName = sqlite3Strlen30( zFunctionName))) ){ + return SQLITE_MISUSE_BKPT; + } + + assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); + extraFlags = enc & SQLITE_DETERMINISTIC; + enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); + +#ifndef SQLITE_OMIT_UTF16 + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + ** + ** If SQLITE_ANY is specified, add three versions of the function + ** to the hash table. + */ + if( enc==SQLITE_UTF16 ){ + enc = SQLITE_UTF16NATIVE; + }else if( enc==SQLITE_ANY ){ + int rc; + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, + pUserData, xFunc, xStep, xFinal, pDestructor); + if( rc==SQLITE_OK ){ + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, + pUserData, xFunc, xStep, xFinal, pDestructor); + } + if( rc!=SQLITE_OK ){ + return rc; + } + enc = SQLITE_UTF16BE; + } +#else + enc = SQLITE_UTF8; +#endif + + /* Check if an existing function is being overridden or deleted. If so, + ** and there are active VMs, then return SQLITE_BUSY. If a function + ** is being overridden/deleted but there are no active VMs, allow the + ** operation to continue but invalidate all precompiled statements. + */ + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); + if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify user-function due to active statements"); + assert( !db->mallocFailed ); + return SQLITE_BUSY; + }else{ + sqlite3ExpirePreparedStatements(db); + } + } + + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); + assert(p || db->mallocFailed); + if( !p ){ + return SQLITE_NOMEM; + } + + /* If an older version of the function with a configured destructor is + ** being replaced invoke the destructor function here. */ + functionDestroy(db, p); + + if( pDestructor ){ + pDestructor->nRef++; + } + p->pDestructor = pDestructor; + p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; + testcase( p->funcFlags & SQLITE_DETERMINISTIC ); + p->xFunc = xFunc; + p->xStep = xStep; + p->xFinalize = xFinal; + p->pUserData = pUserData; + p->nArg = (u16)nArg; return SQLITE_OK; } -/* -** Rtree virtual table module xDestroy method. +/* +** Create new user functions. */ -static int rtreeDestroy(sqlite3_vtab *pVtab){ - Rtree *pRtree = (Rtree *)pVtab; - int rc; - char *zCreate = sqlite3_mprintf( - "DROP TABLE '%q'.'%q_node';" - "DROP TABLE '%q'.'%q_rowid';" - "DROP TABLE '%q'.'%q_parent';", - pRtree->zDb, pRtree->zName, - pRtree->zDb, pRtree->zName, - pRtree->zDb, pRtree->zName - ); - if( !zCreate ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0); - sqlite3_free(zCreate); +SQLITE_API int SQLITE_STDCALL sqlite3_create_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*) +){ + return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, + xFinal, 0); +} + +SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + int rc = SQLITE_ERROR; + FuncDestructor *pArg = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; } - if( rc==SQLITE_OK ){ - rtreeRelease(pRtree); +#endif + sqlite3_mutex_enter(db->mutex); + if( xDestroy ){ + pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); + if( !pArg ){ + xDestroy(p); + goto out; + } + pArg->xDestroy = xDestroy; + pArg->pUserData = p; + } + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); + if( pArg && pArg->nRef==0 ){ + assert( rc!=SQLITE_OK ); + xDestroy(p); + sqlite3DbFree(db, pArg); } + out: + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); return rc; } -/* -** Rtree virtual table module xOpen method. -*/ -static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - int rc = SQLITE_NOMEM; - RtreeCursor *pCsr; - - pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor)); - if( pCsr ){ - memset(pCsr, 0, sizeof(RtreeCursor)); - pCsr->base.pVtab = pVTab; - rc = SQLITE_OK; - } - *ppCursor = (sqlite3_vtab_cursor *)pCsr; +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +){ + int rc; + char *zFunc8; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); + sqlite3DbFree(db, zFunc8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); return rc; } +#endif /* -** Free the RtreeCursor.aConstraint[] array and its contents. +** Declare that a function has been overloaded by a virtual table. +** +** If the function already exists as a regular global function, then +** this routine is a no-op. If the function does not exist, then create +** a new one that always throws a run-time error. +** +** When virtual tables intend to provide an overloaded function, they +** should call this routine to make sure the global function exists. +** A global function must exist in order for name resolution to work +** properly. */ -static void freeCursorConstraints(RtreeCursor *pCsr){ - if( pCsr->aConstraint ){ - int i; /* Used to iterate through constraint array */ - for(i=0; inConstraint; i++){ - sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; - if( pInfo ){ - if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); - sqlite3_free(pInfo); - } - } - sqlite3_free(pCsr->aConstraint); - pCsr->aConstraint = 0; - } -} +SQLITE_API int SQLITE_STDCALL sqlite3_overload_function( + sqlite3 *db, + const char *zName, + int nArg +){ + int nName = sqlite3Strlen30(zName); + int rc = SQLITE_OK; -/* -** Rtree virtual table module xClose method. -*/ -static int rtreeClose(sqlite3_vtab_cursor *cur){ - Rtree *pRtree = (Rtree *)(cur->pVtab); - int ii; - RtreeCursor *pCsr = (RtreeCursor *)cur; - freeCursorConstraints(pCsr); - sqlite3_free(pCsr->aPoint); - for(ii=0; iiaNode[ii]); - sqlite3_free(pCsr); - return SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ + rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, + 0, sqlite3InvalidFunction, 0, 0, 0); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } +#ifndef SQLITE_OMIT_TRACE /* -** Rtree virtual table module xEof method. +** Register a trace function. The pArg from the previously registered trace +** is returned. ** -** Return non-zero if the cursor does not currently point to a valid -** record (i.e if the scan has finished), or zero otherwise. +** A NULL trace function means that no tracing is executes. A non-NULL +** trace is a pointer to a function that is invoked at the start of each +** SQL statement. */ -static int rtreeEof(sqlite3_vtab_cursor *cur){ - RtreeCursor *pCsr = (RtreeCursor *)cur; - return pCsr->atEOF; -} +SQLITE_API void *SQLITE_STDCALL sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ + void *pOld; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pTraceArg; + db->xTrace = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} /* -** Convert raw bits from the on-disk RTree record into a coordinate value. -** The on-disk format is big-endian and needs to be converted for little- -** endian platforms. The on-disk record stores integer coordinates if -** eInt is true and it stores 32-bit floating point records if eInt is -** false. a[] is the four bytes of the on-disk record to be decoded. -** Store the results in "r". +** Register a profile function. The pArg from the previously registered +** profile function is returned. ** -** There are three versions of this macro, one each for little-endian and -** big-endian processors and a third generic implementation. The endian- -** specific implementations are much faster and are preferred if the -** processor endianness is known at compile-time. The SQLITE_BYTEORDER -** macro is part of sqliteInt.h and hence the endian-specific -** implementation will only be used if this module is compiled as part -** of the amalgamation. +** A NULL profile function means that no profiling is executes. A non-NULL +** profile is a pointer to a function that is invoked at the conclusion of +** each SQL statement that is run. */ -#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 -#define RTREE_DECODE_COORD(eInt, a, r) { \ - RtreeCoord c; /* Coordinate decoded */ \ - memcpy(&c.u,a,4); \ - c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ - ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ - r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ -} -#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 -#define RTREE_DECODE_COORD(eInt, a, r) { \ - RtreeCoord c; /* Coordinate decoded */ \ - memcpy(&c.u,a,4); \ - r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ -} -#else -#define RTREE_DECODE_COORD(eInt, a, r) { \ - RtreeCoord c; /* Coordinate decoded */ \ - c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ - +((u32)a[2]<<8) + a[3]; \ - r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ -} +SQLITE_API void *SQLITE_STDCALL sqlite3_profile( + sqlite3 *db, + void (*xProfile)(void*,const char*,sqlite_uint64), + void *pArg +){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } #endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pProfileArg; + db->xProfile = xProfile; + db->pProfileArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +#endif /* SQLITE_OMIT_TRACE */ /* -** Check the RTree node or entry given by pCellData and p against the MATCH -** constraint pConstraint. +** Register a function to be invoked when a transaction commits. +** If the invoked function returns non-zero, then the commit becomes a +** rollback. */ -static int rtreeCallbackConstraint( - RtreeConstraint *pConstraint, /* The constraint to test */ - int eInt, /* True if RTree holding integer coordinates */ - u8 *pCellData, /* Raw cell content */ - RtreeSearchPoint *pSearch, /* Container of this cell */ - sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ - int *peWithin /* OUT: visibility of the cell */ +SQLITE_API void *SQLITE_STDCALL sqlite3_commit_hook( + sqlite3 *db, /* Attach the hook to this database */ + int (*xCallback)(void*), /* Function to invoke on each commit */ + void *pArg /* Argument to the function */ ){ - int i; /* Loop counter */ - sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ - int nCoord = pInfo->nCoord; /* No. of coordinates */ - int rc; /* Callback return code */ - sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ - - assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); - assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); + void *pOld; - if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ - pInfo->iRowid = readInt64(pCellData); - } - pCellData += 8; - for(i=0; iop==RTREE_MATCH ){ - rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, - nCoord, aCoord, &i); - if( i==0 ) *peWithin = NOT_WITHIN; - *prScore = RTREE_ZERO; - }else{ - pInfo->aCoord = aCoord; - pInfo->iLevel = pSearch->iLevel - 1; - pInfo->rScore = pInfo->rParentScore = pSearch->rScore; - pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; - rc = pConstraint->u.xQueryFunc(pInfo); - if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; - if( pInfo->rScore<*prScore || *prScorerScore; - } +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; } - return rc; +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pCommitArg; + db->xCommitCallback = xCallback; + db->pCommitArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; } -/* -** Check the internal RTree node given by pCellData against constraint p. -** If this constraint cannot be satisfied by any child within the node, -** set *peWithin to NOT_WITHIN. +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. */ -static void rtreeNonleafConstraint( - RtreeConstraint *p, /* The constraint to test */ - int eInt, /* True if RTree holds integer coordinates */ - u8 *pCellData, /* Raw cell content as appears on disk */ - int *peWithin /* Adjust downward, as appropriate */ +SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), + void *pArg /* Argument to the function */ ){ - sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ - - /* p->iCoord might point to either a lower or upper bound coordinate - ** in a coordinate pair. But make pCellData point to the lower bound. - */ - pCellData += 8 + 4*(p->iCoord&0xfe); - - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ ); - switch( p->op ){ - case RTREE_LE: - case RTREE_LT: - case RTREE_EQ: - RTREE_DECODE_COORD(eInt, pCellData, val); - /* val now holds the lower bound of the coordinate pair */ - if( p->u.rValue>=val ) return; - if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ - /* Fall through for the RTREE_EQ case */ + void *pRet; - default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ - pCellData += 4; - RTREE_DECODE_COORD(eInt, pCellData, val); - /* val now holds the upper bound of the coordinate pair */ - if( p->u.rValue<=val ) return; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; } - *peWithin = NOT_WITHIN; +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pUpdateArg; + db->xUpdateCallback = xCallback; + db->pUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; } /* -** Check the leaf RTree cell given by pCellData against constraint p. -** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. -** If the constraint is satisfied, leave *peWithin unchanged. -** -** The constraint is of the form: xN op $val -** -** The op is given by p->op. The xN is p->iCoord-th coordinate in -** pCellData. $val is given by p->u.rValue. +** Register a callback to be invoked each time a transaction is rolled +** back by this database connection. */ -static void rtreeLeafConstraint( - RtreeConstraint *p, /* The constraint to test */ - int eInt, /* True if RTree holds integer coordinates */ - u8 *pCellData, /* Raw cell content as appears on disk */ - int *peWithin /* Adjust downward, as appropriate */ +SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*), /* Callback function */ + void *pArg /* Argument to the function */ ){ - RtreeDValue xN; /* Coordinate value converted to a double */ + void *pRet; - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ ); - pCellData += 8 + p->iCoord*4; - RTREE_DECODE_COORD(eInt, pCellData, xN); - switch( p->op ){ - case RTREE_LE: if( xN <= p->u.rValue ) return; break; - case RTREE_LT: if( xN < p->u.rValue ) return; break; - case RTREE_GE: if( xN >= p->u.rValue ) return; break; - case RTREE_GT: if( xN > p->u.rValue ) return; break; - default: if( xN == p->u.rValue ) return; break; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; } - *peWithin = NOT_WITHIN; +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pRollbackArg; + db->xRollbackCallback = xCallback; + db->pRollbackArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; } +#ifndef SQLITE_OMIT_WAL /* -** One of the cells in node pNode is guaranteed to have a 64-bit -** integer value equal to iRowid. Return the index of this cell. -*/ -static int nodeRowidIndex( - Rtree *pRtree, - RtreeNode *pNode, - i64 iRowid, - int *piIndex +** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). +** Invoke sqlite3_wal_checkpoint if the number of frames in the log file +** is greater than sqlite3.pWalArg cast to an integer (the value configured by +** wal_autocheckpoint()). +*/ +SQLITE_PRIVATE int sqlite3WalDefaultHook( + void *pClientData, /* Argument */ + sqlite3 *db, /* Connection */ + const char *zDb, /* Database */ + int nFrame /* Size of WAL */ ){ - int ii; - int nCell = NCELL(pNode); - assert( nCell<200 ); - for(ii=0; ii=SQLITE_PTR_TO_INT(pClientData) ){ + sqlite3BeginBenignMalloc(); + sqlite3_wal_checkpoint(db, zDb); + sqlite3EndBenignMalloc(); } - return SQLITE_CORRUPT_VTAB; + return SQLITE_OK; } +#endif /* SQLITE_OMIT_WAL */ /* -** Return the index of the cell containing a pointer to node pNode -** in its parent. If pNode is the root node, return -1. +** Configure an sqlite3_wal_hook() callback to automatically checkpoint +** a database after committing a transaction if there are nFrame or +** more frames in the log file. Passing zero or a negative value as the +** nFrame parameter disables automatic checkpoints entirely. +** +** The callback registered by this function replaces any existing callback +** registered using sqlite3_wal_hook(). Likewise, registering a callback +** using sqlite3_wal_hook() disables the automatic checkpoint mechanism +** configured by this function. */ -static int nodeParentIndex(Rtree *pRtree, RtreeNode *pNode, int *piIndex){ - RtreeNode *pParent = pNode->pParent; - if( pParent ){ - return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex); +SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ +#ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(nFrame); +#else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( nFrame>0 ){ + sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); + }else{ + sqlite3_wal_hook(db, 0, 0); } - *piIndex = -1; +#endif return SQLITE_OK; } /* -** Compare two search points. Return negative, zero, or positive if the first -** is less than, equal to, or greater than the second. -** -** The rScore is the primary key. Smaller rScore values come first. -** If the rScore is a tie, then use iLevel as the tie breaker with smaller -** iLevel values coming first. In this way, if rScore is the same for all -** SearchPoints, then iLevel becomes the deciding factor and the result -** is a depth-first search, which is the desired default behavior. +** Register a callback to be invoked each time a transaction is written +** into the write-ahead-log by this database connection. */ -static int rtreeSearchPointCompare( - const RtreeSearchPoint *pA, - const RtreeSearchPoint *pB +SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( + sqlite3 *db, /* Attach the hook to this db handle */ + int(*xCallback)(void *, sqlite3*, const char*, int), + void *pArg /* First argument passed to xCallback() */ ){ - if( pA->rScorerScore ) return -1; - if( pA->rScore>pB->rScore ) return +1; - if( pA->iLeveliLevel ) return -1; - if( pA->iLevel>pB->iLevel ) return +1; +#ifndef SQLITE_OMIT_WAL + void *pRet; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pWalArg; + db->xWalCallback = xCallback; + db->pWalArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +#else return 0; +#endif } /* -** Interchange to search points in a cursor. +** Checkpoint database zDb. */ -static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ - RtreeSearchPoint t = p->aPoint[i]; - assert( iaPoint[i] = p->aPoint[j]; - p->aPoint[j] = t; - i++; j++; - if( i=RTREE_CACHE_SZ ){ - nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); - p->aNode[i] = 0; - }else{ - RtreeNode *pTemp = p->aNode[i]; - p->aNode[i] = p->aNode[j]; - p->aNode[j] = pTemp; - } +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +){ +#ifdef SQLITE_OMIT_WAL + return SQLITE_OK; +#else + int rc; /* Return code */ + int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + + /* Initialize the output variables to -1 in case an error occurs. */ + if( pnLog ) *pnLog = -1; + if( pnCkpt ) *pnCkpt = -1; + + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint + ** mode: */ + return SQLITE_MISUSE; + } + + sqlite3_mutex_enter(db->mutex); + if( zDb && zDb[0] ){ + iDb = sqlite3FindDbName(db, zDb); } + if( iDb<0 ){ + rc = SQLITE_ERROR; + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); + }else{ + db->busyHandler.nBusy = 0; + rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); + sqlite3Error(db, rc); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +#endif } + /* -** Return the search point with the lowest current score. +** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points +** to contains a zero-length string, all attached databases are +** checkpointed. */ -static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ - return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ + /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to + ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ + return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); } +#ifndef SQLITE_OMIT_WAL /* -** Get the RtreeNode for the search point with the lowest score. +** Run a checkpoint on database iDb. This is a no-op if database iDb is +** not currently open in WAL mode. +** +** If a transaction is open on the database being checkpointed, this +** function returns SQLITE_LOCKED and a checkpoint is not attempted. If +** an error occurs while running the checkpoint, an SQLite error code is +** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** +** The mutex on database handle db should be held by the caller. The mutex +** associated with the specific b-tree being checkpointed is taken by +** this function while the checkpoint is running. +** +** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are +** checkpointed. If an error is encountered it is returned immediately - +** no attempt is made to checkpoint any remaining databases. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. */ -static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ - sqlite3_int64 id; - int ii = 1 - pCur->bPoint; - assert( ii==0 || ii==1 ); - assert( pCur->bPoint || pCur->nPoint ); - if( pCur->aNode[ii]==0 ){ - assert( pRC!=0 ); - id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; - *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); +SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Used to iterate through attached dbs */ + int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ + + assert( sqlite3_mutex_held(db->mutex) ); + assert( !pnLog || *pnLog==-1 ); + assert( !pnCkpt || *pnCkpt==-1 ); + + for(i=0; inDb && rc==SQLITE_OK; i++){ + if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); + pnLog = 0; + pnCkpt = 0; + if( rc==SQLITE_BUSY ){ + bBusy = 1; + rc = SQLITE_OK; + } + } } - return pCur->aNode[ii]; + + return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; } +#endif /* SQLITE_OMIT_WAL */ /* -** Push a new element onto the priority queue +** This function returns true if main-memory should be used instead of +** a temporary file for transient pager files and statement journals. +** The value returned depends on the value of db->temp_store (runtime +** parameter) and the compile time value of SQLITE_TEMP_STORE. The +** following table describes the relationship between these two values +** and this functions return value. +** +** SQLITE_TEMP_STORE db->temp_store Location of temporary database +** ----------------- -------------- ------------------------------ +** 0 any file (return 0) +** 1 1 file (return 0) +** 1 2 memory (return 1) +** 1 0 file (return 0) +** 2 1 file (return 0) +** 2 2 memory (return 1) +** 2 0 memory (return 1) +** 3 any memory (return 1) */ -static RtreeSearchPoint *rtreeEnqueue( - RtreeCursor *pCur, /* The cursor */ - RtreeDValue rScore, /* Score for the new search point */ - u8 iLevel /* Level for the new search point */ -){ - int i, j; - RtreeSearchPoint *pNew; - if( pCur->nPoint>=pCur->nPointAlloc ){ - int nNew = pCur->nPointAlloc*2 + 8; - pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); - if( pNew==0 ) return 0; - pCur->aPoint = pNew; - pCur->nPointAlloc = nNew; - } - i = pCur->nPoint++; - pNew = pCur->aPoint + i; - pNew->rScore = rScore; - pNew->iLevel = iLevel; - assert( iLevel<=RTREE_MAX_DEPTH ); - while( i>0 ){ - RtreeSearchPoint *pParent; - j = (i-1)/2; - pParent = pCur->aPoint + j; - if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; - rtreeSearchPointSwap(pCur, j, i); - i = j; - pNew = pParent; - } - return pNew; +SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ +#if SQLITE_TEMP_STORE==1 + return ( db->temp_store==2 ); +#endif +#if SQLITE_TEMP_STORE==2 + return ( db->temp_store!=1 ); +#endif +#if SQLITE_TEMP_STORE==3 + return 1; +#endif +#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 + return 0; +#endif } /* -** Allocate a new RtreeSearchPoint and return a pointer to it. Return -** NULL if malloc fails. +** Return UTF-8 encoded English language explanation of the most recent +** error. */ -static RtreeSearchPoint *rtreeSearchPointNew( - RtreeCursor *pCur, /* The cursor */ - RtreeDValue rScore, /* Score for the new search point */ - u8 iLevel /* Level for the new search point */ -){ - RtreeSearchPoint *pNew, *pFirst; - pFirst = rtreeSearchPointFirst(pCur); - pCur->anQueue[iLevel]++; - if( pFirst==0 - || pFirst->rScore>rScore - || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) - ){ - if( pCur->bPoint ){ - int ii; - pNew = rtreeEnqueue(pCur, rScore, iLevel); - if( pNew==0 ) return 0; - ii = (int)(pNew - pCur->aPoint) + 1; - if( iiaNode[ii]==0 ); - pCur->aNode[ii] = pCur->aNode[0]; - }else{ - nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); - } - pCur->aNode[0] = 0; - *pNew = pCur->sPoint; - } - pCur->sPoint.rScore = rScore; - pCur->sPoint.iLevel = iLevel; - pCur->bPoint = 1; - return &pCur->sPoint; +SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3 *db){ + const char *z; + if( !db ){ + return sqlite3ErrStr(SQLITE_NOMEM); + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return sqlite3ErrStr(SQLITE_MISUSE_BKPT); + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = sqlite3ErrStr(SQLITE_NOMEM); }else{ - return rtreeEnqueue(pCur, rScore, iLevel); + testcase( db->pErr==0 ); + z = (char*)sqlite3_value_text(db->pErr); + assert( !db->mallocFailed ); + if( z==0 ){ + z = sqlite3ErrStr(db->errCode); + } } + sqlite3_mutex_leave(db->mutex); + return z; } -#if 0 -/* Tracing routines for the RtreeSearchPoint queue */ -static void tracePoint(RtreeSearchPoint *p, int idx, RtreeCursor *pCur){ - if( idx<0 ){ printf(" s"); }else{ printf("%2d", idx); } - printf(" %d.%05lld.%02d %g %d", - p->iLevel, p->id, p->iCell, p->rScore, p->eWithin - ); - idx++; - if( idxaNode[idx]); - }else{ - printf("\n"); +#ifndef SQLITE_OMIT_UTF16 +/* +** Return UTF-16 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3 *db){ + static const u16 outOfMem[] = { + 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 + }; + static const u16 misuse[] = { + 'l', 'i', 'b', 'r', 'a', 'r', 'y', ' ', + 'r', 'o', 'u', 't', 'i', 'n', 'e', ' ', + 'c', 'a', 'l', 'l', 'e', 'd', ' ', + 'o', 'u', 't', ' ', + 'o', 'f', ' ', + 's', 'e', 'q', 'u', 'e', 'n', 'c', 'e', 0 + }; + + const void *z; + if( !db ){ + return (void *)outOfMem; } -} -static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ - int ii; - printf("=== %9s ", zPrefix); - if( pCur->bPoint ){ - tracePoint(&pCur->sPoint, -1, pCur); + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return (void *)misuse; } - for(ii=0; iinPoint; ii++){ - if( ii>0 || pCur->bPoint ) printf(" "); - tracePoint(&pCur->aPoint[ii], ii, pCur); + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = (void *)outOfMem; + }else{ + z = sqlite3_value_text16(db->pErr); + if( z==0 ){ + sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); + z = sqlite3_value_text16(db->pErr); + } + /* A malloc() may have failed within the call to sqlite3_value_text16() + ** above. If this is the case, then the db->mallocFailed flag needs to + ** be cleared before returning. Do this directly, instead of via + ** sqlite3ApiExit(), to avoid setting the database handle error message. + */ + db->mallocFailed = 0; } + sqlite3_mutex_leave(db->mutex); + return z; } -# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) -#else -# define RTREE_QUEUE_TRACE(A,B) /* no-op */ -#endif +#endif /* SQLITE_OMIT_UTF16 */ -/* Remove the search point with the lowest current score. +/* +** Return the most recent error code generated by an SQLite routine. If NULL is +** passed to this function, we assume a malloc() failed during sqlite3_open(). */ -static void rtreeSearchPointPop(RtreeCursor *p){ - int i, j, k, n; - i = 1 - p->bPoint; - assert( i==0 || i==1 ); - if( p->aNode[i] ){ - nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); - p->aNode[i] = 0; +SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; } - if( p->bPoint ){ - p->anQueue[p->sPoint.iLevel]--; - p->bPoint = 0; - }else if( p->nPoint ){ - p->anQueue[p->aPoint[0].iLevel]--; - n = --p->nPoint; - p->aPoint[0] = p->aPoint[n]; - if( naNode[1] = p->aNode[n+1]; - p->aNode[n+1] = 0; - } - i = 0; - while( (j = i*2+1)aPoint[k], &p->aPoint[j])<0 ){ - if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ - rtreeSearchPointSwap(p, i, k); - i = k; - }else{ - break; - } - }else{ - if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ - rtreeSearchPointSwap(p, i, j); - i = j; - }else{ - break; - } - } - } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; + } + return db->errCode & db->errMask; +} +SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; } + return db->errCode; } +/* +** Return a string that describes the kind of error specified in the +** argument. For now, this simply calls the internal sqlite3ErrStr() +** function. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int rc){ + return sqlite3ErrStr(rc); +} /* -** Continue the search on cursor pCur until the front of the queue -** contains an entry suitable for returning as a result-set row, -** or until the RtreeSearchPoint queue is empty, indicating that the -** query has completed. +** Create a new collating function for database "db". The name is zName +** and the encoding is enc. */ -static int rtreeStepToLeaf(RtreeCursor *pCur){ - RtreeSearchPoint *p; - Rtree *pRtree = RTREE_OF_CURSOR(pCur); - RtreeNode *pNode; - int eWithin; - int rc = SQLITE_OK; - int nCell; - int nConstraint = pCur->nConstraint; - int ii; - int eInt; - RtreeSearchPoint x; +static int createCollation( + sqlite3* db, + const char *zName, + u8 enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + CollSeq *pColl; + int enc2; + + assert( sqlite3_mutex_held(db->mutex) ); - eInt = pRtree->eCoordType==RTREE_COORD_INT32; - while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ - pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); - if( rc ) return rc; - nCell = NCELL(pNode); - assert( nCell<200 ); - while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); - eWithin = FULLY_WITHIN; - for(ii=0; iiaConstraint + ii; - if( pConstraint->op>=RTREE_MATCH ){ - rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, - &rScore, &eWithin); - if( rc ) return rc; - }else if( p->iLevel==1 ){ - rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); - }else{ - rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + */ + enc2 = enc; + testcase( enc2==SQLITE_UTF16 ); + testcase( enc2==SQLITE_UTF16_ALIGNED ); + if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ + enc2 = SQLITE_UTF16NATIVE; + } + if( enc2SQLITE_UTF16BE ){ + return SQLITE_MISUSE_BKPT; + } + + /* Check if this call is removing or replacing an existing collation + ** sequence. If so, and there are active VMs, return busy. If there + ** are no active VMs, invalidate any pre-compiled statements. + */ + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); + if( pColl && pColl->xCmp ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify collation sequence due to active statements"); + return SQLITE_BUSY; + } + sqlite3ExpirePreparedStatements(db); + + /* If collation sequence pColl was created directly by a call to + ** sqlite3_create_collation, and not generated by synthCollSeq(), + ** then any copies made by synthCollSeq() need to be invalidated. + ** Also, collation destructor - CollSeq.xDel() - function may need + ** to be called. + */ + if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); + int j; + for(j=0; j<3; j++){ + CollSeq *p = &aColl[j]; + if( p->enc==pColl->enc ){ + if( p->xDel ){ + p->xDel(p->pUser); + } + p->xCmp = 0; } - if( eWithin==NOT_WITHIN ) break; - } - p->iCell++; - if( eWithin==NOT_WITHIN ) continue; - x.iLevel = p->iLevel - 1; - if( x.iLevel ){ - x.id = readInt64(pCellData); - x.iCell = 0; - }else{ - x.id = p->id; - x.iCell = p->iCell - 1; - } - if( p->iCell>=nCell ){ - RTREE_QUEUE_TRACE(pCur, "POP-S:"); - rtreeSearchPointPop(pCur); } - if( rScoreeWithin = eWithin; - p->id = x.id; - p->iCell = x.iCell; - RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); - break; - } - if( p->iCell>=nCell ){ - RTREE_QUEUE_TRACE(pCur, "POP-Se:"); - rtreeSearchPointPop(pCur); } } - pCur->atEOF = p==0; + + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); + if( pColl==0 ) return SQLITE_NOMEM; + pColl->xCmp = xCompare; + pColl->pUser = pCtx; + pColl->xDel = xDel; + pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); + sqlite3Error(db, SQLITE_OK); return SQLITE_OK; } -/* -** Rtree virtual table module xNext method. -*/ -static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ - RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - int rc = SQLITE_OK; - /* Move to the next entry that matches the configured constraints. */ - RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); - rtreeSearchPointPop(pCsr); - rc = rtreeStepToLeaf(pCsr); - return rc; -} +/* +** This array defines hard upper bounds on limit values. The +** initializer must be kept in sync with the SQLITE_LIMIT_* +** #defines in sqlite3.h. +*/ +static const int aHardLimit[] = { + SQLITE_MAX_LENGTH, + SQLITE_MAX_SQL_LENGTH, + SQLITE_MAX_COLUMN, + SQLITE_MAX_EXPR_DEPTH, + SQLITE_MAX_COMPOUND_SELECT, + SQLITE_MAX_VDBE_OP, + SQLITE_MAX_FUNCTION_ARG, + SQLITE_MAX_ATTACHED, + SQLITE_MAX_LIKE_PATTERN_LENGTH, + SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ + SQLITE_MAX_TRIGGER_DEPTH, + SQLITE_MAX_WORKER_THREADS, +}; -/* -** Rtree virtual table module xRowid method. +/* +** Make sure the hard limits are set to reasonable values */ -static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ - RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); - int rc = SQLITE_OK; - RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); - if( rc==SQLITE_OK && p ){ - *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); - } - return rc; -} +#if SQLITE_MAX_LENGTH<100 +# error SQLITE_MAX_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH<100 +# error SQLITE_MAX_SQL_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH +# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH +#endif +#if SQLITE_MAX_COMPOUND_SELECT<2 +# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 +#endif +#if SQLITE_MAX_VDBE_OP<40 +# error SQLITE_MAX_VDBE_OP must be at least 40 +#endif +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 +#endif +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 +# error SQLITE_MAX_ATTACHED must be between 0 and 125 +#endif +#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 +# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 +#endif +#if SQLITE_MAX_COLUMN>32767 +# error SQLITE_MAX_COLUMN must not exceed 32767 +#endif +#if SQLITE_MAX_TRIGGER_DEPTH<1 +# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 +#endif +#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 +# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 +#endif -/* -** Rtree virtual table module xColumn method. + +/* +** Change the value of a limit. Report the old value. +** If an invalid limit index is supplied, report -1. +** Make no changes but still report the old value if the +** new limit is negative. +** +** A new lower limit does not shrink existing constructs. +** It merely prevents new constructs that exceed the limit +** from forming. */ -static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ - Rtree *pRtree = (Rtree *)cur->pVtab; - RtreeCursor *pCsr = (RtreeCursor *)cur; - RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); - RtreeCoord c; - int rc = SQLITE_OK; - RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); +SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ + int oldLimit; - if( rc ) return rc; - if( p==0 ) return SQLITE_OK; - if( i==0 ){ - sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); - }else{ - if( rc ) return rc; - nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); -#ifndef SQLITE_RTREE_INT_ONLY - if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ - sqlite3_result_double(ctx, c.f); - }else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } #endif - { - assert( pRtree->eCoordType==RTREE_COORD_INT32 ); - sqlite3_result_int(ctx, c.i); + + /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME + ** there is a hard upper bound set at compile-time by a C preprocessor + ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to + ** "_MAX_".) + */ + assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); + assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); + assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); + assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); + assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); + assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== + SQLITE_MAX_LIKE_PATTERN_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); + assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); + + + if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ + return -1; + } + oldLimit = db->aLimit[limitId]; + if( newLimit>=0 ){ /* IMP: R-52476-28732 */ + if( newLimit>aHardLimit[limitId] ){ + newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ } + db->aLimit[limitId] = newLimit; } - return SQLITE_OK; + return oldLimit; /* IMP: R-53341-35419 */ } -/* -** Use nodeAcquire() to obtain the leaf node containing the record with -** rowid iRowid. If successful, set *ppLeaf to point to the node and -** return SQLITE_OK. If there is no such record in the table, set -** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf -** to zero and return an SQLite error code. +/* +** This function is used to parse both URIs and non-URI filenames passed by the +** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database +** URIs specified as part of ATTACH statements. +** +** The first argument to this function is the name of the VFS to use (or +** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" +** query parameter. The second argument contains the URI (or non-URI filename) +** itself. When this function is called the *pFlags variable should contain +** the default flags to open the database handle with. The value stored in +** *pFlags may be updated before returning if the URI filename contains +** "cache=xxx" or "mode=xxx" query parameters. +** +** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to +** the VFS that should be used to open the database file. *pzFile is set to +** point to a buffer containing the name of the file to open. It is the +** responsibility of the caller to eventually call sqlite3_free() to release +** this buffer. +** +** If an error occurs, then an SQLite error code is returned and *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to eventually release +** this buffer by calling sqlite3_free(). */ -static int findLeafNode( - Rtree *pRtree, /* RTree to search */ - i64 iRowid, /* The rowid searching for */ - RtreeNode **ppLeaf, /* Write the node here */ - sqlite3_int64 *piNode /* Write the node-id here */ +SQLITE_PRIVATE int sqlite3ParseUri( + const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ + const char *zUri, /* Nul-terminated URI to parse */ + unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + char **pzFile, /* OUT: Filename component of URI */ + char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ ){ - int rc; - *ppLeaf = 0; - sqlite3_bind_int64(pRtree->pReadRowid, 1, iRowid); - if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ - i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); - if( piNode ) *piNode = iNode; - rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); - sqlite3_reset(pRtree->pReadRowid); + int rc = SQLITE_OK; + unsigned int flags = *pFlags; + const char *zVfs = zDefaultVfs; + char *zFile; + char c; + int nUri = sqlite3Strlen30(zUri); + + assert( *pzErrMsg==0 ); + + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ + ){ + char *zOpt; + int eState; /* Parser state when parsing URI */ + int iIn; /* Input character index */ + int iOut = 0; /* Output character index */ + u64 nByte = nUri+2; /* Bytes of space to allocate */ + + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + ** method that there may be extra parameters following the file-name. */ + flags |= SQLITE_OPEN_URI; + + for(iIn=0; iIn=0 && octet<256 ); + if( octet==0 ){ + /* This branch is taken when "%00" appears within the URI. In this + ** case we ignore all text in the remainder of the path, name or + ** value currently being parsed. So ignore the current character + ** and skip to the next "?", "=" or "&", as appropriate. */ + while( (c = zUri[iIn])!=0 && c!='#' + && (eState!=0 || c!='?') + && (eState!=1 || (c!='=' && c!='&')) + && (eState!=2 || c!='&') + ){ + iIn++; + } + continue; + } + c = octet; + }else if( eState==1 && (c=='&' || c=='=') ){ + if( zFile[iOut-1]==0 ){ + /* An empty option name. Ignore this option altogether. */ + while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; + continue; + } + if( c=='&' ){ + zFile[iOut++] = '\0'; + }else{ + eState = 2; + } + c = 0; + }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ + c = 0; + eState = 1; + } + zFile[iOut++] = c; + } + if( eState==1 ) zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + + /* Check if there were any options specified that should be interpreted + ** here. Options that are interpreted here include "vfs" and those that + ** correspond to flags that may be passed to the sqlite3_open_v2() + ** method. */ + zOpt = &zFile[sqlite3Strlen30(zFile)+1]; + while( zOpt[0] ){ + int nOpt = sqlite3Strlen30(zOpt); + char *zVal = &zOpt[nOpt+1]; + int nVal = sqlite3Strlen30(zVal); + + if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ + zVfs = zVal; + }else{ + struct OpenMode { + const char *z; + int mode; + } *aMode = 0; + char *zModeType = 0; + int mask = 0; + int limit = 0; + + if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ + static struct OpenMode aCacheMode[] = { + { "shared", SQLITE_OPEN_SHAREDCACHE }, + { "private", SQLITE_OPEN_PRIVATECACHE }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; + aMode = aCacheMode; + limit = mask; + zModeType = "cache"; + } + if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ + static struct OpenMode aOpenMode[] = { + { "ro", SQLITE_OPEN_READONLY }, + { "rw", SQLITE_OPEN_READWRITE }, + { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, + { "memory", SQLITE_OPEN_MEMORY }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; + aMode = aOpenMode; + limit = mask & flags; + zModeType = "access"; + } + + if( aMode ){ + int i; + int mode = 0; + for(i=0; aMode[i].z; i++){ + const char *z = aMode[i].z; + if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ + mode = aMode[i].mode; + break; + } + } + if( mode==0 ){ + *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); + rc = SQLITE_ERROR; + goto parse_uri_out; + } + if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ + *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", + zModeType, zVal); + rc = SQLITE_PERM; + goto parse_uri_out; + } + flags = (flags & ~mask) | mode; + } + } + + zOpt = &zVal[nVal+1]; + } + }else{ - rc = sqlite3_reset(pRtree->pReadRowid); + zFile = sqlite3_malloc64(nUri+2); + if( !zFile ) return SQLITE_NOMEM; + memcpy(zFile, zUri, nUri); + zFile[nUri] = '\0'; + zFile[nUri+1] = '\0'; + flags &= ~SQLITE_OPEN_URI; + } + + *ppVfs = sqlite3_vfs_find(zVfs); + if( *ppVfs==0 ){ + *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); + rc = SQLITE_ERROR; + } + parse_uri_out: + if( rc!=SQLITE_OK ){ + sqlite3_free(zFile); + zFile = 0; } + *pFlags = flags; + *pzFile = zFile; return rc; } + /* -** This function is called to configure the RtreeConstraint object passed -** as the second argument for a MATCH constraint. The value passed as the -** first argument to this function is the right-hand operand to the MATCH -** operator. +** This routine does the work of opening a database on behalf of +** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" +** is UTF-8 encoded. */ -static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ - RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ - sqlite3_rtree_query_info *pInfo; /* Callback information */ - int nBlob; /* Size of the geometry function blob */ - int nExpected; /* Expected size of the BLOB */ +static int openDatabase( + const char *zFilename, /* Database filename UTF-8 encoded */ + sqlite3 **ppDb, /* OUT: Returned database handle */ + unsigned int flags, /* Operational flags */ + const char *zVfs /* Name of the VFS to use */ +){ + sqlite3 *db; /* Store allocated handle here */ + int rc; /* Return code */ + int isThreadsafe; /* True for threadsafe connections */ + char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ + char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ - /* Check that value is actually a blob. */ - if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif - /* Check that the blob is roughly the right size. */ - nBlob = sqlite3_value_bytes(pValue); - if( nBlob<(int)sizeof(RtreeMatchArg) ){ - return SQLITE_ERROR; + /* Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ){ + return SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */ } - pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); - if( !pInfo ) return SQLITE_NOMEM; - memset(pInfo, 0, sizeof(*pInfo)); - pBlob = (RtreeMatchArg*)&pInfo[1]; - - memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); - nExpected = (int)(sizeof(RtreeMatchArg) + - pBlob->nParam*sizeof(sqlite3_value*) + - (pBlob->nParam-1)*sizeof(RtreeDValue)); - if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ - sqlite3_free(pInfo); - return SQLITE_ERROR; + if( sqlite3GlobalConfig.bCoreMutex==0 ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_NOMUTEX ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_FULLMUTEX ){ + isThreadsafe = 1; + }else{ + isThreadsafe = sqlite3GlobalConfig.bFullMutex; + } + if( flags & SQLITE_OPEN_PRIVATECACHE ){ + flags &= ~SQLITE_OPEN_SHAREDCACHE; + }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ + flags |= SQLITE_OPEN_SHAREDCACHE; } - pInfo->pContext = pBlob->cb.pContext; - pInfo->nParam = pBlob->nParam; - pInfo->aParam = pBlob->aParam; - pInfo->apSqlParam = pBlob->apSqlParam; - if( pBlob->cb.xGeom ){ - pCons->u.xGeom = pBlob->cb.xGeom; - }else{ - pCons->op = RTREE_QUERY; - pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; + /* Remove harmful bits from the flags parameter + ** + ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were + ** dealt with in the previous code block. Besides these, the only + ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, + ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, + ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask + ** off all other flags. + */ + flags &= ~( SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_MAIN_DB | + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_TRANSIENT_DB | + SQLITE_OPEN_MAIN_JOURNAL | + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_SUBJOURNAL | + SQLITE_OPEN_MASTER_JOURNAL | + SQLITE_OPEN_NOMUTEX | + SQLITE_OPEN_FULLMUTEX | + SQLITE_OPEN_WAL + ); + + /* Allocate the sqlite data structure */ + db = sqlite3MallocZero( sizeof(sqlite3) ); + if( db==0 ) goto opendb_out; + if( isThreadsafe ){ + db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( db->mutex==0 ){ + sqlite3_free(db); + db = 0; + goto opendb_out; + } } - pCons->pInfo = pInfo; - return SQLITE_OK; -} + sqlite3_mutex_enter(db->mutex); + db->errMask = 0xff; + db->nDb = 2; + db->magic = SQLITE_MAGIC_BUSY; + db->aDb = db->aDbStatic; -/* -** Rtree virtual table module xFilter method. -*/ -static int rtreeFilter( - sqlite3_vtab_cursor *pVtabCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; - RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - RtreeNode *pRoot = 0; - int ii; - int rc = SQLITE_OK; - int iCell = 0; + assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); + memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; + db->autoCommit = 1; + db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; + db->nextPagesize = 0; + db->nMaxSorterMmap = 0x7FFFFFFF; + db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill +#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX + | SQLITE_AutoIndex +#endif +#if SQLITE_DEFAULT_CKPTFULLFSYNC + | SQLITE_CkptFullFSync +#endif +#if SQLITE_DEFAULT_FILE_FORMAT<4 + | SQLITE_LegacyFileFmt +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + | SQLITE_LoadExtension +#endif +#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS + | SQLITE_RecTriggers +#endif +#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS + | SQLITE_ForeignKeys +#endif +#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) + | SQLITE_ReverseOrder +#endif + ; + sqlite3HashInit(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3HashInit(&db->aModule); +#endif - rtreeReference(pRtree); + /* Add the default collation sequence BINARY. BINARY works for both UTF-8 + ** and UTF-16, so add a version for each to avoid any unnecessary + ** conversions. The only error that can occur here is a malloc() failure. + ** + ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating + ** functions: + */ + createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); + if( db->mallocFailed ){ + goto opendb_out; + } + /* EVIDENCE-OF: R-08308-17224 The default collating function for all + ** strings is BINARY. + */ + db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); + assert( db->pDfltColl!=0 ); - /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ - freeCursorConstraints(pCsr); - sqlite3_free(pCsr->aPoint); - memset(pCsr, 0, sizeof(RtreeCursor)); - pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + /* Parse the filename/URI argument. */ + db->openFlags = flags; + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3_free(zErrMsg); + goto opendb_out; + } - pCsr->iStrategy = idxNum; - if( idxNum==1 ){ - /* Special case - lookup by rowid. */ - RtreeNode *pLeaf; /* Leaf on which the required cell resides */ - RtreeSearchPoint *p; /* Search point for the leaf */ - i64 iRowid = sqlite3_value_int64(argv[0]); - i64 iNode = 0; - rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); - if( rc==SQLITE_OK && pLeaf!=0 ){ - p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); - assert( p!=0 ); /* Always returns pCsr->sPoint */ - pCsr->aNode[0] = pLeaf; - p->id = iNode; - p->eWithin = PARTLY_WITHIN; - rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); - p->iCell = iCell; - RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); - }else{ - pCsr->atEOF = 1; - } - }else{ - /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array - ** with the configured constraints. - */ - rc = nodeAcquire(pRtree, 1, 0, &pRoot); - if( rc==SQLITE_OK && argc>0 ){ - pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); - pCsr->nConstraint = argc; - if( !pCsr->aConstraint ){ - rc = SQLITE_NOMEM; - }else{ - memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); - memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); - assert( (idxStr==0 && argc==0) - || (idxStr && (int)strlen(idxStr)==argc*2) ); - for(ii=0; iiaConstraint[ii]; - p->op = idxStr[ii*2]; - p->iCoord = idxStr[ii*2+1]-'0'; - if( p->op>=RTREE_MATCH ){ - /* A MATCH operator. The right-hand-side must be a blob that - ** can be cast into an RtreeMatchArg object. One created using - ** an sqlite3_rtree_geometry_callback() SQL user function. - */ - rc = deserializeGeometry(argv[ii], p); - if( rc!=SQLITE_OK ){ - break; - } - p->pInfo->nCoord = pRtree->nDim*2; - p->pInfo->anQueue = pCsr->anQueue; - p->pInfo->mxLevel = pRtree->iDepth + 1; - }else{ -#ifdef SQLITE_RTREE_INT_ONLY - p->u.rValue = sqlite3_value_int64(argv[ii]); -#else - p->u.rValue = sqlite3_value_double(argv[ii]); -#endif - } - } - } - } - if( rc==SQLITE_OK ){ - RtreeSearchPoint *pNew; - pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); - if( pNew==0 ) return SQLITE_NOMEM; - pNew->id = 1; - pNew->iCell = 0; - pNew->eWithin = PARTLY_WITHIN; - assert( pCsr->bPoint==1 ); - pCsr->aNode[0] = pRoot; - pRoot = 0; - RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); - rc = rtreeStepToLeaf(pCsr); + /* Open the backend database driver */ + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, + flags | SQLITE_OPEN_MAIN_DB); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM; } + sqlite3Error(db, rc); + goto opendb_out; } + sqlite3BtreeEnter(db->aDb[0].pBt); + db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db); + sqlite3BtreeLeave(db->aDb[0].pBt); + db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); - nodeRelease(pRtree, pRoot); - rtreeRelease(pRtree); - return rc; -} + /* The default safety_level for the main database is 'full'; for the temp + ** database it is 'NONE'. This matches the pager layer defaults. + */ + db->aDb[0].zName = "main"; + db->aDb[0].safety_level = 3; + db->aDb[1].zName = "temp"; + db->aDb[1].safety_level = 1; -/* -** Set the pIdxInfo->estimatedRows variable to nRow. Unless this -** extension is currently being used by a version of SQLite too old to -** support estimatedRows. In that case this function is a no-op. -*/ -static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ -#if SQLITE_VERSION_NUMBER>=3008002 - if( sqlite3_libversion_number()>=3008002 ){ - pIdxInfo->estimatedRows = nRow; + db->magic = SQLITE_MAGIC_OPEN; + if( db->mallocFailed ){ + goto opendb_out; } -#endif -} - -/* -** Rtree virtual table module xBestIndex method. There are three -** table scan strategies to choose from (in order from most to -** least desirable): -** -** idxNum idxStr Strategy -** ------------------------------------------------ -** 1 Unused Direct lookup by rowid. -** 2 See below R-tree query or full-table scan. -** ------------------------------------------------ -** -** If strategy 1 is used, then idxStr is not meaningful. If strategy -** 2 is used, idxStr is formatted to contain 2 bytes for each -** constraint used. The first two bytes of idxStr correspond to -** the constraint in sqlite3_index_info.aConstraintUsage[] with -** (argvIndex==1) etc. -** -** The first of each pair of bytes in idxStr identifies the constraint -** operator as follows: -** -** Operator Byte Value -** ---------------------- -** = 0x41 ('A') -** <= 0x42 ('B') -** < 0x43 ('C') -** >= 0x44 ('D') -** > 0x45 ('E') -** MATCH 0x46 ('F') -** ---------------------- -** -** The second of each pair of bytes identifies the coordinate column -** to which the constraint applies. The leftmost coordinate column -** is 'a', the second from the left 'b' etc. -*/ -static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ - Rtree *pRtree = (Rtree*)tab; - int rc = SQLITE_OK; - int ii; - int bMatch = 0; /* True if there exists a MATCH constraint */ - i64 nRow; /* Estimated rows returned by this scan */ - int iIdx = 0; - char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; - memset(zIdxStr, 0, sizeof(zIdxStr)); + /* Register all built-in functions, but do not attempt to read the + ** database schema yet. This is delayed until the first time the database + ** is accessed. + */ + sqlite3Error(db, SQLITE_OK); + sqlite3RegisterBuiltinFunctions(db); - /* Check if there exists a MATCH constraint - even an unusable one. If there - ** is, do not consider the lookup-by-rowid plan as using such a plan would - ** require the VDBE to evaluate the MATCH constraint, which is not currently - ** possible. */ - for(ii=0; iinConstraint; ii++){ - if( pIdxInfo->aConstraint[ii].op==SQLITE_INDEX_CONSTRAINT_MATCH ){ - bMatch = 1; + /* Load automatic extensions - extensions that have been registered + ** using the sqlite3_automatic_extension() API. + */ + rc = sqlite3_errcode(db); + if( rc==SQLITE_OK ){ + sqlite3AutoLoadExtensions(db); + rc = sqlite3_errcode(db); + if( rc!=SQLITE_OK ){ + goto opendb_out; } } - assert( pIdxInfo->idxStr==0 ); - for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ - struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; +#ifdef SQLITE_ENABLE_FTS1 + if( !db->mallocFailed ){ + extern int sqlite3Fts1Init(sqlite3*); + rc = sqlite3Fts1Init(db); + } +#endif - if( bMatch==0 && p->usable - && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ - ){ - /* We have an equality constraint on the rowid. Use strategy 1. */ - int jj; - for(jj=0; jjaConstraintUsage[jj].argvIndex = 0; - pIdxInfo->aConstraintUsage[jj].omit = 0; - } - pIdxInfo->idxNum = 1; - pIdxInfo->aConstraintUsage[ii].argvIndex = 1; - pIdxInfo->aConstraintUsage[jj].omit = 1; +#ifdef SQLITE_ENABLE_FTS2 + if( !db->mallocFailed && rc==SQLITE_OK ){ + extern int sqlite3Fts2Init(sqlite3*); + rc = sqlite3Fts2Init(db); + } +#endif - /* This strategy involves a two rowid lookups on an B-Tree structures - ** and then a linear search of an R-Tree node. This should be - ** considered almost as quick as a direct rowid lookup (for which - ** sqlite uses an internal cost of 0.0). It is expected to return - ** a single row. - */ - pIdxInfo->estimatedCost = 30.0; - setEstimatedRows(pIdxInfo, 1); - return SQLITE_OK; - } +#ifdef SQLITE_ENABLE_FTS3 + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts3Init(db); + } +#endif - if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ - u8 op; - switch( p->op ){ - case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; - case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; - case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; - case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; - case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; - default: - assert( p->op==SQLITE_INDEX_CONSTRAINT_MATCH ); - op = RTREE_MATCH; - break; - } - zIdxStr[iIdx++] = op; - zIdxStr[iIdx++] = p->iColumn - 1 + '0'; - pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); - pIdxInfo->aConstraintUsage[ii].omit = 1; - } +#ifdef SQLITE_ENABLE_ICU + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3IcuInit(db); } +#endif - pIdxInfo->idxNum = 2; - pIdxInfo->needToFreeIdxStr = 1; - if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ - return SQLITE_NOMEM; +#ifdef SQLITE_ENABLE_RTREE + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3RtreeInit(db); } +#endif - nRow = pRtree->nRowEst >> (iIdx/2); - pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; - setEstimatedRows(pIdxInfo, nRow); +#ifdef SQLITE_ENABLE_DBSTAT_VTAB + if( !db->mallocFailed && rc==SQLITE_OK){ + int sqlite3_dbstat_register(sqlite3*); + rc = sqlite3_dbstat_register(db); + } +#endif - return rc; -} + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking + ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking + ** mode. Doing nothing at all also makes NORMAL the default. + */ +#ifdef SQLITE_DEFAULT_LOCKING_MODE + db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; + sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), + SQLITE_DEFAULT_LOCKING_MODE); +#endif -/* -** Return the N-dimensional volumn of the cell stored in *p. -*/ -static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ - RtreeDValue area = (RtreeDValue)1; - int ii; - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); + if( rc ) sqlite3Error(db, rc); + + /* Enable the lookaside-malloc subsystem */ + setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside); + + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); + +opendb_out: + sqlite3_free(zOpen); + if( db ){ + assert( db->mutex!=0 || isThreadsafe==0 + || sqlite3GlobalConfig.bFullMutex==0 ); + sqlite3_mutex_leave(db->mutex); } - return area; + rc = sqlite3_errcode(db); + assert( db!=0 || rc==SQLITE_NOMEM ); + if( rc==SQLITE_NOMEM ){ + sqlite3_close(db); + db = 0; + }else if( rc!=SQLITE_OK ){ + db->magic = SQLITE_MAGIC_SICK; + } + *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif + return sqlite3ApiExit(0, rc); } /* -** Return the margin length of cell p. The margin length is the sum -** of the objects size in each dimension. +** Open a new database handle. */ -static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ - RtreeDValue margin = (RtreeDValue)0; - int ii; - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); - } - return margin; +SQLITE_API int SQLITE_STDCALL sqlite3_open( + const char *zFilename, + sqlite3 **ppDb +){ + return openDatabase(zFilename, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); +} +SQLITE_API int SQLITE_STDCALL sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +){ + return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); } +#ifndef SQLITE_OMIT_UTF16 /* -** Store the union of cells p1 and p2 in p1. +** Open a new database handle. */ -static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ - int ii; - if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f); - p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f); +SQLITE_API int SQLITE_STDCALL sqlite3_open16( + const void *zFilename, + sqlite3 **ppDb +){ + char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ + sqlite3_value *pVal; + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + if( zFilename==0 ) zFilename = "\000\000"; + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zFilename8 ){ + rc = openDatabase(zFilename8, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); + assert( *ppDb || rc==SQLITE_NOMEM ); + if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ + SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE; } }else{ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i); - p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i); - } + rc = SQLITE_NOMEM; } -} + sqlite3ValueFree(pVal); -/* -** Return true if the area covered by p2 is a subset of the area covered -** by p1. False otherwise. -*/ -static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ - int ii; - int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - RtreeCoord *a1 = &p1->aCoord[ii]; - RtreeCoord *a2 = &p2->aCoord[ii]; - if( (!isInt && (a2[0].fa1[1].f)) - || ( isInt && (a2[0].ia1[1].i)) - ){ - return 0; - } - } - return 1; + return sqlite3ApiExit(0, rc); } +#endif /* SQLITE_OMIT_UTF16 */ /* -** Return the amount cell p would grow by if it were unioned with pCell. +** Register a new collation sequence with the database handle db. */ -static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ - RtreeDValue area; - RtreeCell cell; - memcpy(&cell, p, sizeof(RtreeCell)); - area = cellArea(pRtree, &cell); - cellUnion(pRtree, &cell, pCell); - return (cellArea(pRtree, &cell)-area); -} - -static RtreeDValue cellOverlap( - Rtree *pRtree, - RtreeCell *p, - RtreeCell *aCell, - int nCell +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) ){ - int ii; - RtreeDValue overlap = RTREE_ZERO; - for(ii=0; iinDim*2); jj+=2){ - RtreeDValue x1, x2; - x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); - x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); - if( x2iDepth-iHeight); ii++){ - int iCell; - sqlite3_int64 iBest = 0; - - RtreeDValue fMinGrowth = RTREE_ZERO; - RtreeDValue fMinArea = RTREE_ZERO; - - int nCell = NCELL(pNode); - RtreeCell cell; - RtreeNode *pChild; - RtreeCell *aCell = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} - /* Select the child node which will be enlarged the least if pCell - ** is inserted into it. Resolve ties by choosing the entry with - ** the smallest area. - */ - for(iCell=0; iCellmutex); + assert( !db->mallocFailed ); + zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); + if( zName8 ){ + rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); + sqlite3DbFree(db, zName8); } - - *ppLeaf = pNode; + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); return rc; } +#endif /* SQLITE_OMIT_UTF16 */ /* -** A cell with the same content as pCell has just been inserted into -** the node pNode. This function updates the bounding box cells in -** all ancestor elements. +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. */ -static int AdjustTree( - Rtree *pRtree, /* Rtree table */ - RtreeNode *pNode, /* Adjust ancestry of this node. */ - RtreeCell *pCell /* This cell was just inserted */ +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) ){ - RtreeNode *p = pNode; - while( p->pParent ){ - RtreeNode *pParent = p->pParent; - RtreeCell cell; - int iCell; - - if( nodeParentIndex(pRtree, p, &iCell) ){ - return SQLITE_CORRUPT_VTAB; - } - - nodeGetCell(pRtree, pParent, iCell, &cell); - if( !cellContains(pRtree, &cell, pCell) ){ - cellUnion(pRtree, &cell, pCell); - nodeOverwriteCell(pRtree, pParent, &cell, iCell); - } - - p = pParent; - } +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = xCollNeeded; + db->xCollNeeded16 = 0; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } +#ifndef SQLITE_OMIT_UTF16 /* -** Write mapping (iRowid->iNode) to the _rowid table. +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. */ -static int rowidWrite(Rtree *pRtree, sqlite3_int64 iRowid, sqlite3_int64 iNode){ - sqlite3_bind_int64(pRtree->pWriteRowid, 1, iRowid); - sqlite3_bind_int64(pRtree->pWriteRowid, 2, iNode); - sqlite3_step(pRtree->pWriteRowid); - return sqlite3_reset(pRtree->pWriteRowid); +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed16( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = 0; + db->xCollNeeded16 = xCollNeeded16; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; } +#endif /* SQLITE_OMIT_UTF16 */ +#ifndef SQLITE_OMIT_DEPRECATED /* -** Write mapping (iNode->iPar) to the _parent table. +** This function is now an anachronism. It used to be used to recover from a +** malloc() failure, but SQLite now does this automatically. */ -static int parentWrite(Rtree *pRtree, sqlite3_int64 iNode, sqlite3_int64 iPar){ - sqlite3_bind_int64(pRtree->pWriteParent, 1, iNode); - sqlite3_bind_int64(pRtree->pWriteParent, 2, iPar); - sqlite3_step(pRtree->pWriteParent); - return sqlite3_reset(pRtree->pWriteParent); +SQLITE_API int SQLITE_STDCALL sqlite3_global_recover(void){ + return SQLITE_OK; } +#endif -static int rtreeInsertCell(Rtree *, RtreeNode *, RtreeCell *, int); - +/* +** Test to see whether or not the database connection is in autocommit +** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on +** by default. Autocommit is disabled by a BEGIN statement and reenabled +** by the next COMMIT or ROLLBACK. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->autoCommit; +} /* -** Arguments aIdx, aDistance and aSpare all point to arrays of size -** nIdx. The aIdx array contains the set of integers from 0 to -** (nIdx-1) in no particular order. This function sorts the values -** in aIdx according to the indexed values in aDistance. For -** example, assuming the inputs: -** -** aIdx = { 0, 1, 2, 3 } -** aDistance = { 5.0, 2.0, 7.0, 6.0 } -** -** this function sets the aIdx array to contain: +** The following routines are substitutes for constants SQLITE_CORRUPT, +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error +** constants. They serve two purposes: ** -** aIdx = { 0, 1, 2, 3 } +** 1. Serve as a convenient place to set a breakpoint in a debugger +** to detect when version error conditions occurs. ** -** The aSpare array is used as temporary working space by the -** sorting algorithm. +** 2. Invoke sqlite3_log() to provide the source code location where +** a low-level error is first detected. */ -static void SortByDistance( - int *aIdx, - int nIdx, - RtreeDValue *aDistance, - int *aSpare -){ - if( nIdx>1 ){ - int iLeft = 0; - int iRight = 0; - - int nLeft = nIdx/2; - int nRight = nIdx-nLeft; - int *aLeft = aIdx; - int *aRight = &aIdx[nLeft]; - - SortByDistance(aLeft, nLeft, aDistance, aSpare); - SortByDistance(aRight, nRight, aDistance, aSpare); - - memcpy(aSpare, aLeft, sizeof(int)*nLeft); - aLeft = aSpare; +SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CORRUPT, + "database corruption at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_CORRUPT; +} +SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_MISUSE, + "misuse at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_MISUSE; +} +SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CANTOPEN, + "cannot open file at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_CANTOPEN; +} - while( iLeft1 ){ + int rc; + char *zErrMsg = 0; + Table *pTab = 0; + Column *pCol = 0; + int iCol = 0; + char const *zDataType = 0; + char const *zCollSeq = 0; + int notnull = 0; + int primarykey = 0; + int autoinc = 0; - int iLeft = 0; - int iRight = 0; - int nLeft = nIdx/2; - int nRight = nIdx-nLeft; - int *aLeft = aIdx; - int *aRight = &aIdx[nLeft]; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zTableName==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif - SortByDimension(pRtree, aLeft, nLeft, iDim, aCell, aSpare); - SortByDimension(pRtree, aRight, nRight, iDim, aCell, aSpare); + /* Ensure the database schema has been loaded */ + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + rc = sqlite3Init(db, &zErrMsg); + if( SQLITE_OK!=rc ){ + goto error_out; + } - memcpy(aSpare, aLeft, sizeof(int)*nLeft); - aLeft = aSpare; - while( iLeftpSelect ){ + pTab = 0; + goto error_out; + } + + /* Find the column for which info is requested */ + if( zColumnName==0 ){ + /* Query for existance of table only */ + }else{ + for(iCol=0; iColnCol; iCol++){ + pCol = &pTab->aCol[iCol]; + if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ + break; + } + } + if( iCol==pTab->nCol ){ + if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ + iCol = pTab->iPKey; + pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; }else{ - aIdx[iLeft+iRight] = aRight[iRight]; - iRight++; + pTab = 0; + goto error_out; } } + } + + /* The following block stores the meta information that will be returned + ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey + ** and autoinc. At this point there are two possibilities: + ** + ** 1. The specified column name was rowid", "oid" or "_rowid_" + ** and there is no explicitly declared IPK column. + ** + ** 2. The table is not a view and the column name identified an + ** explicitly declared column. Copy meta information from *pCol. + */ + if( pCol ){ + zDataType = pCol->zType; + zCollSeq = pCol->zColl; + notnull = pCol->notNull!=0; + primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; + autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; + }else{ + zDataType = "INTEGER"; + primarykey = 1; + } + if( !zCollSeq ){ + zCollSeq = "BINARY"; + } -#if 0 - /* Check that the sort worked */ - { - int jj; - for(jj=1; jjmutex); + return rc; } /* -** Implementation of the R*-tree variant of SplitNode from Beckman[1990]. +** Sleep for a little while. Return the amount of time slept. */ -static int splitNodeStartree( - Rtree *pRtree, - RtreeCell *aCell, - int nCell, - RtreeNode *pLeft, - RtreeNode *pRight, - RtreeCell *pBboxLeft, - RtreeCell *pBboxRight -){ - int **aaSorted; - int *aSpare; - int ii; +SQLITE_API int SQLITE_STDCALL sqlite3_sleep(int ms){ + sqlite3_vfs *pVfs; + int rc; + pVfs = sqlite3_vfs_find(0); + if( pVfs==0 ) return 0; - int iBestDim = 0; - int iBestSplit = 0; - RtreeDValue fBestMargin = RTREE_ZERO; + /* This function works in milliseconds, but the underlying OsSleep() + ** API uses microseconds. Hence the 1000's. + */ + rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + return rc; +} - int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); +/* +** Enable or disable the extended result codes. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->errMask = onoff ? 0xffffffff : 0xff; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} - aaSorted = (int **)sqlite3_malloc(nByte); - if( !aaSorted ){ - return SQLITE_NOMEM; - } +/* +** Invoke the xFileControl method on a particular database. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ + int rc = SQLITE_ERROR; + Btree *pBtree; - aSpare = &((int *)&aaSorted[pRtree->nDim])[pRtree->nDim*nCell]; - memset(aaSorted, 0, nByte); - for(ii=0; iinDim; ii++){ - int jj; - aaSorted[ii] = &((int *)&aaSorted[pRtree->nDim])[ii*nCell]; - for(jj=0; jjmutex); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + assert( fd!=0 ); + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( fd->pMethods ){ + rc = sqlite3OsFileControl(fd, op, pArg); + }else{ + rc = SQLITE_NOTFOUND; } - SortByDimension(pRtree, aaSorted[ii], nCell, ii, aCell, aSpare); + sqlite3BtreeLeave(pBtree); } + sqlite3_mutex_leave(db->mutex); + return rc; +} - for(ii=0; iinDim; ii++){ - RtreeDValue margin = RTREE_ZERO; - RtreeDValue fBestOverlap = RTREE_ZERO; - RtreeDValue fBestArea = RTREE_ZERO; - int iBestLeft = 0; - int nLeft; +/* +** Interface to the testing logic. +*/ +SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ + int rc = 0; +#ifndef SQLITE_OMIT_BUILTIN_TEST + va_list ap; + va_start(ap, op); + switch( op ){ - for( - nLeft=RTREE_MINCELLS(pRtree); - nLeft<=(nCell-RTREE_MINCELLS(pRtree)); - nLeft++ - ){ - RtreeCell left; - RtreeCell right; - int kk; - RtreeDValue overlap; - RtreeDValue area; + /* + ** Save the current state of the PRNG. + */ + case SQLITE_TESTCTRL_PRNG_SAVE: { + sqlite3PrngSaveState(); + break; + } - memcpy(&left, &aCell[aaSorted[ii][0]], sizeof(RtreeCell)); - memcpy(&right, &aCell[aaSorted[ii][nCell-1]], sizeof(RtreeCell)); - for(kk=1; kk<(nCell-1); kk++){ - if( kk0 ){ - RtreeNode *pChild = nodeHashLookup(pRtree, iRowid); - if( pChild ){ - nodeRelease(pRtree, pChild->pParent); - nodeReference(pNode); - pChild->pParent = pNode; + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) + ** + ** Set the PENDING byte to the value in the argument, if X>0. + ** Make no changes if X==0. Return the value of the pending byte + ** as it existing before this routine was called. + ** + ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in + ** an incompatible database file format. Changing the PENDING byte + ** while any database connection is open results in undefined and + ** deleterious behavior. + */ + case SQLITE_TESTCTRL_PENDING_BYTE: { + rc = PENDING_BYTE; +#ifndef SQLITE_OMIT_WSD + { + unsigned int newVal = va_arg(ap, unsigned int); + if( newVal ) sqlite3PendingByte = newVal; + } +#endif + break; } - } - return xSetMapping(pRtree, iRowid, pNode->iNode); -} -static int SplitNode( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell, - int iHeight -){ - int i; - int newCellIsRight = 0; + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) + ** + ** This action provides a run-time test to see whether or not + ** assert() was enabled at compile-time. If X is true and assert() + ** is enabled, then the return value is true. If X is true and + ** assert() is disabled, then the return value is zero. If X is + ** false and assert() is enabled, then the assertion fires and the + ** process aborts. If X is false and assert() is disabled, then the + ** return value is zero. + */ + case SQLITE_TESTCTRL_ASSERT: { + volatile int x = 0; + assert( (x = va_arg(ap,int))!=0 ); + rc = x; + break; + } - int rc = SQLITE_OK; - int nCell = NCELL(pNode); - RtreeCell *aCell; - int *aiUsed; - RtreeNode *pLeft = 0; - RtreeNode *pRight = 0; + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) + ** + ** This action provides a run-time test to see how the ALWAYS and + ** NEVER macros were defined at compile-time. + ** + ** The return value is ALWAYS(X). + ** + ** The recommended test is X==2. If the return value is 2, that means + ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the + ** default setting. If the return value is 1, then ALWAYS() is either + ** hard-coded to true or else it asserts if its argument is false. + ** The first behavior (hard-coded to true) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second + ** behavior (assert if the argument to ALWAYS() is false) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. + ** + ** The run-time test procedure might look something like this: + ** + ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ + ** // ALWAYS() and NEVER() are no-op pass-through macros + ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ + ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. + ** }else{ + ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. + ** } + */ + case SQLITE_TESTCTRL_ALWAYS: { + int x = va_arg(ap,int); + rc = ALWAYS(x); + break; + } - RtreeCell leftbbox; - RtreeCell rightbbox; + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); + ** + ** The integer returned reveals the byte-order of the computer on which + ** SQLite is running: + ** + ** 1 big-endian, determined at run-time + ** 10 little-endian, determined at run-time + ** 432101 big-endian, determined at compile-time + ** 123410 little-endian, determined at compile-time + */ + case SQLITE_TESTCTRL_BYTEORDER: { + rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; + break; + } - /* Allocate an array and populate it with a copy of pCell and - ** all cells from node pLeft. Then zero the original node. - */ - aCell = sqlite3_malloc((sizeof(RtreeCell)+sizeof(int))*(nCell+1)); - if( !aCell ){ - rc = SQLITE_NOMEM; - goto splitnode_out; - } - aiUsed = (int *)&aCell[nCell+1]; - memset(aiUsed, 0, sizeof(int)*(nCell+1)); - for(i=0; imutex); + sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); + sqlite3_mutex_leave(db->mutex); + break; + } - if( pNode->iNode==1 ){ - pRight = nodeNew(pRtree, pNode); - pLeft = nodeNew(pRtree, pNode); - pRtree->iDepth++; - pNode->isDirty = 1; - writeInt16(pNode->zData, pRtree->iDepth); - }else{ - pLeft = pNode; - pRight = nodeNew(pRtree, pLeft->pParent); - nodeReference(pLeft); - } + /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) + ** + ** Enable or disable various optimizations for testing purposes. The + ** argument N is a bitmask of optimizations to be disabled. For normal + ** operation N should be 0. The idea is that a test program (like the + ** SQL Logic Test or SLT test module) can run the same SQL multiple times + ** with various optimizations disabled to verify that the same answer + ** is obtained in every case. + */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); + break; + } - if( !pLeft || !pRight ){ - rc = SQLITE_NOMEM; - goto splitnode_out; - } +#ifdef SQLITE_N_KEYWORD + /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) + ** + ** If zWord is a keyword recognized by the parser, then return the + ** number of keywords. Or if zWord is not a keyword, return 0. + ** + ** This test feature is only available in the amalgamation since + ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite + ** is built using separate source files. + */ + case SQLITE_TESTCTRL_ISKEYWORD: { + const char *zWord = va_arg(ap, const char*); + int n = sqlite3Strlen30(zWord); + rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; + break; + } +#endif - memset(pLeft->zData, 0, pRtree->iNodeSize); - memset(pRight->zData, 0, pRtree->iNodeSize); + /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); + ** + ** Pass pFree into sqlite3ScratchFree(). + ** If sz>0 then allocate a scratch buffer into pNew. + */ + case SQLITE_TESTCTRL_SCRATCHMALLOC: { + void *pFree, **ppNew; + int sz; + sz = va_arg(ap, int); + ppNew = va_arg(ap, void**); + pFree = va_arg(ap, void*); + if( sz ) *ppNew = sqlite3ScratchMalloc(sz); + sqlite3ScratchFree(pFree); + break; + } - rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, - &leftbbox, &rightbbox); - if( rc!=SQLITE_OK ){ - goto splitnode_out; - } + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + ** + ** If parameter onoff is non-zero, configure the wrappers so that all + ** subsequent calls to localtime() and variants fail. If onoff is zero, + ** undo this setting. + */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: { + sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + break; + } - /* Ensure both child nodes have node numbers assigned to them by calling - ** nodeWrite(). Node pRight always needs a node number, as it was created - ** by nodeNew() above. But node pLeft sometimes already has a node number. - ** In this case avoid the all to nodeWrite(). - */ - if( SQLITE_OK!=(rc = nodeWrite(pRtree, pRight)) - || (0==pLeft->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pLeft))) - ){ - goto splitnode_out; - } + /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); + ** + ** Set or clear a flag that indicates that the database file is always well- + ** formed and never corrupt. This flag is clear by default, indicating that + ** database files might have arbitrary corruption. Setting the flag during + ** testing causes certain assert() statements in the code to be activated + ** that demonstrat invariants on well-formed database files. + */ + case SQLITE_TESTCTRL_NEVER_CORRUPT: { + sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); + break; + } - rightbbox.iRowid = pRight->iNode; - leftbbox.iRowid = pLeft->iNode; - if( pNode->iNode==1 ){ - rc = rtreeInsertCell(pRtree, pLeft->pParent, &leftbbox, iHeight+1); - if( rc!=SQLITE_OK ){ - goto splitnode_out; - } - }else{ - RtreeNode *pParent = pLeft->pParent; - int iCell; - rc = nodeParentIndex(pRtree, pLeft, &iCell); - if( rc==SQLITE_OK ){ - nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); - rc = AdjustTree(pRtree, pParent, &leftbbox); - } - if( rc!=SQLITE_OK ){ - goto splitnode_out; + /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); + ** + ** Set the VDBE coverage callback function to xCallback with context + ** pointer ptr. + */ + case SQLITE_TESTCTRL_VDBE_COVERAGE: { +#ifdef SQLITE_VDBE_COVERAGE + typedef void (*branch_callback)(void*,int,u8,u8); + sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); + sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); +#endif + break; } - } - if( (rc = rtreeInsertCell(pRtree, pRight->pParent, &rightbbox, iHeight+1)) ){ - goto splitnode_out; - } - for(i=0; iiRowid ){ - newCellIsRight = 1; + /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ + case SQLITE_TESTCTRL_SORTER_MMAP: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->nMaxSorterMmap = va_arg(ap, int); + break; } - if( rc!=SQLITE_OK ){ - goto splitnode_out; + + /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); + ** + ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if + ** not. + */ + case SQLITE_TESTCTRL_ISINIT: { + if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; + break; } - } - if( pNode->iNode==1 ){ - for(i=0; i0 then reset the schema for all databases, causing + ** the schema to be reparsed the next time it is needed. This has the + ** effect of erasing all imposter tables. + */ + case SQLITE_TESTCTRL_IMPOSTER: { + sqlite3 *db = va_arg(ap, sqlite3*); + sqlite3_mutex_enter(db->mutex); + db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); + db->init.busy = db->init.imposterTable = va_arg(ap,int); + db->init.newTnum = va_arg(ap,int); + if( db->init.busy==0 && db->init.newTnum>0 ){ + sqlite3ResetAllSchemasOfConnection(db); } + sqlite3_mutex_leave(db->mutex); + break; } - }else if( newCellIsRight==0 ){ - rc = updateMapping(pRtree, pCell->iRowid, pLeft, iHeight); - } - - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pRight); - pRight = 0; - } - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pLeft); - pLeft = 0; } - -splitnode_out: - nodeRelease(pRtree, pRight); - nodeRelease(pRtree, pLeft); - sqlite3_free(aCell); + va_end(ap); +#endif /* SQLITE_OMIT_BUILTIN_TEST */ return rc; } /* -** If node pLeaf is not the root of the r-tree and its pParent pointer is -** still NULL, load all ancestor nodes of pLeaf into memory and populate -** the pLeaf->pParent chain all the way up to the root node. +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. ** -** This operation is required when a row is deleted (or updated - an update -** is implemented as a delete followed by an insert). SQLite provides the -** rowid of the row to delete, which can be used to find the leaf on which -** the entry resides (argument pLeaf). Once the leaf is located, this -** function is called to determine its ancestry. +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. */ -static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ - int rc = SQLITE_OK; - RtreeNode *pChild = pLeaf; - while( rc==SQLITE_OK && pChild->iNode!=1 && pChild->pParent==0 ){ - int rc2 = SQLITE_OK; /* sqlite3_reset() return code */ - sqlite3_bind_int64(pRtree->pReadParent, 1, pChild->iNode); - rc = sqlite3_step(pRtree->pReadParent); - if( rc==SQLITE_ROW ){ - RtreeNode *pTest; /* Used to test for reference loops */ - i64 iNode; /* Node number of parent node */ - - /* Before setting pChild->pParent, test that we are not creating a - ** loop of references (as we would if, say, pChild==pParent). We don't - ** want to do this as it leads to a memory leak when trying to delete - ** the referenced counted node structures. - */ - iNode = sqlite3_column_int64(pRtree->pReadParent, 0); - for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent); - if( !pTest ){ - rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent); - } - } - rc = sqlite3_reset(pRtree->pReadParent); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT_VTAB; - pChild = pChild->pParent; +SQLITE_API const char *SQLITE_STDCALL sqlite3_uri_parameter(const char *zFilename, const char *zParam){ + if( zFilename==0 || zParam==0 ) return 0; + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + int x = strcmp(zFilename, zParam); + zFilename += sqlite3Strlen30(zFilename) + 1; + if( x==0 ) return zFilename; + zFilename += sqlite3Strlen30(zFilename) + 1; } - return rc; + return 0; } -static int deleteCell(Rtree *, RtreeNode *, int, int); - -static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ - int rc; - int rc2; - RtreeNode *pParent = 0; - int iCell; - - assert( pNode->nRef==1 ); - - /* Remove the entry in the parent cell. */ - rc = nodeParentIndex(pRtree, pNode, &iCell); - if( rc==SQLITE_OK ){ - pParent = pNode->pParent; - pNode->pParent = 0; - rc = deleteCell(pRtree, pParent, iCell, iHeight+1); - } - rc2 = nodeRelease(pRtree, pParent); - if( rc==SQLITE_OK ){ - rc = rc2; - } - if( rc!=SQLITE_OK ){ - return rc; - } - - /* Remove the xxx_node entry. */ - sqlite3_bind_int64(pRtree->pDeleteNode, 1, pNode->iNode); - sqlite3_step(pRtree->pDeleteNode); - if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteNode)) ){ - return rc; - } +/* +** Return a boolean value for a query parameter. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + bDflt = bDflt!=0; + return z ? sqlite3GetBoolean(z, bDflt) : bDflt; +} - /* Remove the xxx_parent entry. */ - sqlite3_bind_int64(pRtree->pDeleteParent, 1, pNode->iNode); - sqlite3_step(pRtree->pDeleteParent); - if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteParent)) ){ - return rc; +/* +** Return a 64-bit integer value for a query parameter. +*/ +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_uri_int64( + const char *zFilename, /* Filename as passed to xOpen */ + const char *zParam, /* URI parameter sought */ + sqlite3_int64 bDflt /* return if parameter is missing */ +){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + sqlite3_int64 v; + if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ + bDflt = v; } - - /* Remove the node from the in-memory hash table and link it into - ** the Rtree.pDeleted list. Its contents will be re-inserted later on. - */ - nodeHashDelete(pRtree, pNode); - pNode->iNode = iHeight; - pNode->pNext = pRtree->pDeleted; - pNode->nRef++; - pRtree->pDeleted = pNode; - - return SQLITE_OK; + return bDflt; } -static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ - RtreeNode *pParent = pNode->pParent; - int rc = SQLITE_OK; - if( pParent ){ - int ii; - int nCell = NCELL(pNode); - RtreeCell box; /* Bounding box for pNode */ - nodeGetCell(pRtree, pNode, 0, &box); - for(ii=1; iiiNode; - rc = nodeParentIndex(pRtree, pNode, &ii); - if( rc==SQLITE_OK ){ - nodeOverwriteCell(pRtree, pParent, &box, ii); - rc = fixBoundingBox(pRtree, pParent); +/* +** Return the Btree pointer identified by zDbName. Return NULL if not found. +*/ +SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ + int i; + for(i=0; inDb; i++){ + if( db->aDb[i].pBt + && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) + ){ + return db->aDb[i].pBt; } } - return rc; + return 0; } /* -** Delete the cell at index iCell of node pNode. After removing the -** cell, adjust the r-tree data structure if required. +** Return the filename of the database associated with a database +** connection. */ -static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ - RtreeNode *pParent; - int rc; - - if( SQLITE_OK!=(rc = fixLeafParent(pRtree, pNode)) ){ - return rc; +SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const char *zDbName){ + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeGetFilename(pBt) : 0; +} - /* Remove the cell from the node. This call just moves bytes around - ** the in-memory node image, so it cannot fail. - */ - nodeDeleteCell(pRtree, pNode, iCell); - - /* If the node is not the tree root and now has less than the minimum - ** number of cells, remove it from the tree. Otherwise, update the - ** cell in the parent node so that it tightly contains the updated - ** node. - */ - pParent = pNode->pParent; - assert( pParent || pNode->iNode==1 ); - if( pParent ){ - if( NCELL(pNode)nDim; iDim++){ - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); - } - } - for(iDim=0; iDimnDim; iDim++){ - aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); - } +/* +** Head of a linked list of all sqlite3 objects created by this process +** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection +** is not NULL. This variable may only accessed while the STATIC_MASTER +** mutex is held. +*/ +static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; - for(ii=0; iinDim; iDim++){ - RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - - DCOORD(aCell[ii].aCoord[iDim*2])); - aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); - } - } +#ifndef NDEBUG +/* +** This function is a complex assert() that verifies the following +** properties of the blocked connections list: +** +** 1) Each entry in the list has a non-NULL value for either +** pUnlockConnection or pBlockingConnection, or both. +** +** 2) All entries in the list that share a common value for +** xUnlockNotify are grouped together. +** +** 3) If the argument db is not NULL, then none of the entries in the +** blocked connections list have pUnlockConnection or pBlockingConnection +** set to db. This is used when closing connection db. +*/ +static void checkListProperties(sqlite3 *db){ + sqlite3 *p; + for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ + int seen = 0; + sqlite3 *p2; - SortByDistance(aOrder, nCell, aDistance, aSpare); - nodeZero(pRtree, pNode); + /* Verify property (1) */ + assert( p->pUnlockConnection || p->pBlockingConnection ); - for(ii=0; rc==SQLITE_OK && ii<(nCell-(RTREE_MINCELLS(pRtree)+1)); ii++){ - RtreeCell *p = &aCell[aOrder[ii]]; - nodeInsertCell(pRtree, pNode, p); - if( p->iRowid==pCell->iRowid ){ - if( iHeight==0 ){ - rc = rowidWrite(pRtree, p->iRowid, pNode->iNode); - }else{ - rc = parentWrite(pRtree, p->iRowid, pNode->iNode); - } + /* Verify property (2) */ + for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ + if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; + assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); + assert( db==0 || p->pUnlockConnection!=db ); + assert( db==0 || p->pBlockingConnection!=db ); } } - if( rc==SQLITE_OK ){ - rc = fixBoundingBox(pRtree, pNode); - } - for(; rc==SQLITE_OK && iiiNode currently contains - ** the height of the sub-tree headed by the cell. - */ - RtreeNode *pInsert; - RtreeCell *p = &aCell[aOrder[ii]]; - rc = ChooseLeaf(pRtree, p, iHeight, &pInsert); - if( rc==SQLITE_OK ){ - int rc2; - rc = rtreeInsertCell(pRtree, pInsert, p, iHeight); - rc2 = nodeRelease(pRtree, pInsert); - if( rc==SQLITE_OK ){ - rc = rc2; - } +} +#else +# define checkListProperties(x) +#endif + +/* +** Remove connection db from the blocked connections list. If connection +** db is not currently a part of the list, this function is a no-op. +*/ +static void removeFromBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ + if( *pp==db ){ + *pp = (*pp)->pNextBlocked; + break; } } +} - sqlite3_free(aCell); - return rc; +/* +** Add connection db to the blocked connections list. It is assumed +** that it is not already a part of the list. +*/ +static void addToBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for( + pp=&sqlite3BlockedList; + *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; + pp=&(*pp)->pNextBlocked + ); + db->pNextBlocked = *pp; + *pp = db; } /* -** Insert cell pCell into node pNode. Node pNode is the head of a -** subtree iHeight high (leaf nodes have iHeight==0). +** Obtain the STATIC_MASTER mutex. */ -static int rtreeInsertCell( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell, - int iHeight -){ - int rc = SQLITE_OK; - if( iHeight>0 ){ - RtreeNode *pChild = nodeHashLookup(pRtree, pCell->iRowid); - if( pChild ){ - nodeRelease(pRtree, pChild->pParent); - nodeReference(pNode); - pChild->pParent = pNode; - } - } - if( nodeInsertCell(pRtree, pNode, pCell) ){ - if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ - rc = SplitNode(pRtree, pNode, pCell, iHeight); - }else{ - pRtree->iReinsertHeight = iHeight; - rc = Reinsert(pRtree, pNode, pCell, iHeight); - } - }else{ - rc = AdjustTree(pRtree, pNode, pCell); - if( rc==SQLITE_OK ){ - if( iHeight==0 ){ - rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); - }else{ - rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); - } - } - } - return rc; +static void enterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + checkListProperties(0); +} + +/* +** Release the STATIC_MASTER mutex. +*/ +static void leaveMutex(void){ + assertMutexHeld(); + checkListProperties(0); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } -static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ - int ii; +/* +** Register an unlock-notify callback. +** +** This is called after connection "db" has attempted some operation +** but has received an SQLITE_LOCKED error because another connection +** (call it pOther) in the same process was busy using the same shared +** cache. pOther is found by looking at db->pBlockingConnection. +** +** If there is no blocking connection, the callback is invoked immediately, +** before this routine returns. +** +** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate +** a deadlock. +** +** Otherwise, make arrangements to invoke xNotify when pOther drops +** its locks. +** +** Each call to this routine overrides any prior callbacks registered +** on the same "db". If xNotify==0 then any prior callbacks are immediately +** cancelled. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( + sqlite3 *db, + void (*xNotify)(void **, int), + void *pArg +){ int rc = SQLITE_OK; - int nCell = NCELL(pNode); - for(ii=0; rc==SQLITE_OK && iimutex); + enterMutex(); - /* Find a node to store this cell in. pNode->iNode currently contains - ** the height of the sub-tree headed by the cell. + if( xNotify==0 ){ + removeFromBlockedList(db); + db->pBlockingConnection = 0; + db->pUnlockConnection = 0; + db->xUnlockNotify = 0; + db->pUnlockArg = 0; + }else if( 0==db->pBlockingConnection ){ + /* The blocking transaction has been concluded. Or there never was a + ** blocking transaction. In either case, invoke the notify callback + ** immediately. */ - rc = ChooseLeaf(pRtree, &cell, (int)pNode->iNode, &pInsert); - if( rc==SQLITE_OK ){ - int rc2; - rc = rtreeInsertCell(pRtree, pInsert, &cell, (int)pNode->iNode); - rc2 = nodeRelease(pRtree, pInsert); - if( rc==SQLITE_OK ){ - rc = rc2; - } + xNotify(&pArg, 1); + }else{ + sqlite3 *p; + + for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} + if( p ){ + rc = SQLITE_LOCKED; /* Deadlock detected. */ + }else{ + db->pUnlockConnection = db->pBlockingConnection; + db->xUnlockNotify = xNotify; + db->pUnlockArg = pArg; + removeFromBlockedList(db); + addToBlockedList(db); } } + + leaveMutex(); + assert( !db->mallocFailed ); + sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); + sqlite3_mutex_leave(db->mutex); return rc; } /* -** Select a currently unused rowid for a new r-tree record. +** This function is called while stepping or preparing a statement +** associated with connection db. The operation will return SQLITE_LOCKED +** to the user because it requires a lock that will not be available +** until connection pBlocker concludes its current transaction. */ -static int newRowid(Rtree *pRtree, i64 *piRowid){ - int rc; - sqlite3_bind_null(pRtree->pWriteRowid, 1); - sqlite3_bind_null(pRtree->pWriteRowid, 2); - sqlite3_step(pRtree->pWriteRowid); - rc = sqlite3_reset(pRtree->pWriteRowid); - *piRowid = sqlite3_last_insert_rowid(pRtree->db); - return rc; +SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ + enterMutex(); + if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ + addToBlockedList(db); + } + db->pBlockingConnection = pBlocker; + leaveMutex(); } /* -** Remove the entry with rowid=iDelete from the r-tree structure. +** This function is called when +** the transaction opened by database db has just finished. Locks held +** by database connection db have been released. +** +** This function loops through each entry in the blocked connections +** list and does the following: +** +** 1) If the sqlite3.pBlockingConnection member of a list entry is +** set to db, then set pBlockingConnection=0. +** +** 2) If the sqlite3.pUnlockConnection member of a list entry is +** set to db, then invoke the configured unlock-notify callback and +** set pUnlockConnection=0. +** +** 3) If the two steps above mean that pBlockingConnection==0 and +** pUnlockConnection==0, remove the entry from the blocked connections +** list. */ -static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ - int rc; /* Return code */ - RtreeNode *pLeaf = 0; /* Leaf node containing record iDelete */ - int iCell; /* Index of iDelete cell in pLeaf */ - RtreeNode *pRoot; /* Root node of rtree structure */ - +SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ + void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ + int nArg = 0; /* Number of entries in aArg[] */ + sqlite3 **pp; /* Iterator variable */ + void **aArg; /* Arguments to the unlock callback */ + void **aDyn = 0; /* Dynamically allocated space for aArg[] */ + void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ - /* Obtain a reference to the root node to initialize Rtree.iDepth */ - rc = nodeAcquire(pRtree, 1, 0, &pRoot); + aArg = aStatic; + enterMutex(); /* Enter STATIC_MASTER mutex */ - /* Obtain a reference to the leaf node that contains the entry - ** about to be deleted. - */ - if( rc==SQLITE_OK ){ - rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); - } + /* This loop runs once for each entry in the blocked-connections list. */ + for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ + sqlite3 *p = *pp; - /* Delete the cell in question from the leaf node. */ - if( rc==SQLITE_OK ){ - int rc2; - rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); - if( rc==SQLITE_OK ){ - rc = deleteCell(pRtree, pLeaf, iCell, 0); - } - rc2 = nodeRelease(pRtree, pLeaf); - if( rc==SQLITE_OK ){ - rc = rc2; + /* Step 1. */ + if( p->pBlockingConnection==db ){ + p->pBlockingConnection = 0; } - } - /* Delete the corresponding entry in the _rowid table. */ - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); - sqlite3_step(pRtree->pDeleteRowid); - rc = sqlite3_reset(pRtree->pDeleteRowid); - } + /* Step 2. */ + if( p->pUnlockConnection==db ){ + assert( p->xUnlockNotify ); + if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ + xUnlockNotify(aArg, nArg); + nArg = 0; + } - /* Check if the root node now has exactly one child. If so, remove - ** it, schedule the contents of the child for reinsertion and - ** reduce the tree height by one. - ** - ** This is equivalent to copying the contents of the child into - ** the root node (the operation that Gutman's paper says to perform - ** in this scenario). - */ - if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ - int rc2; - RtreeNode *pChild; - i64 iChild = nodeGetRowid(pRtree, pRoot, 0); - rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); - if( rc==SQLITE_OK ){ - rc = removeNode(pRtree, pChild, pRtree->iDepth-1); - } - rc2 = nodeRelease(pRtree, pChild); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK ){ - pRtree->iDepth--; - writeInt16(pRoot->zData, pRtree->iDepth); - pRoot->isDirty = 1; + sqlite3BeginBenignMalloc(); + assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); + assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); + if( (!aDyn && nArg==(int)ArraySize(aStatic)) + || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) + ){ + /* The aArg[] array needs to grow. */ + void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); + if( pNew ){ + memcpy(pNew, aArg, nArg*sizeof(void *)); + sqlite3_free(aDyn); + aDyn = aArg = pNew; + }else{ + /* This occurs when the array of context pointers that need to + ** be passed to the unlock-notify callback is larger than the + ** aStatic[] array allocated on the stack and the attempt to + ** allocate a larger array from the heap has failed. + ** + ** This is a difficult situation to handle. Returning an error + ** code to the caller is insufficient, as even if an error code + ** is returned the transaction on connection db will still be + ** closed and the unlock-notify callbacks on blocked connections + ** will go unissued. This might cause the application to wait + ** indefinitely for an unlock-notify callback that will never + ** arrive. + ** + ** Instead, invoke the unlock-notify callback with the context + ** array already accumulated. We can then clear the array and + ** begin accumulating any further context pointers without + ** requiring any dynamic allocation. This is sub-optimal because + ** it means that instead of one callback with a large array of + ** context pointers the application will receive two or more + ** callbacks with smaller arrays of context pointers, which will + ** reduce the applications ability to prioritize multiple + ** connections. But it is the best that can be done under the + ** circumstances. + */ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + } + sqlite3EndBenignMalloc(); + + aArg[nArg++] = p->pUnlockArg; + xUnlockNotify = p->xUnlockNotify; + p->pUnlockConnection = 0; + p->xUnlockNotify = 0; + p->pUnlockArg = 0; } - } - /* Re-insert the contents of any underfull nodes removed from the tree. */ - for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ - if( rc==SQLITE_OK ){ - rc = reinsertNodeContent(pRtree, pLeaf); + /* Step 3. */ + if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ + /* Remove connection p from the blocked connections list. */ + *pp = p->pNextBlocked; + p->pNextBlocked = 0; + }else{ + pp = &p->pNextBlocked; } - pRtree->pDeleted = pLeaf->pNext; - sqlite3_free(pLeaf); } - /* Release the reference to the root node. */ - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pRoot); - }else{ - nodeRelease(pRtree, pRoot); + if( nArg!=0 ){ + xUnlockNotify(aArg, nArg); } + sqlite3_free(aDyn); + leaveMutex(); /* Leave STATIC_MASTER mutex */ +} - return rc; +/* +** This is called when the database connection passed as an argument is +** being closed. The connection is removed from the blocked list. +*/ +SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ + sqlite3ConnectionUnlocked(db); + enterMutex(); + removeFromBlockedList(db); + checkListProperties(db); + leaveMutex(); } +#endif +/************** End of notify.c **********************************************/ +/************** Begin file fts3.c ********************************************/ /* -** Rounding constants for float->double conversion. +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. */ -#define RNDTOWARDS (1.0 - 1.0/8388608.0) /* Round towards zero */ -#define RNDAWAY (1.0 + 1.0/8388608.0) /* Round away from zero */ -#if !defined(SQLITE_RTREE_INT_ONLY) /* -** Convert an sqlite3_value into an RtreeValue (presumably a float) -** while taking care to round toward negative or positive, respectively. +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ + +/* The full-text index is stored in a series of b+tree (-like) +** structures called segments which map terms to doclists. The +** structures are like b+trees in layout, but are constructed from the +** bottom up in optimal fashion and are not updatable. Since trees +** are built from the bottom up, things will be described from the +** bottom up. +** +** +**** Varints **** +** The basic unit of encoding is a variable-length integer called a +** varint. We encode variable-length integers in little-endian order +** using seven bits * per byte as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** and so on. +** +** This is similar in concept to how sqlite encodes "varints" but +** the encoding is not the same. SQLite varints are big-endian +** are are limited to 9 bytes in length whereas FTS3 varints are +** little-endian and can be up to 10 bytes in length (in theory). +** +** Example encodings: +** +** 1: 0x01 +** 127: 0x7f +** 128: 0x81 0x00 +** +** +**** Document lists **** +** A doclist (document list) holds a docid-sorted list of hits for a +** given term. Doclists hold docids and associated token positions. +** A docid is the unique integer identifier for a single document. +** A position is the index of a word within the document. The first +** word of the document has a position of 0. +** +** FTS3 used to optionally store character offsets using a compile-time +** option. But that functionality is no longer supported. +** +** A doclist is stored like this: +** +** array { +** varint docid; (delta from previous doclist) +** array { (position list for column 0) +** varint position; (2 more than the delta from previous position) +** } +** array { +** varint POS_COLUMN; (marks start of position list for new column) +** varint column; (index of new column) +** array { +** varint position; (2 more than the delta from previous position) +** } +** } +** varint POS_END; (marks end of positions for this document. +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. A "position" is an index of a token in the token stream +** generated by the tokenizer. Note that POS_END and POS_COLUMN occur +** in the same logical place as the position element, and act as sentinals +** ending a position list array. POS_END is 0. POS_COLUMN is 1. +** The positions numbers are not stored literally but rather as two more +** than the difference from the prior position, or the just the position plus +** 2 for the first position. Example: +** +** label: A B C D E F G H I J K +** value: 123 5 9 1 1 14 35 0 234 72 0 +** +** The 123 value is the first docid. For column zero in this document +** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 +** at D signals the start of a new column; the 1 at E indicates that the +** new column is column number 1. There are two positions at 12 and 45 +** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. +** +** A "position-list" is the list of positions for multiple columns for +** a single docid. A "column-list" is the set of positions for a single +** column. Hence, a position-list consists of one or more column-lists, +** a document record consists of a docid followed by a position-list and +** a doclist consists of one or more document records. +** +** A bare doclist omits the position information, becoming an +** array of varint-encoded docids. +** +**** Segment leaf nodes **** +** Segment leaf nodes store terms and doclists, ordered by term. Leaf +** nodes are written using LeafWriter, and read using LeafReader (to +** iterate through a single leaf node's data) and LeavesReader (to +** iterate through a segment's entire leaf layer). Leaf nodes have +** the format: +** +** varint iHeight; (height from leaf level, always 0) +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of prefix shared with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix];(unshared suffix of next term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. +** +** Leaf nodes are broken into blocks which are stored contiguously in +** the %_segments table in sorted order. This means that when the end +** of a node is reached, the next term is in the node with the next +** greater node id. +** +** New data is spilled to a new leaf node when the current node +** exceeds LEAF_MAX bytes (default 2048). New data which itself is +** larger than STANDALONE_MIN (default 1024) is placed in a standalone +** node (a leaf node with a single term and doclist). The goal of +** these settings is to pack together groups of small doclists while +** making it efficient to directly access large doclists. The +** assumption is that large doclists represent terms which are more +** likely to be query targets. +** +** TODO(shess) It may be useful for blocking decisions to be more +** dynamic. For instance, it may make more sense to have a 2.5k leaf +** node rather than splitting into 2k and .5k nodes. My intuition is +** that this might extend through 2x or 4x the pagesize. +** +** +**** Segment interior nodes **** +** Segment interior nodes store blockids for subtree nodes and terms +** to describe what data is stored by the each subtree. Interior +** nodes are written using InteriorWriter, and read using +** InteriorReader. InteriorWriters are created as needed when +** SegmentWriter creates new leaf nodes, or when an interior node +** itself grows too big and must be split. The format of interior +** nodes: +** +** varint iHeight; (height from leaf level, always >0) +** varint iBlockid; (block id of node's leftmost subtree) +** optional { +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of shared prefix with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix]; (unshared suffix of next term) +** } +** } +** +** Here, optional { X } means an optional element, while array { X } +** means zero or more occurrences of X, adjacent in memory. +** +** An interior node encodes n terms separating n+1 subtrees. The +** subtree blocks are contiguous, so only the first subtree's blockid +** is encoded. The subtree at iBlockid will contain all terms less +** than the first term encoded (or all terms if no term is encoded). +** Otherwise, for terms greater than or equal to pTerm[i] but less +** than pTerm[i+1], the subtree for that term will be rooted at +** iBlockid+i. Interior nodes only store enough term data to +** distinguish adjacent children (if the rightmost term of the left +** child is "something", and the leftmost term of the right child is +** "wicked", only "w" is stored). +** +** New data is spilled to a new interior node at the same height when +** the current node exceeds INTERIOR_MAX bytes (default 2048). +** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing +** interior nodes and making the tree too skinny. The interior nodes +** at a given height are naturally tracked by interior nodes at +** height+1, and so on. +** +** +**** Segment directory **** +** The segment directory in table %_segdir stores meta-information for +** merging and deleting segments, and also the root node of the +** segment's tree. +** +** The root node is the top node of the segment's tree after encoding +** the entire segment, restricted to ROOT_MAX bytes (default 1024). +** This could be either a leaf node or an interior node. If the top +** node requires more than ROOT_MAX bytes, it is flushed to %_segments +** and a new root interior node is generated (which should always fit +** within ROOT_MAX because it only needs space for 2 varints, the +** height and the blockid of the previous root). +** +** The meta-information in the segment directory is: +** level - segment level (see below) +** idx - index within level +** - (level,idx uniquely identify a segment) +** start_block - first leaf node +** leaves_end_block - last leaf node +** end_block - last block (including interior nodes) +** root - contents of root node +** +** If the root node is a leaf node, then start_block, +** leaves_end_block, and end_block are all 0. +** +** +**** Segment merging **** +** To amortize update costs, segments are grouped into levels and +** merged in batches. Each increase in level represents exponentially +** more documents. +** +** New documents (actually, document updates) are tokenized and +** written individually (using LeafWriter) to a level 0 segment, with +** incrementing idx. When idx reaches MERGE_COUNT (default 16), all +** level 0 segments are merged into a single level 1 segment. Level 1 +** is populated like level 0, and eventually MERGE_COUNT level 1 +** segments are merged to a single level 2 segment (representing +** MERGE_COUNT^2 updates), and so on. +** +** A segment merge traverses all segments at a given level in +** parallel, performing a straightforward sorted merge. Since segment +** leaf nodes are written in to the %_segments table in order, this +** merge traverses the underlying sqlite disk structures efficiently. +** After the merge, all segment blocks from the merged level are +** deleted. +** +** MERGE_COUNT controls how often we merge segments. 16 seems to be +** somewhat of a sweet spot for insertion performance. 32 and 64 show +** very similar performance numbers to 16 on insertion, though they're +** a tiny bit slower (perhaps due to more overhead in merge-time +** sorting). 8 is about 20% slower than 16, 4 about 50% slower than +** 16, 2 about 66% slower than 16. +** +** At query time, high MERGE_COUNT increases the number of segments +** which need to be scanned and merged. For instance, with 100k docs +** inserted: +** +** MERGE_COUNT segments +** 16 25 +** 8 12 +** 4 10 +** 2 6 +** +** This appears to have only a moderate impact on queries for very +** frequent terms (which are somewhat dominated by segment merge +** costs), and infrequent and non-existent terms still seem to be fast +** even with many segments. +** +** TODO(shess) That said, it would be nice to have a better query-side +** argument for MERGE_COUNT of 16. Also, it is possible/likely that +** optimizations to things like doclist merging will swing the sweet +** spot around. +** +** +** +**** Handling of deletions and updates **** +** Since we're using a segmented structure, with no docid-oriented +** index into the term index, we clearly cannot simply update the term +** index when a document is deleted or updated. For deletions, we +** write an empty doclist (varint(docid) varint(POS_END)), for updates +** we simply write the new doclist. Segment merges overwrite older +** data for a particular docid with newer data, so deletes or updates +** will eventually overtake the earlier data and knock it out. The +** query logic likewise merges doclists so that newer data knocks out +** older data. */ -static RtreeValue rtreeValueDown(sqlite3_value *v){ - double d = sqlite3_value_double(v); - float f = (float)d; - if( f>d ){ - f = (float)(d*(d<0 ? RNDAWAY : RNDTOWARDS)); - } - return f; -} -static RtreeValue rtreeValueUp(sqlite3_value *v){ - double d = sqlite3_value_double(v); - float f = (float)d; - if( fbase.zErrMsg) to an appropriate value and returns -** SQLITE_CONSTRAINT. +** 2009 Nov 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** Parameter iCol is the index of the leftmost column involved in the -** constraint failure. If it is 0, then the constraint that failed is -** the unique constraint on the id column. Otherwise, it is the rtree -** (c1<=c2) constraint on columns iCol and iCol+1 that has failed. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** ** -** If an OOM occurs, SQLITE_NOMEM is returned instead of SQLITE_CONSTRAINT. */ -static int rtreeConstraintError(Rtree *pRtree, int iCol){ - sqlite3_stmt *pStmt = 0; - char *zSql; - int rc; +#ifndef _FTSINT_H +#define _FTSINT_H - assert( iCol==0 || iCol%2 ); - zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", pRtree->zDb, pRtree->zName); - if( zSql ){ - rc = sqlite3_prepare_v2(pRtree->db, zSql, -1, &pStmt, 0); - }else{ - rc = SQLITE_NOMEM; - } - sqlite3_free(zSql); +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif - if( rc==SQLITE_OK ){ - if( iCol==0 ){ - const char *zCol = sqlite3_column_name(pStmt, 0); - pRtree->base.zErrMsg = sqlite3_mprintf( - "UNIQUE constraint failed: %s.%s", pRtree->zName, zCol - ); - }else{ - const char *zCol1 = sqlite3_column_name(pStmt, iCol); - const char *zCol2 = sqlite3_column_name(pStmt, iCol+1); - pRtree->base.zErrMsg = sqlite3_mprintf( - "rtree constraint failed: %s.(%s<=%s)", pRtree->zName, zCol1, zCol2 - ); - } - } +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif - sqlite3_finalize(pStmt); - return (rc==SQLITE_OK ? SQLITE_CONSTRAINT : rc); -} +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +SQLITE_EXTENSION_INIT3 +#endif +/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ +/************** Begin file fts3_tokenizer.h **********************************/ +/* +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. +*/ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ /* -** The xUpdate method for rtree module virtual tables. +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. */ -static int rtreeUpdate( - sqlite3_vtab *pVtab, - int nData, - sqlite3_value **azData, - sqlite_int64 *pRowid -){ - Rtree *pRtree = (Rtree *)pVtab; - int rc = SQLITE_OK; - RtreeCell cell; /* New cell to insert if nData>1 */ - int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; - rtreeReference(pRtree); - assert(nData>=1); +struct sqlite3_tokenizer_module { - cell.iRowid = 0; /* Used only to suppress a compiler warning */ + /* + ** Structure version. Should always be set to 0 or 1. + */ + int iVersion; - /* Constraint handling. A write operation on an r-tree table may return - ** SQLITE_CONSTRAINT for two reasons: + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: ** - ** 1. A duplicate rowid value, or - ** 2. The supplied data violates the "x2>=x1" constraint. + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) ** - ** In the first case, if the conflict-handling mode is REPLACE, then - ** the conflicting row can be removed before proceeding. In the second - ** case, SQLITE_CONSTRAINT must be returned regardless of the - ** conflict-handling mode specified by the user. + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialized by + ** this callback. The caller will do so. */ - if( nData>1 ){ - int ii; - - /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. - ** - ** NB: nData can only be less than nDim*2+3 if the rtree is mis-declared - ** with "column" that are interpreted as table constraints. - ** Example: CREATE VIRTUAL TABLE bad USING rtree(x,y,CHECK(y>5)); - ** This problem was discovered after years of use, so we silently ignore - ** these kinds of misdeclared tables to avoid breaking any legacy. - */ - assert( nData<=(pRtree->nDim*2 + 3) ); - -#ifndef SQLITE_RTREE_INT_ONLY - if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ - for(ii=0; iicell.aCoord[ii+1].f ){ - rc = rtreeConstraintError(pRtree, ii+1); - goto constraint; - } - } - }else -#endif - { - for(ii=0; iicell.aCoord[ii+1].i ){ - rc = rtreeConstraintError(pRtree, ii+1); - goto constraint; - } - } - } - - /* If a rowid value was supplied, check if it is already present in - ** the table. If so, the constraint has failed. */ - if( sqlite3_value_type(azData[2])!=SQLITE_NULL ){ - cell.iRowid = sqlite3_value_int64(azData[2]); - if( sqlite3_value_type(azData[0])==SQLITE_NULL - || sqlite3_value_int64(azData[0])!=cell.iRowid - ){ - int steprc; - sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); - steprc = sqlite3_step(pRtree->pReadRowid); - rc = sqlite3_reset(pRtree->pReadRowid); - if( SQLITE_ROW==steprc ){ - if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ - rc = rtreeDeleteRowid(pRtree, cell.iRowid); - }else{ - rc = rtreeConstraintError(pRtree, 0); - goto constraint; - } - } - } - bHaveRowid = 1; - } - } + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); - /* If azData[0] is not an SQL NULL value, it is the rowid of a - ** record to delete from the r-tree table. The following block does - ** just that. + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). */ - if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ - rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(azData[0])); - } + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); - /* If the azData[] array contains more than one element, elements - ** (azData[2]..azData[argc-1]) contain a new record to insert into - ** the r-tree structure. + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). */ - if( rc==SQLITE_OK && nData>1 ){ - /* Insert the new record into the r-tree */ - RtreeNode *pLeaf = 0; - - /* Figure out the rowid of the new row. */ - if( bHaveRowid==0 ){ - rc = newRowid(pRtree, &cell.iRowid); - } - *pRowid = cell.iRowid; - - if( rc==SQLITE_OK ){ - rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); - } - if( rc==SQLITE_OK ){ - int rc2; - pRtree->iReinsertHeight = -1; - rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); - rc2 = nodeRelease(pRtree, pLeaf); - if( rc==SQLITE_OK ){ - rc = rc2; - } - } - } - -constraint: - rtreeRelease(pRtree); - return rc; -} - -/* -** The xRename method for rtree module virtual tables. -*/ -static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ - Rtree *pRtree = (Rtree *)pVtab; - int rc = SQLITE_NOMEM; - char *zSql = sqlite3_mprintf( - "ALTER TABLE %Q.'%q_node' RENAME TO \"%w_node\";" - "ALTER TABLE %Q.'%q_parent' RENAME TO \"%w_parent\";" - "ALTER TABLE %Q.'%q_rowid' RENAME TO \"%w_rowid\";" - , pRtree->zDb, pRtree->zName, zNewName - , pRtree->zDb, pRtree->zName, zNewName - , pRtree->zDb, pRtree->zName, zNewName + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ ); - if( zSql ){ - rc = sqlite3_exec(pRtree->db, zSql, 0, 0, 0); - sqlite3_free(zSql); - } - return rc; -} -/* -** This function populates the pRtree->nRowEst variable with an estimate -** of the number of rows in the virtual table. If possible, this is based -** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. -*/ -static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ - const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; - char *zSql; - sqlite3_stmt *p; - int rc; - i64 nRow = 0; + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). + */ + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); - rc = sqlite3_table_column_metadata( - db, pRtree->zDb, "sqlite_stat1",0,0,0,0,0,0 + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ ); - if( rc!=SQLITE_OK ){ - pRtree->nRowEst = RTREE_DEFAULT_ROWEST; - return rc==SQLITE_ERROR ? SQLITE_OK : rc; - } - zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); - if( rc==SQLITE_OK ){ - if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); - rc = sqlite3_finalize(p); - }else if( rc!=SQLITE_NOMEM ){ - rc = SQLITE_OK; - } - - if( rc==SQLITE_OK ){ - if( nRow==0 ){ - pRtree->nRowEst = RTREE_DEFAULT_ROWEST; - }else{ - pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); - } - } - sqlite3_free(zSql); - } - return rc; -} + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ -static sqlite3_module rtreeModule = { - 0, /* iVersion */ - rtreeCreate, /* xCreate - create a table */ - rtreeConnect, /* xConnect - connect to an existing table */ - rtreeBestIndex, /* xBestIndex - Determine search strategy */ - rtreeDisconnect, /* xDisconnect - Disconnect from a table */ - rtreeDestroy, /* xDestroy - Drop a table */ - rtreeOpen, /* xOpen - open a cursor */ - rtreeClose, /* xClose - close a cursor */ - rtreeFilter, /* xFilter - configure scan constraints */ - rtreeNext, /* xNext - advance a cursor */ - rtreeEof, /* xEof */ - rtreeColumn, /* xColumn - read data */ - rtreeRowid, /* xRowid - read data */ - rtreeUpdate, /* xUpdate - write data */ - 0, /* xBegin - begin transaction */ - 0, /* xSync - sync transaction */ - 0, /* xCommit - commit transaction */ - 0, /* xRollback - rollback transaction */ - 0, /* xFindFunction - function overloading */ - rtreeRename, /* xRename - rename the table */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0 /* xRollbackTo */ + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); }; -static int rtreeSqlInit( - Rtree *pRtree, - sqlite3 *db, - const char *zDb, - const char *zPrefix, - int isCreate -){ - int rc = SQLITE_OK; - - #define N_STATEMENT 9 - static const char *azSql[N_STATEMENT] = { - /* Read and write the xxx_node table */ - "SELECT data FROM '%q'.'%q_node' WHERE nodeno = :1", - "INSERT OR REPLACE INTO '%q'.'%q_node' VALUES(:1, :2)", - "DELETE FROM '%q'.'%q_node' WHERE nodeno = :1", +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; - /* Read and write the xxx_rowid table */ - "SELECT nodeno FROM '%q'.'%q_rowid' WHERE rowid = :1", - "INSERT OR REPLACE INTO '%q'.'%q_rowid' VALUES(:1, :2)", - "DELETE FROM '%q'.'%q_rowid' WHERE rowid = :1", +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; - /* Read and write the xxx_parent table */ - "SELECT parentnode FROM '%q'.'%q_parent' WHERE nodeno = :1", - "INSERT OR REPLACE INTO '%q'.'%q_parent' VALUES(:1, :2)", - "DELETE FROM '%q'.'%q_parent' WHERE nodeno = :1" - }; - sqlite3_stmt **appStmt[N_STATEMENT]; - int i; +int fts3_global_term_cnt(int iTerm, int iCol); +int fts3_term_cnt(int iTerm, int iCol); - pRtree->db = db; - if( isCreate ){ - char *zCreate = sqlite3_mprintf( -"CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);" -"CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);" -"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY," - " parentnode INTEGER);" -"INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))", - zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize - ); - if( !zCreate ){ - return SQLITE_NOMEM; - } - rc = sqlite3_exec(db, zCreate, 0, 0, 0); - sqlite3_free(zCreate); - if( rc!=SQLITE_OK ){ - return rc; - } - } +#endif /* _FTS3_TOKENIZER_H_ */ - appStmt[0] = &pRtree->pReadNode; - appStmt[1] = &pRtree->pWriteNode; - appStmt[2] = &pRtree->pDeleteNode; - appStmt[3] = &pRtree->pReadRowid; - appStmt[4] = &pRtree->pWriteRowid; - appStmt[5] = &pRtree->pDeleteRowid; - appStmt[6] = &pRtree->pReadParent; - appStmt[7] = &pRtree->pWriteParent; - appStmt[8] = &pRtree->pDeleteParent; +/************** End of fts3_tokenizer.h **************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ +/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ +/************** Begin file fts3_hash.h ***************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implementation +** used in SQLite. We've modified it slightly to serve as a standalone +** hash table implementation for the full-text indexing module. +** +*/ +#ifndef _FTS3_HASH_H_ +#define _FTS3_HASH_H_ - rc = rtreeQueryStat1(db, pRtree); - for(i=0; iiNodeSize is populated and SQLITE_OK returned. -** Otherwise, an SQLite error code is returned. +** There are 2 different modes of operation for a hash table: ** -** If this function is being called as part of an xConnect(), then the rtree -** table already exists. In this case the node-size is determined by inspecting -** the root node of the tree. +** FTS3_HASH_STRING pKey points to a string that is nKey bytes long +** (including the null-terminator, if any). Case +** is respected in comparisons. ** -** Otherwise, for an xCreate(), use 64 bytes less than the database page-size. -** This ensures that each node is stored on a single database page. If the -** database page-size is so large that more than RTREE_MAXCELLS entries -** would fit in a single node, use a smaller node-size. +** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. +** memcmp() is used to compare keys. +** +** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. */ -static int getNodeSize( - sqlite3 *db, /* Database handle */ - Rtree *pRtree, /* Rtree handle */ - int isCreate, /* True for xCreate, false for xConnect */ - char **pzErr /* OUT: Error message, if any */ -){ - int rc; - char *zSql; - if( isCreate ){ - int iPageSize = 0; - zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb); - rc = getIntFromStmt(db, zSql, &iPageSize); - if( rc==SQLITE_OK ){ - pRtree->iNodeSize = iPageSize-64; - if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)iNodeSize ){ - pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS; - } - }else{ - *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - } - }else{ - zSql = sqlite3_mprintf( - "SELECT length(data) FROM '%q'.'%q_node' WHERE nodeno = 1", - pRtree->zDb, pRtree->zName - ); - rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); - if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - } - } - - sqlite3_free(zSql); - return rc; -} +#define FTS3_HASH_STRING 1 +#define FTS3_HASH_BINARY 2 -/* -** This function is the implementation of both the xConnect and xCreate -** methods of the r-tree virtual table. -** -** argv[0] -> module name -** argv[1] -> database name -** argv[2] -> table name -** argv[...] -> column names... +/* +** Access routines. To delete, insert a NULL pointer. */ -static int rtreeInit( - sqlite3 *db, /* Database connection */ - void *pAux, /* One of the RTREE_COORD_* constants */ - int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ - sqlite3_vtab **ppVtab, /* OUT: New virtual table */ - char **pzErr, /* OUT: Error message, if any */ - int isCreate /* True for xCreate, false for xConnect */ -){ - int rc = SQLITE_OK; - Rtree *pRtree; - int nDb; /* Length of string argv[1] */ - int nName; /* Length of string argv[2] */ - int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32); +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); +SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); - const char *aErrMsg[] = { - 0, /* 0 */ - "Wrong number of columns for an rtree table", /* 1 */ - "Too few columns for an rtree table", /* 2 */ - "Too many columns for an rtree table" /* 3 */ - }; +/* +** Shorthand for the functions above +*/ +#define fts3HashInit sqlite3Fts3HashInit +#define fts3HashInsert sqlite3Fts3HashInsert +#define fts3HashFind sqlite3Fts3HashFind +#define fts3HashClear sqlite3Fts3HashClear +#define fts3HashFindElem sqlite3Fts3HashFindElem - int iErr = (argc<6) ? 2 : argc>(RTREE_MAX_DIMENSIONS*2+4) ? 3 : argc%2; - if( aErrMsg[iErr] ){ - *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]); - return SQLITE_ERROR; - } +/* +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Fts3Hash h; +** Fts3HashElem *p; +** ... +** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ +** SomeStructure *pData = fts3HashData(p); +** // do something with pData +** } +*/ +#define fts3HashFirst(H) ((H)->first) +#define fts3HashNext(E) ((E)->next) +#define fts3HashData(E) ((E)->data) +#define fts3HashKey(E) ((E)->pKey) +#define fts3HashKeysize(E) ((E)->nKey) - sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); +/* +** Number of entries in a hash table +*/ +#define fts3HashCount(H) ((H)->count) - /* Allocate the sqlite3_vtab structure */ - nDb = (int)strlen(argv[1]); - nName = (int)strlen(argv[2]); - pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); - if( !pRtree ){ - return SQLITE_NOMEM; - } - memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); - pRtree->nBusy = 1; - pRtree->base.pModule = &rtreeModule; - pRtree->zDb = (char *)&pRtree[1]; - pRtree->zName = &pRtree->zDb[nDb+1]; - pRtree->nDim = (argc-4)/2; - pRtree->nBytesPerCell = 8 + pRtree->nDim*4*2; - pRtree->eCoordType = eCoordType; - memcpy(pRtree->zDb, argv[1], nDb); - memcpy(pRtree->zName, argv[2], nName); +#endif /* _FTS3_HASH_H_ */ - /* Figure out the node size to use. */ - rc = getNodeSize(db, pRtree, isCreate, pzErr); +/************** End of fts3_hash.h *******************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ - /* Create/Connect to the underlying relational database schema. If - ** that is successful, call sqlite3_declare_vtab() to configure - ** the r-tree table schema. - */ - if( rc==SQLITE_OK ){ - if( (rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate)) ){ - *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - }else{ - char *zSql = sqlite3_mprintf("CREATE TABLE x(%s", argv[3]); - char *zTmp; - int ii; - for(ii=4; zSql && iinBusy==1 ); - rtreeRelease(pRtree); - } - return rc; -} +/* +** This constant controls how often segments are merged. Once there are +** FTS3_MERGE_COUNT segments of level N, they are merged into a single +** segment of level N+1. +*/ +#define FTS3_MERGE_COUNT 16 /* -** Implementation of a scalar function that decodes r-tree nodes to -** human readable strings. This can be used for debugging and analysis. -** -** The scalar function takes two arguments: (1) the number of dimensions -** to the rtree (between 1 and 5, inclusive) and (2) a blob of data containing -** an r-tree node. For a two-dimensional r-tree structure called "rt", to -** deserialize all nodes, a statement like: -** -** SELECT rtreenode(2, data) FROM rt_node; -** -** The human readable string takes the form of a Tcl list with one -** entry for each cell in the r-tree node. Each entry is itself a -** list, containing the 8-byte rowid/pageno followed by the -** *2 coordinates. +** This is the maximum amount of data (in bytes) to store in the +** Fts3Table.pendingTerms hash table. Normally, the hash table is +** populated as documents are inserted/updated/deleted in a transaction +** and used to create a new segment when the transaction is committed. +** However if this limit is reached midway through a transaction, a new +** segment is created and the hash table cleared immediately. */ -static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ - char *zText = 0; - RtreeNode node; - Rtree tree; - int ii; +#define FTS3_MAX_PENDING_DATA (1*1024*1024) - UNUSED_PARAMETER(nArg); - memset(&node, 0, sizeof(RtreeNode)); - memset(&tree, 0, sizeof(Rtree)); - tree.nDim = sqlite3_value_int(apArg[0]); - tree.nBytesPerCell = 8 + 8 * tree.nDim; - node.zData = (u8 *)sqlite3_value_blob(apArg[1]); +/* +** Macro to return the number of elements in an array. SQLite has a +** similar macro called ArraySize(). Use a different name to avoid +** a collision when building an amalgamation with built-in FTS3. +*/ +#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) - for(ii=0; ii(y)?(x):(y)) #endif - nCell = (int)strlen(zCell); - } - if( zText ){ - char *zTextNew = sqlite3_mprintf("%s {%s}", zText, zCell); - sqlite3_free(zText); - zText = zTextNew; - }else{ - zText = sqlite3_mprintf("{%s}", zCell); - } - } - - sqlite3_result_text(ctx, zText, -1, sqlite3_free); -} +/* +** Maximum length of a varint encoded integer. The varint format is different +** from that used by SQLite, so the maximum length is 10, not 9. +*/ +#define FTS3_VARINT_MAX 10 -/* This routine implements an SQL function that returns the "depth" parameter -** from the front of a blob that is an r-tree node. For example: +/* +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. ** -** SELECT rtreedepth(data) FROM rt_node WHERE nodeno=1; +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. ** -** The depth value is 0 for all nodes other than the root node, and the root -** node always has nodeno=1, so the example above is the primary use for this -** routine. This routine is intended for testing and analysis only. +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. */ -static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ - UNUSED_PARAMETER(nArg); - if( sqlite3_value_type(apArg[0])!=SQLITE_BLOB - || sqlite3_value_bytes(apArg[0])<2 - ){ - sqlite3_result_error(ctx, "Invalid argument to rtreedepth()", -1); - }else{ - u8 *zBlob = (u8 *)sqlite3_value_blob(apArg[0]); - sqlite3_result_int(ctx, readInt16(zBlob)); - } -} +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" /* -** Register the r-tree module with database handle db. This creates the -** virtual table module "rtree" and the debugging/analysis scalar -** function "rtreenode". +** The testcase() macro is only used by the amalgamation. If undefined, +** make it a no-op. */ -SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db){ - const int utf8 = SQLITE_UTF8; - int rc; - - rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0); - } - if( rc==SQLITE_OK ){ -#ifdef SQLITE_RTREE_INT_ONLY - void *c = (void *)RTREE_COORD_INT32; -#else - void *c = (void *)RTREE_COORD_REAL32; +#ifndef testcase +# define testcase(X) #endif - rc = sqlite3_create_module_v2(db, "rtree", &rtreeModule, c, 0); - } - if( rc==SQLITE_OK ){ - void *c = (void *)RTREE_COORD_INT32; - rc = sqlite3_create_module_v2(db, "rtree_i32", &rtreeModule, c, 0); - } - - return rc; -} /* -** This routine deletes the RtreeGeomCallback object that was attached -** one of the SQL functions create by sqlite3_rtree_geometry_callback() -** or sqlite3_rtree_query_callback(). In other words, this routine is the -** destructor for an RtreeGeomCallback objecct. This routine is called when -** the corresponding SQL function is deleted. +** Terminator values for position-lists and column-lists. */ -static void rtreeFreeCallback(void *p){ - RtreeGeomCallback *pInfo = (RtreeGeomCallback*)p; - if( pInfo->xDestructor ) pInfo->xDestructor(pInfo->pContext); - sqlite3_free(p); -} +#define POS_COLUMN (1) /* Column-list terminator */ +#define POS_END (0) /* Position-list terminator */ /* -** This routine frees the BLOB that is returned by geomCallback(). +** This section provides definitions to allow the +** FTS3 extension to be compiled outside of the +** amalgamation. */ -static void rtreeMatchArgFree(void *pArg){ - int i; - RtreeMatchArg *p = (RtreeMatchArg*)pArg; - for(i=0; inParam; i++){ - sqlite3_value_free(p->apSqlParam[i]); - } - sqlite3_free(p); -} - +#ifndef SQLITE_AMALGAMATION /* -** Each call to sqlite3_rtree_geometry_callback() or -** sqlite3_rtree_query_callback() creates an ordinary SQLite -** scalar function that is implemented by this routine. -** -** All this function does is construct an RtreeMatchArg object that -** contains the geometry-checking callback routines and a list of -** parameters to this function, then return that RtreeMatchArg object -** as a BLOB. -** -** The R-Tree MATCH operator will read the returned BLOB, deserialize -** the RtreeMatchArg object, and use the RtreeMatchArg object to figure -** out which elements of the R-Tree should be returned by the query. +** Macros indicating that conditional expressions are always true or +** false. */ -static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ - RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); - RtreeMatchArg *pBlob; - int nBlob; - int memErr = 0; - - nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue) - + nArg*sizeof(sqlite3_value*); - pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); - if( !pBlob ){ - sqlite3_result_error_nomem(ctx); - }else{ - int i; - pBlob->magic = RTREE_GEOMETRY_MAGIC; - pBlob->cb = pGeomCtx[0]; - pBlob->apSqlParam = (sqlite3_value**)&pBlob->aParam[nArg]; - pBlob->nParam = nArg; - for(i=0; iapSqlParam[i] = sqlite3_value_dup(aArg[i]); - if( pBlob->apSqlParam[i]==0 ) memErr = 1; -#ifdef SQLITE_RTREE_INT_ONLY - pBlob->aParam[i] = sqlite3_value_int64(aArg[i]); +#ifdef SQLITE_COVERAGE_TEST +# define ALWAYS(x) (1) +# define NEVER(X) (0) +#elif defined(SQLITE_DEBUG) +# define ALWAYS(x) sqlite3Fts3Always((x)!=0) +# define NEVER(x) sqlite3Fts3Never((x)!=0) +SQLITE_PRIVATE int sqlite3Fts3Always(int b); +SQLITE_PRIVATE int sqlite3Fts3Never(int b); #else - pBlob->aParam[i] = sqlite3_value_double(aArg[i]); +# define ALWAYS(x) (x) +# define NEVER(x) (x) #endif - } - if( memErr ){ - sqlite3_result_error_nomem(ctx); - rtreeMatchArgFree(pBlob); - }else{ - sqlite3_result_blob(ctx, pBlob, nBlob, rtreeMatchArgFree); - } - } -} /* -** Register a new geometry function for use with the r-tree MATCH operator. +** Internal types used by SQLite. */ -SQLITE_API int sqlite3_rtree_geometry_callback( - sqlite3 *db, /* Register SQL function on this connection */ - const char *zGeom, /* Name of the new SQL function */ - int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ - void *pContext /* Extra data associated with the callback */ -){ - RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ - - /* Allocate and populate the context object. */ - pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); - if( !pGeomCtx ) return SQLITE_NOMEM; - pGeomCtx->xGeom = xGeom; - pGeomCtx->xQueryFunc = 0; - pGeomCtx->xDestructor = 0; - pGeomCtx->pContext = pContext; - return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, - (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback - ); -} +typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ +typedef short int i16; /* 2-byte (or larger) signed integer */ +typedef unsigned int u32; /* 4-byte unsigned integer */ +typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ /* -** Register a new 2nd-generation geometry function for use with the -** r-tree MATCH operator. +** Macro used to suppress compiler warnings for unused parameters. */ -SQLITE_API int sqlite3_rtree_query_callback( - sqlite3 *db, /* Register SQL function on this connection */ - const char *zQueryFunc, /* Name of new SQL function */ - int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ - void *pContext, /* Extra data passed into the callback */ - void (*xDestructor)(void*) /* Destructor for the extra data */ -){ - RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ - - /* Allocate and populate the context object. */ - pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); - if( !pGeomCtx ) return SQLITE_NOMEM; - pGeomCtx->xGeom = 0; - pGeomCtx->xQueryFunc = xQueryFunc; - pGeomCtx->xDestructor = xDestructor; - pGeomCtx->pContext = pContext; - return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, - (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback - ); -} +#define UNUSED_PARAMETER(x) (void)(x) -#if !SQLITE_CORE -#ifdef _WIN32 -__declspec(dllexport) +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 #endif -SQLITE_API int sqlite3_rtree_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi) - return sqlite3RtreeInit(db); -} + +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) #endif +#endif /* SQLITE_AMALGAMATION */ + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); +# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() +#else +# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB #endif -/************** End of rtree.c ***********************************************/ -/************** Begin file icu.c *********************************************/ +typedef struct Fts3Table Fts3Table; +typedef struct Fts3Cursor Fts3Cursor; +typedef struct Fts3Expr Fts3Expr; +typedef struct Fts3Phrase Fts3Phrase; +typedef struct Fts3PhraseToken Fts3PhraseToken; + +typedef struct Fts3Doclist Fts3Doclist; +typedef struct Fts3SegFilter Fts3SegFilter; +typedef struct Fts3DeferredToken Fts3DeferredToken; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; + /* -** 2007 May 6 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ -** -** This file implements an integration between the ICU library -** ("International Components for Unicode", an open-source library -** for handling unicode data) and SQLite. The integration uses -** ICU to provide the following to SQLite: -** -** * An implementation of the SQL regexp() function (and hence REGEXP -** operator) using the ICU uregex_XX() APIs. -** -** * Implementations of the SQL scalar upper() and lower() functions -** for case mapping. -** -** * Integration of ICU and SQLite collation sequences. -** -** * An implementation of the LIKE operator that uses ICU to -** provide case-independent matching. +** A connection to a fulltext index is an instance of the following +** structure. The xCreate and xConnect methods create an instance +** of this structure and xDestroy and xDisconnect free that instance. +** All other methods receive a pointer to the structure as one of their +** arguments. */ +struct Fts3Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + sqlite3 *db; /* The database connection */ + const char *zDb; /* logical database name */ + const char *zName; /* virtual table name */ + int nColumn; /* number of named columns in virtual table */ + char **azColumn; /* column names. malloced */ + u8 *abNotindexed; /* True for 'notindexed' columns */ + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ + char *zLanguageid; /* languageid=xxx option, or NULL */ + int nAutoincrmerge; /* Value configured by 'automerge' */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + /* Precompiled statements used by the implementation. Each of these + ** statements is run and reset within a single virtual table API call. + */ + sqlite3_stmt *aStmt[40]; -/* Include ICU headers */ -#include -#include -#include -#include + char *zReadExprlist; + char *zWriteExprlist; -/* #include */ + int nNodeSize; /* Soft limit for node size */ + u8 bFts4; /* True for FTS4, false for FTS3 */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ + u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ -#ifndef SQLITE_CORE -/* #include "sqlite3ext.h" */ - SQLITE_EXTENSION_INIT1 -#else -/* #include "sqlite3.h" */ + /* + ** The following array of hash tables is used to buffer pending index + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. + */ + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + int iPrevLangid; /* Langid of recently inserted document */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + /* State variables used for validating that the transaction control + ** methods of the virtual table are called at appropriate times. These + ** values do not contribute to FTS functionality; they are used for + ** verifying the operation of the SQLite core. + */ + int inTransaction; /* True after xBegin but before xCommit/xRollback */ + int mxSavepoint; /* Largest valid xSavepoint integer */ #endif +#ifdef SQLITE_TEST + /* True to disable the incremental doclist optimization. This is controled + ** by special insert command 'test-no-incr-doclist'. */ + int bNoIncrDoclist; +#endif +}; + /* -** Maximum length (in bytes) of the pattern in a LIKE or GLOB -** operator. +** When the core wants to read from the virtual table, it creates a +** virtual table cursor (an instance of the following structure) using +** the xOpen method. Cursors are destroyed using the xClose method. */ -#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH -# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 -#endif +struct Fts3Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + i16 eSearch; /* Search strategy (see below) */ + u8 isEof; /* True if at End Of Results */ + u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ + Fts3Expr *pExpr; /* Parsed MATCH query string */ + int iLangid; /* Language being queried for */ + int nPhrase; /* Number of matchable phrases in query */ + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ + sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ + char *pNextId; /* Pointer into the body of aDoclist */ + char *aDoclist; /* List of docids for full-text queries */ + int nDoclist; /* Size of buffer at aDoclist */ + u8 bDesc; /* True to sort in descending order */ + int eEvalmode; /* An FTS3_EVAL_XX constant */ + int nRowAvg; /* Average size of database rows, in pages */ + sqlite3_int64 nDoc; /* Documents in table */ + i64 iMinDocid; /* Minimum docid to return */ + i64 iMaxDocid; /* Maximum docid to return */ + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ + u32 *aMatchinfo; /* Information about most recent match */ + int nMatchinfo; /* Number of elements in aMatchinfo[] */ + char *zMatchinfo; /* Matchinfo specification */ +}; + +#define FTS3_EVAL_FILTER 0 +#define FTS3_EVAL_NEXT 1 +#define FTS3_EVAL_MATCHINFO 2 /* -** Version of sqlite3_free() that is always a function, never a macro. +** The Fts3Cursor.eSearch member is always set to one of the following. +** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index +** of the column to be searched. For example, in +** +** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); +** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; +** +** Because the LHS of the MATCH operator is 2nd column "b", +** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, +** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" +** indicating that all columns should be searched, +** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. */ -static void xFree(void *p){ - sqlite3_free(p); -} +#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ +#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ +#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ /* -** This lookup table is used to help decode the first byte of -** a multi-byte UTF8 character. It is copied here from SQLite source -** code file utf8.c. +** The lower 16-bits of the sqlite3_index_info.idxNum value set by +** the xBestIndex() method contains the Fts3Cursor.eSearch value described +** above. The upper 16-bits contain a combination of the following +** bits, used to describe extra constraints on full-text searches. */ -static const unsigned char icuUtf8Trans1[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ +#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ +#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ }; -#define SQLITE_ICU_READ_UTF8(zIn, c) \ - c = *(zIn++); \ - if( c>=0xc0 ){ \ - c = icuUtf8Trans1[c-0xc0]; \ - while( (*zIn & 0xc0)==0x80 ){ \ - c = (c<<6) + (0x3f & *(zIn++)); \ - } \ - } +/* +** A "phrase" is a sequence of one or more tokens that must match in +** sequence. A single token is the base case and the most common case. +** For a sequence of tokens contained in double-quotes (i.e. "one two three") +** nToken will be the number of tokens in the string. +*/ +struct Fts3PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer z */ + int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ -#define SQLITE_ICU_SKIP_UTF8(zIn) \ - assert( *zIn ); \ - if( *(zIn++)>=0xc0 ){ \ - while( (*zIn & 0xc0)==0x80 ){zIn++;} \ - } + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ +}; + +struct Fts3Phrase { + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an + ** OR condition. */ + char *pOrPoslist; + i64 iOrDocid; + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ +}; /* -** Compare two UTF-8 strings for equality where the first string is -** a "LIKE" expression. Return true (1) if they are the same and -** false (0) if they are different. +** A tree of these objects forms the RHS of a MATCH operator. +** +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. +** +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. */ -static int icuLikeCompare( - const uint8_t *zPattern, /* LIKE pattern */ - const uint8_t *zString, /* The UTF-8 string to compare against */ - const UChar32 uEsc /* The escape character */ -){ - static const int MATCH_ONE = (UChar32)'_'; - static const int MATCH_ALL = (UChar32)'%'; +struct Fts3Expr { + int eType; /* One of the FTSQUERY_XXX values defined below */ + int nNear; /* Valid if eType==FTSQUERY_NEAR */ + Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ - int prevEscape = 0; /* True if the previous character was uEsc */ + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ - while( 1 ){ + u32 *aMI; +}; - /* Read (and consume) the next character from the input pattern. */ - UChar32 uPattern; - SQLITE_ICU_READ_UTF8(zPattern, uPattern); - if( uPattern==0 ) break; +/* +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: +** +** "a OR b AND c NOT d NEAR e" +** +** is equivalent to: +** +** "a OR (b AND (c NOT (d NEAR e)))" +*/ +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 - /* There are now 4 possibilities: - ** - ** 1. uPattern is an unescaped match-all character "%", - ** 2. uPattern is an unescaped match-one character "_", - ** 3. uPattern is an unescaped escape character, or - ** 4. uPattern is to be handled as an ordinary character - */ - if( !prevEscape && uPattern==MATCH_ALL ){ - /* Case 1. */ - uint8_t c; - /* Skip any MATCH_ALL or MATCH_ONE characters that follow a - ** MATCH_ALL. For each MATCH_ONE, skip one character in the - ** test string. - */ - while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ - if( c==MATCH_ONE ){ - if( *zString==0 ) return 0; - SQLITE_ICU_SKIP_UTF8(zString); - } - zPattern++; - } +/* fts3_write.c */ +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, + sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); - if( *zPattern==0 ) return 1; +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); - while( *zString ){ - if( icuLikeCompare(zPattern, zString, uEsc) ){ - return 1; - } - SQLITE_ICU_SKIP_UTF8(zString); - } - return 0; +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); +#else +# define sqlite3Fts3FreeDeferredTokens(x) +# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK +# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK +# define sqlite3Fts3FreeDeferredDoclists(x) +# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK +#endif - }else if( !prevEscape && uPattern==MATCH_ONE ){ - /* Case 2. */ - if( *zString==0 ) return 0; - SQLITE_ICU_SKIP_UTF8(zString); +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); - }else if( !prevEscape && uPattern==uEsc){ - /* Case 3. */ - prevEscape = 1; +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 - }else{ - /* Case 4. */ - UChar32 uString; - SQLITE_ICU_READ_UTF8(zString, uString); - uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT); - uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT); - if( uString!=uPattern ){ - return 0; - } - prevEscape = 0; - } - } +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); - return *zString==0; -} +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, + int, int, int, const char *, int, int, int, Fts3MultiSegReader *); -/* -** Implementation of the like() SQL function. This function implements -** the build-in LIKE operator. The first argument to the function is the -** pattern and the second argument is the string. So, the SQL statements: -** -** A LIKE B -** -** is implemented as like(B, A). If there is an escape character E, -** -** A LIKE B ESCAPE E -** -** is mapped to like(B, A, E). -*/ -static void icuLikeFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *zA = sqlite3_value_text(argv[0]); - const unsigned char *zB = sqlite3_value_text(argv[1]); - UChar32 uEsc = 0; +/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ +#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 +#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 +#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 +#define FTS3_SEGMENT_PREFIX 0x00000008 +#define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 + +/* Type passed as 4th argument to SegmentReaderIterate() */ +struct Fts3SegFilter { + const char *zTerm; + int nTerm; + int iCol; + int flags; +}; + +struct Fts3MultiSegReader { + /* Used internally by sqlite3Fts3SegReaderXXX() calls */ + Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ + int nSegment; /* Size of apSegment array */ + int nAdvance; /* How many seg-readers to advance */ + Fts3SegFilter *pFilter; /* Pointer to filter object */ + char *aBuffer; /* Buffer to merge doclists in */ + int nBuffer; /* Allocated size of aBuffer[] in bytes */ + + int iColFilter; /* If >=0, filter for this column */ + int bRestart; + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ + + /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ + char *zTerm; /* Pointer to term buffer */ + int nTerm; /* Size of zTerm in bytes */ + char *aDoclist; /* Pointer to doclist buffer */ + int nDoclist; /* Size of aDoclist[] in bytes */ +}; + +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + +#define fts3GetVarint32(p, piVal) ( \ + (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ +) + +/* fts3.c */ +SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); + +/* fts3_tokenizer.c */ +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, + sqlite3_tokenizer **, char ** +); +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); - /* Limit the length of the LIKE or GLOB pattern to avoid problems - ** of deep recursion and N*N behavior in patternCompare(). - */ - if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ - sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); - return; - } +/* fts3_snippet.c */ +SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); +SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, + const char *, const char *, int, int +); +SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); +/* fts3_expr.c */ +SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, + char **, int, int, int, const char *, int, Fts3Expr **, char ** +); +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); +#endif - if( argc==3 ){ - /* The escape character string must consist of a single UTF-8 character. - ** Otherwise, return an error. - */ - int nE= sqlite3_value_bytes(argv[2]); - const unsigned char *zE = sqlite3_value_text(argv[2]); - int i = 0; - if( zE==0 ) return; - U8_NEXT(zE, i, nE, uEsc); - if( i!=nE){ - sqlite3_result_error(context, - "ESCAPE expression must be a single character", -1); - return; - } - } +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, + sqlite3_tokenizer_cursor ** +); - if( zA && zB ){ - sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); - } -} +/* fts3_aux.c */ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); -/* -** This function is called when an ICU function called from within -** the implementation of an SQL scalar function returns an error. -** -** The scalar function context passed as the first argument is -** loaded with an error message based on the following two args. -*/ -static void icuFunctionError( - sqlite3_context *pCtx, /* SQLite scalar function context */ - const char *zName, /* Name of ICU function that failed */ - UErrorCode e /* Error code returned by ICU function */ -){ - char zBuf[128]; - sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); - zBuf[127] = '\0'; - sqlite3_result_error(pCtx, zBuf, -1); -} +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); -/* -** Function to delete compiled regexp objects. Registered as -** a destructor function with sqlite3_set_auxdata(). -*/ -static void icuRegexpDelete(void *p){ - URegularExpression *pExpr = (URegularExpression *)p; - uregex_close(pExpr); -} +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); -/* -** Implementation of SQLite REGEXP operator. This scalar function takes -** two arguments. The first is a regular expression pattern to compile -** the second is a string to match against that pattern. If either -** argument is an SQL NULL, then NULL Is returned. Otherwise, the result -** is 1 if the string matches the pattern, or 0 otherwise. -** -** SQLite maps the regexp() function to the regexp() operator such -** that the following two are equivalent: -** -** zString REGEXP zPattern -** regexp(zPattern, zString) -** -** Uses the following ICU regexp APIs: -** -** uregex_open() -** uregex_matches() -** uregex_close() -*/ -static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ - UErrorCode status = U_ZERO_ERROR; - URegularExpression *pExpr; - UBool res; - const UChar *zString = sqlite3_value_text16(apArg[1]); +/* fts3_tokenize_vtab.c */ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); - (void)nArg; /* Unused parameter */ +/* fts3_unicode2.c (functions generated by parsing unicode text files) */ +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); +#endif - /* If the left hand side of the regexp operator is NULL, - ** then the result is also NULL. - */ - if( !zString ){ - return; - } +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ +#endif /* _FTSINT_H */ - pExpr = sqlite3_get_auxdata(p, 0); - if( !pExpr ){ - const UChar *zPattern = sqlite3_value_text16(apArg[0]); - if( !zPattern ){ - return; - } - pExpr = uregex_open(zPattern, -1, 0, 0, &status); +/************** End of fts3Int.h *********************************************/ +/************** Continuing where we left off in fts3.c ***********************/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - if( U_SUCCESS(status) ){ - sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); - }else{ - assert(!pExpr); - icuFunctionError(p, "uregex_open", status); - return; - } - } +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) +# define SQLITE_CORE 1 +#endif - /* Configure the text that the regular expression operates on. */ - uregex_setText(pExpr, zString, -1, &status); - if( !U_SUCCESS(status) ){ - icuFunctionError(p, "uregex_setText", status); - return; - } +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ - /* Attempt the match */ - res = uregex_matches(pExpr, 0, &status); - if( !U_SUCCESS(status) ){ - icuFunctionError(p, "uregex_matches", status); - return; - } +#ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT1 +#endif - /* Set the text that the regular expression operates on to a NULL - ** pointer. This is not really necessary, but it is tidier than - ** leaving the regular expression object configured with an invalid - ** pointer after this function returns. - */ - uregex_setText(pExpr, 0, 0, &status); +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); - /* Return 1 or 0. */ - sqlite3_result_int(p, res ? 1 : 0); -} +#ifndef SQLITE_AMALGAMATION +# if defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; } +SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; } +# endif +#endif -/* -** Implementations of scalar functions for case mapping - upper() and -** lower(). Function upper() converts its input to upper-case (ABC). -** Function lower() converts to lower-case (abc). -** -** ICU provides two types of case mapping, "general" case mapping and -** "language specific". Refer to ICU documentation for the differences -** between the two. -** -** To utilise "general" case mapping, the upper() or lower() scalar -** functions are invoked with one argument: -** -** upper('ABC') -> 'abc' -** lower('abc') -> 'ABC' -** -** To access ICU "language specific" case mapping, upper() or lower() -** should be invoked with two arguments. The second argument is the name -** of the locale to use. Passing an empty string ("") or SQL NULL value -** as the second argument is the same as invoking the 1 argument version -** of upper() or lower(). -** -** lower('I', 'en_us') -> 'i' -** lower('I', 'tr_tr') -> '\u131' (small dotless i) -** -** http://www.icu-project.org/userguide/posix.html#case_mappings +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. +** The number of bytes written is returned. */ -static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ - const UChar *zInput; /* Pointer to input string */ - UChar *zOutput = 0; /* Pointer to output buffer */ - int nInput; /* Size of utf-16 input string in bytes */ - int nOut; /* Size of output buffer in bytes */ - int cnt; - int bToUpper; /* True for toupper(), false for tolower() */ - UErrorCode status; - const char *zLocale = 0; +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ + unsigned char *q = (unsigned char *) p; + sqlite_uint64 vu = v; + do{ + *q++ = (unsigned char) ((vu & 0x7f) | 0x80); + vu >>= 7; + }while( vu!=0 ); + q[-1] &= 0x7f; /* turn off high bit in final byte */ + assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); + return (int) (q - (unsigned char *)p); +} - assert(nArg==1 || nArg==2); - bToUpper = (sqlite3_user_data(p)!=0); - if( nArg==2 ){ - zLocale = (const char *)sqlite3_value_text(apArg[1]); - } +#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ + v = (v & mask1) | ( (*ptr++) << shift ); \ + if( (v & mask2)==0 ){ var = v; return ret; } +#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ + v = (*ptr++); \ + if( (v & mask2)==0 ){ var = v; return ret; } - zInput = sqlite3_value_text16(apArg[0]); - if( !zInput ){ - return; - } - nOut = nInput = sqlite3_value_bytes16(apArg[0]); - if( nOut==0 ){ - sqlite3_result_text16(p, "", 0, SQLITE_STATIC); - return; - } +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ + const char *pStart = p; + u32 a; + u64 b; + int shift; - for(cnt=0; cnt<2; cnt++){ - UChar *zNew = sqlite3_realloc(zOutput, nOut); - if( zNew==0 ){ - sqlite3_free(zOutput); - sqlite3_result_error_nomem(p); - return; - } - zOutput = zNew; - status = U_ZERO_ERROR; - if( bToUpper ){ - nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); - }else{ - nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); - } + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); + b = (a & 0x0FFFFFFF ); - if( U_SUCCESS(status) ){ - sqlite3_result_text16(p, zOutput, nOut, xFree); - }else if( status==U_BUFFER_OVERFLOW_ERROR ){ - assert( cnt==0 ); - continue; - }else{ - icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); - } - return; + for(shift=28; shift<=63; shift+=7){ + u64 c = *p++; + b += (c&0x7F) << shift; + if( (c & 0x80)==0 ) break; } - assert( 0 ); /* Unreachable */ + *v = b; + return (int)(p - pStart); } /* -** Collation sequence destructor function. The pCtx argument points to -** a UCollator structure previously allocated using ucol_open(). +** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a +** 32-bit integer before it is returned. */ -static void icuCollationDel(void *pCtx){ - UCollator *p = (UCollator *)pCtx; - ucol_close(p); +SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){ + u32 a; + +#ifndef fts3GetVarint32 + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *pi, 1); +#else + a = (*p++); + assert( a & 0x80 ); +#endif + + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *pi, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *pi, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4); + a = (a & 0x0FFFFFFF ); + *pi = (int)(a | ((u32)(*p & 0x0F) << 28)); + return 5; } /* -** Collation sequence comparison function. The pCtx argument points to -** a UCollator structure previously allocated using ucol_open(). +** Return the number of bytes required to encode v as a varint */ -static int icuCollationColl( - void *pCtx, - int nLeft, - const void *zLeft, - int nRight, - const void *zRight -){ - UCollationResult res; - UCollator *p = (UCollator *)pCtx; - res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2); - switch( res ){ - case UCOL_LESS: return -1; - case UCOL_GREATER: return +1; - case UCOL_EQUAL: return 0; - } - assert(!"Unexpected return value from ucol_strcoll()"); - return 0; +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){ + int i = 0; + do{ + i++; + v >>= 7; + }while( v!=0 ); + return i; } /* -** Implementation of the scalar function icu_load_collation(). +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. ** -** This scalar function is used to add ICU collation based collation -** types to an SQLite database connection. It is intended to be called -** as follows: +** Examples: ** -** SELECT icu_load_collation(, ); +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno ** -** Where is a string containing an ICU locale identifier (i.e. -** "en_AU", "tr_TR" etc.) and is the name of the -** collation sequence to create. */ -static void icuLoadCollation( - sqlite3_context *p, - int nArg, - sqlite3_value **apArg -){ - sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); - UErrorCode status = U_ZERO_ERROR; - const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ - const char *zName; /* SQL Collation sequence name (eg. "japanese") */ - UCollator *pUCollator; /* ICU library collation object */ - int rc; /* Return code from sqlite3_create_collation_x() */ - - assert(nArg==2); - (void)nArg; /* Unused parameter */ - zLocale = (const char *)sqlite3_value_text(apArg[0]); - zName = (const char *)sqlite3_value_text(apArg[1]); +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ + char quote; /* Quote character (if any ) */ - if( !zLocale || !zName ){ - return; - } + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + int iIn = 1; /* Index of next byte to read from input */ + int iOut = 0; /* Index of next byte to write to output */ - pUCollator = ucol_open(zLocale, &status); - if( !U_SUCCESS(status) ){ - icuFunctionError(p, "ucol_open", status); - return; - } - assert(p); + /* If the first byte was a '[', then the close-quote character is a ']' */ + if( quote=='[' ) quote = ']'; - rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator, - icuCollationColl, icuCollationDel - ); - if( rc!=SQLITE_OK ){ - ucol_close(pUCollator); - sqlite3_result_error(p, "Error registering collation function", -1); + while( z[iIn] ){ + if( z[iIn]==quote ){ + if( z[iIn+1]!=quote ) break; + z[iOut++] = quote; + iIn += 2; + }else{ + z[iOut++] = z[iIn++]; + } + } + z[iOut] = '\0'; } } /* -** Register the ICU extension functions with database db. +** Read a single varint from the doclist at *pp and advance *pp to point +** to the first byte past the end of the varint. Add the value of the varint +** to *pVal. */ -SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db){ - struct IcuScalar { - const char *zName; /* Function name */ - int nArg; /* Number of arguments */ - int enc; /* Optimal text encoding */ - void *pContext; /* sqlite3_user_data() context */ - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } scalars[] = { - {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, - - {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - - {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - - {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, - {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, - - {"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation}, - }; - - int rc = SQLITE_OK; - int i; - - for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ - struct IcuScalar *p = &scalars[i]; - rc = sqlite3_create_function( - db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0 - ); - } - - return rc; -} - -#if !SQLITE_CORE -#ifdef _WIN32 -__declspec(dllexport) -#endif -SQLITE_API int sqlite3_icu_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi) - return sqlite3IcuInit(db); +static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + *pVal += iVal; } -#endif - -#endif -/************** End of icu.c *************************************************/ -/************** Begin file fts3_icu.c ****************************************/ /* -** 2007 June 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and sets *pVal by the varint value. ** -************************************************************************* -** This file implements a tokenizer for fts3 based on the ICU library. +** Argument pStart points to the first byte of the doclist that the +** varint is part of. */ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#ifdef SQLITE_ENABLE_ICU - -/* #include */ -/* #include */ -/* #include "fts3_tokenizer.h" */ - -#include -/* #include */ -/* #include */ -#include - -typedef struct IcuTokenizer IcuTokenizer; -typedef struct IcuCursor IcuCursor; - -struct IcuTokenizer { - sqlite3_tokenizer base; - char *zLocale; -}; - -struct IcuCursor { - sqlite3_tokenizer_cursor base; - - UBreakIterator *pIter; /* ICU break-iterator object */ - int nChar; /* Number of UChar elements in pInput */ - UChar *aChar; /* Copy of input using utf-16 encoding */ - int *aOffset; /* Offsets of each character in utf-8 input */ +static void fts3GetReverseVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p; - int nBuffer; - char *zBuffer; + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; - int iToken; -}; + sqlite3Fts3GetVarint(p, &iVal); + *pVal = iVal; +} /* -** Create a new tokenizer instance. +** The xDisconnect() virtual table method. */ -static int icuCreate( - int argc, /* Number of entries in argv[] */ - const char * const *argv, /* Tokenizer creation arguments */ - sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ -){ - IcuTokenizer *p; - int n = 0; +static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int i; - if( argc>0 ){ - n = strlen(argv[0])+1; - } - p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n); - if( !p ){ - return SQLITE_NOMEM; - } - memset(p, 0, sizeof(IcuTokenizer)); + assert( p->nPendingData==0 ); + assert( p->pSegments==0 ); - if( n ){ - p->zLocale = (char *)&p[1]; - memcpy(p->zLocale, argv[0], n); + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); } + sqlite3_free(p->zSegmentsTbl); + sqlite3_free(p->zReadExprlist); + sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); + sqlite3_free(p->zLanguageid); - *ppTokenizer = (sqlite3_tokenizer *)p; + /* Invoke the tokenizer destructor to free the tokenizer. */ + p->pTokenizer->pModule->xDestroy(p->pTokenizer); + sqlite3_free(p); return SQLITE_OK; } /* -** Destroy a tokenizer +** Write an error message into *pzErr */ -static int icuDestroy(sqlite3_tokenizer *pTokenizer){ - IcuTokenizer *p = (IcuTokenizer *)pTokenizer; - sqlite3_free(p); - return SQLITE_OK; +SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ + va_list ap; + sqlite3_free(*pzErr); + va_start(ap, zFormat); + *pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); } /* -** Prepare to begin tokenizing a particular string. The input -** string to be tokenized is pInput[0..nBytes-1]. A cursor -** used to incrementally tokenize this string is returned in -** *ppCursor. +** Construct one or more SQL statements from the format string given +** and then evaluate those statements. The success code is written +** into *pRc. +** +** If *pRc is initially non-zero then this routine is a no-op. */ -static int icuOpen( - sqlite3_tokenizer *pTokenizer, /* The tokenizer */ - const char *zInput, /* Input string */ - int nInput, /* Length of zInput in bytes */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +static void fts3DbExec( + int *pRc, /* Success code */ + sqlite3 *db, /* Database in which to run SQL */ + const char *zFormat, /* Format string for SQL */ + ... /* Arguments to the format string */ ){ - IcuTokenizer *p = (IcuTokenizer *)pTokenizer; - IcuCursor *pCsr; + va_list ap; + char *zSql; + if( *pRc ) return; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +} - const int32_t opt = U_FOLD_CASE_DEFAULT; - UErrorCode status = U_ZERO_ERROR; - int nChar; +/* +** The xDestroy() virtual table method. +*/ +static int fts3DestroyMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return code */ + const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ - UChar32 c; - int iInput = 0; - int iOut = 0; + /* Drop the shadow tables */ + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); + } + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); - *ppCursor = 0; + /* If everything has worked, invoke fts3DisconnectMethod() to free the + ** memory associated with the Fts3Table structure and return SQLITE_OK. + ** Otherwise, return an SQLite error code. + */ + return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); +} - if( zInput==0 ){ - nInput = 0; - zInput = ""; - }else if( nInput<0 ){ - nInput = strlen(zInput); - } - nChar = nInput+1; - pCsr = (IcuCursor *)sqlite3_malloc( - sizeof(IcuCursor) + /* IcuCursor */ - ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ - (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ - ); - if( !pCsr ){ - return SQLITE_NOMEM; - } - memset(pCsr, 0, sizeof(IcuCursor)); - pCsr->aChar = (UChar *)&pCsr[1]; - pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; - pCsr->aOffset[iOut] = iInput; - U8_NEXT(zInput, iInput, nInput, c); - while( c>0 ){ - int isError = 0; - c = u_foldCase(c, opt); - U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); - if( isError ){ - sqlite3_free(pCsr); - return SQLITE_ERROR; +/* +** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table +** passed as the first argument. This is done as part of the xConnect() +** and xCreate() methods. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ + const char *zLanguageid; + + zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); } - pCsr->aOffset[iOut] = iInput; - if( iInputzName, zLanguageid + ); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; }else{ - c = 0; + rc = sqlite3_declare_vtab(p->db, zSql); } - } - pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); - if( !U_SUCCESS(status) ){ - sqlite3_free(pCsr); - return SQLITE_ERROR; + sqlite3_free(zSql); + sqlite3_free(zCols); + *pRc = rc; } - pCsr->nChar = iOut; - - ubrk_first(pCsr->pIter); - *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; - return SQLITE_OK; } /* -** Close a tokenization cursor previously opened by a call to icuOpen(). +** Create the %_stat table if it does not already exist. */ -static int icuClose(sqlite3_tokenizer_cursor *pCursor){ - IcuCursor *pCsr = (IcuCursor *)pCursor; - ubrk_close(pCsr->pIter); - sqlite3_free(pCsr->zBuffer); - sqlite3_free(pCsr); - return SQLITE_OK; +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; } /* -** Extract the next token from a tokenization cursor. +** Create the backing store tables (%_content, %_segments and %_segdir) +** required by the FTS3 table passed as the only argument. This is done +** as part of the vtab xCreate() method. +** +** If the p->bHasDocsize boolean is true (indicating that this is an +** FTS4 table, not an FTS3 table) then also create the %_docsize and +** %_stat tables required by FTS4. */ -static int icuNext( - sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ - const char **ppToken, /* OUT: *ppToken is the token text */ - int *pnBytes, /* OUT: Number of bytes in token */ - int *piStartOffset, /* OUT: Starting offset of token */ - int *piEndOffset, /* OUT: Ending offset of token */ - int *piPosition /* OUT: Position integer of token */ -){ - IcuCursor *pCsr = (IcuCursor *)pCursor; - - int iStart = 0; - int iEnd = 0; - int nByte = 0; +static int fts3CreateTables(Fts3Table *p){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3 *db = p->db; /* The database connection */ - while( iStart==iEnd ){ - UChar32 c; + if( p->zContentTbl==0 ){ + const char *zLanguageid = p->zLanguageid; + char *zContentCols; /* Columns of %_content table */ - iStart = ubrk_current(pCsr->pIter); - iEnd = ubrk_next(pCsr->pIter); - if( iEnd==UBRK_DONE ){ - return SQLITE_DONE; + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); } - - while( iStartaChar, iWhite, pCsr->nChar, c); - if( u_isspace(c) ){ - iStart = iWhite; - }else{ - break; - } + if( zLanguageid && zContentCols ){ + zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); } - assert(iStart<=iEnd); + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); } - do { - UErrorCode status = U_ZERO_ERROR; - if( nByte ){ - char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); - if( !zNew ){ - return SQLITE_NOMEM; - } - pCsr->zBuffer = zNew; - pCsr->nBuffer = nByte; - } - - u_strToUTF8( - pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ - &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ - &status /* Output success/failure */ + /* Create other tables */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + p->zDb, p->zName + ); + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segdir'(" + "level INTEGER," + "idx INTEGER," + "start_block INTEGER," + "leaves_end_block INTEGER," + "end_block INTEGER," + "root BLOB," + "PRIMARY KEY(level, idx)" + ");", + p->zDb, p->zName + ); + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + p->zDb, p->zName ); - } while( nByte>pCsr->nBuffer ); - - *ppToken = pCsr->zBuffer; - *pnBytes = nByte; - *piStartOffset = pCsr->aOffset[iStart]; - *piEndOffset = pCsr->aOffset[iEnd]; - *piPosition = pCsr->iToken++; - - return SQLITE_OK; + } + assert( p->bHasStat==p->bFts4 ); + if( p->bHasStat ){ + sqlite3Fts3CreateStatTable(&rc, p); + } + return rc; } /* -** The set of routines that implement the simple tokenizer -*/ -static const sqlite3_tokenizer_module icuTokenizerModule = { - 0, /* iVersion */ - icuCreate, /* xCreate */ - icuDestroy, /* xCreate */ - icuOpen, /* xOpen */ - icuClose, /* xClose */ - icuNext, /* xNext */ - 0, /* xLanguageid */ -}; - -/* -** Set *ppModule to point at the implementation of the ICU tokenizer. +** Store the current database page-size in bytes in p->nPgsz. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. */ -SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( - sqlite3_tokenizer_module const**ppModule -){ - *ppModule = &icuTokenizerModule; +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ + + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + p->nPgsz = sqlite3_column_int(pStmt, 0); + rc = sqlite3_finalize(pStmt); + }else if( rc==SQLITE_AUTH ){ + p->nPgsz = 1024; + rc = SQLITE_OK; + } + } + assert( p->nPgsz>0 || rc!=SQLITE_OK ); + sqlite3_free(zSql); + *pRc = rc; + } } -#endif /* defined(SQLITE_ENABLE_ICU) */ -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_icu.c ********************************************/ -/************** Begin file sqlite3rbu.c **************************************/ /* -** 2014 August 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** +** "Special" FTS4 arguments are column specifications of the following form: ** -** OVERVIEW +** = ** -** The RBU extension requires that the RBU update be packaged as an -** SQLite database. The tables it expects to find are described in -** sqlite3rbu.h. Essentially, for each table xyz in the target database -** that the user wishes to write to, a corresponding data_xyz table is -** created in the RBU database and populated with one row for each row to -** update, insert or delete from the target table. -** -** The update proceeds in three stages: -** -** 1) The database is updated. The modified database pages are written -** to a *-oal file. A *-oal file is just like a *-wal file, except -** that it is named "-oal" instead of "-wal". -** Because regular SQLite clients do not look for file named -** "-oal", they go on using the original database in -** rollback mode while the *-oal file is being generated. -** -** During this stage RBU does not update the database by writing -** directly to the target tables. Instead it creates "imposter" -** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses -** to update each b-tree individually. All updates required by each -** b-tree are completed before moving on to the next, and all -** updates are done in sorted key order. -** -** 2) The "-oal" file is moved to the equivalent "-wal" -** location using a call to rename(2). Before doing this the RBU -** module takes an EXCLUSIVE lock on the database file, ensuring -** that there are no other active readers. -** -** Once the EXCLUSIVE lock is released, any other database readers -** detect the new *-wal file and read the database in wal mode. At -** this point they see the new version of the database - including -** the updates made as part of the RBU update. -** -** 3) The new *-wal file is checkpointed. This proceeds in the same way -** as a regular database checkpoint, except that a single frame is -** checkpointed each time sqlite3rbu_step() is called. If the RBU -** handle is closed before the entire *-wal file is checkpointed, -** the checkpoint progress is saved in the RBU database and the -** checkpoint can be resumed by another RBU client at some point in -** the future. -** -** POTENTIAL PROBLEMS -** -** The rename() call might not be portable. And RBU is not currently -** syncing the directory after renaming the file. -** -** When state is saved, any commit to the *-oal file and the commit to -** the RBU update database are not atomic. So if the power fails at the -** wrong moment they might get out of sync. As the main database will be -** committed before the RBU update database this will likely either just -** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE -** constraint violations). -** -** If some client does modify the target database mid RBU update, or some -** other error occurs, the RBU extension will keep throwing errors. It's -** not really clear how to get out of this state. The system could just -** by delete the RBU update database and *-oal file and have the device -** download the update again and start over. -** -** At present, for an UPDATE, both the new.* and old.* records are -** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all -** fields are collected. This means we're probably writing a lot more -** data to disk when saving the state of an ongoing update to the RBU -** update database than is strictly necessary. -** +** There may not be whitespace surrounding the "=" character. The +** term may be quoted, but the may not. */ +static int fts3IsSpecialColumn( + const char *z, + int *pnKey, + char **pzValue +){ + char *zValue; + const char *zCsr = z; -/* #include */ -/* #include */ -/* #include */ + while( *zCsr!='=' ){ + if( *zCsr=='\0' ) return 0; + zCsr++; + } -/* #include "sqlite3.h" */ + *pnKey = (int)(zCsr-z); + zValue = sqlite3_mprintf("%s", &zCsr[1]); + if( zValue ){ + sqlite3Fts3Dequote(zValue); + } + *pzValue = zValue; + return 1; +} -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) -/************** Include sqlite3rbu.h in the middle of sqlite3rbu.c ***********/ -/************** Begin file sqlite3rbu.h **************************************/ /* -** 2014 August 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains the public interface for the RBU extension. +** Append the output of a printf() style formatting to an existing string. */ +static void fts3Appendf( + int *pRc, /* IN/OUT: Error code */ + char **pz, /* IN/OUT: Pointer to string buffer */ + const char *zFormat, /* Printf format string to append */ + ... /* Arguments for printf format string */ +){ + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( z && *pz ){ + char *z2 = sqlite3_mprintf("%s%s", *pz, z); + sqlite3_free(z); + z = z2; + } + if( z==0 ) *pRc = SQLITE_NOMEM; + sqlite3_free(*pz); + *pz = z; + } +} /* -** SUMMARY -** -** Writing a transaction containing a large number of operations on -** b-tree indexes that are collectively larger than the available cache -** memory can be very inefficient. -** -** The problem is that in order to update a b-tree, the leaf page (at least) -** containing the entry being inserted or deleted must be modified. If the -** working set of leaves is larger than the available cache memory, then a -** single leaf that is modified more than once as part of the transaction -** may be loaded from or written to the persistent media multiple times. -** Additionally, because the index updates are likely to be applied in -** random order, access to pages within the database is also likely to be in -** random order, which is itself quite inefficient. -** -** One way to improve the situation is to sort the operations on each index -** by index key before applying them to the b-tree. This leads to an IO -** pattern that resembles a single linear scan through the index b-tree, -** and all but guarantees each modified leaf page is loaded and stored -** exactly once. SQLite uses this trick to improve the performance of -** CREATE INDEX commands. This extension allows it to be used to improve -** the performance of large transactions on existing databases. -** -** Additionally, this extension allows the work involved in writing the -** large transaction to be broken down into sub-transactions performed -** sequentially by separate processes. This is useful if the system cannot -** guarantee that a single update process will run for long enough to apply -** the entire update, for example because the update is being applied on a -** mobile device that is frequently rebooted. Even after the writer process -** has committed one or more sub-transactions, other database clients continue -** to read from the original database snapshot. In other words, partially -** applied transactions are not visible to other clients. -** -** "RBU" stands for "Resumable Bulk Update". As in a large database update -** transmitted via a wireless network to a mobile device. A transaction -** applied using this extension is hence refered to as an "RBU update". -** -** -** LIMITATIONS -** -** An "RBU update" transaction is subject to the following limitations: -** -** * The transaction must consist of INSERT, UPDATE and DELETE operations -** only. -** -** * INSERT statements may not use any default values. -** -** * UPDATE and DELETE statements must identify their target rows by -** non-NULL PRIMARY KEY values. Rows with NULL values stored in PRIMARY -** KEY fields may not be updated or deleted. If the table being written -** has no PRIMARY KEY, affected rows must be identified by rowid. -** -** * UPDATE statements may not modify PRIMARY KEY columns. -** -** * No triggers will be fired. -** -** * No foreign key violations are detected or reported. -** -** * CHECK constraints are not enforced. -** -** * No constraint handling mode except for "OR ROLLBACK" is supported. -** -** -** PREPARATION -** -** An "RBU update" is stored as a separate SQLite database. A database -** containing an RBU update is an "RBU database". For each table in the -** target database to be updated, the RBU database should contain a table -** named "data_" containing the same set of columns as the -** target table, and one more - "rbu_control". The data_% table should -** have no PRIMARY KEY or UNIQUE constraints, but each column should have -** the same type as the corresponding column in the target database. -** The "rbu_control" column should have no type at all. For example, if -** the target database contains: -** -** CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c UNIQUE); -** -** Then the RBU database should contain: -** -** CREATE TABLE data_t1(a INTEGER, b TEXT, c, rbu_control); -** -** The order of the columns in the data_% table does not matter. -** -** Instead of a regular table, the RBU database may also contain virtual -** tables or view named using the data_ naming scheme. -** -** Instead of the plain data_ naming scheme, RBU database tables -** may also be named data_, where is any sequence -** of zero or more numeric characters (0-9). This can be significant because -** tables within the RBU database are always processed in order sorted by -** name. By judicious selection of the portion of the names -** of the RBU tables the user can therefore control the order in which they -** are processed. This can be useful, for example, to ensure that "external -** content" FTS4 tables are updated before their underlying content tables. -** -** If the target database table is a virtual table or a table that has no -** PRIMARY KEY declaration, the data_% table must also contain a column -** named "rbu_rowid". This column is mapped to the tables implicit primary -** key column - "rowid". Virtual tables for which the "rowid" column does -** not function like a primary key value cannot be updated using RBU. For -** example, if the target db contains either of the following: -** -** CREATE VIRTUAL TABLE x1 USING fts3(a, b); -** CREATE TABLE x1(a, b) -** -** then the RBU database should contain: -** -** CREATE TABLE data_x1(a, b, rbu_rowid, rbu_control); -** -** All non-hidden columns (i.e. all columns matched by "SELECT *") of the -** target table must be present in the input table. For virtual tables, -** hidden columns are optional - they are updated by RBU if present in -** the input table, or not otherwise. For example, to write to an fts4 -** table with a hidden languageid column such as: -** -** CREATE VIRTUAL TABLE ft1 USING fts4(a, b, languageid='langid'); -** -** Either of the following input table schemas may be used: -** -** CREATE TABLE data_ft1(a, b, langid, rbu_rowid, rbu_control); -** CREATE TABLE data_ft1(a, b, rbu_rowid, rbu_control); -** -** For each row to INSERT into the target database as part of the RBU -** update, the corresponding data_% table should contain a single record -** with the "rbu_control" column set to contain integer value 0. The -** other columns should be set to the values that make up the new record -** to insert. -** -** If the target database table has an INTEGER PRIMARY KEY, it is not -** possible to insert a NULL value into the IPK column. Attempting to -** do so results in an SQLITE_MISMATCH error. -** -** For each row to DELETE from the target database as part of the RBU -** update, the corresponding data_% table should contain a single record -** with the "rbu_control" column set to contain integer value 1. The -** real primary key values of the row to delete should be stored in the -** corresponding columns of the data_% table. The values stored in the -** other columns are not used. -** -** For each row to UPDATE from the target database as part of the RBU -** update, the corresponding data_% table should contain a single record -** with the "rbu_control" column set to contain a value of type text. -** The real primary key values identifying the row to update should be -** stored in the corresponding columns of the data_% table row, as should -** the new values of all columns being update. The text value in the -** "rbu_control" column must contain the same number of characters as -** there are columns in the target database table, and must consist entirely -** of 'x' and '.' characters (or in some special cases 'd' - see below). For -** each column that is being updated, the corresponding character is set to -** 'x'. For those that remain as they are, the corresponding character of the -** rbu_control value should be set to '.'. For example, given the tables -** above, the update statement: -** -** UPDATE t1 SET c = 'usa' WHERE a = 4; -** -** is represented by the data_t1 row created by: -** -** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..x'); -** -** Instead of an 'x' character, characters of the rbu_control value specified -** for UPDATEs may also be set to 'd'. In this case, instead of updating the -** target table with the value stored in the corresponding data_% column, the -** user-defined SQL function "rbu_delta()" is invoked and the result stored in -** the target table column. rbu_delta() is invoked with two arguments - the -** original value currently stored in the target table column and the -** value specified in the data_xxx table. -** -** For example, this row: -** -** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..d'); -** -** is similar to an UPDATE statement such as: -** -** UPDATE t1 SET c = rbu_delta(c, 'usa') WHERE a = 4; -** -** Finally, if an 'f' character appears in place of a 'd' or 's' in an -** ota_control string, the contents of the data_xxx table column is assumed -** to be a "fossil delta" - a patch to be applied to a blob value in the -** format used by the fossil source-code management system. In this case -** the existing value within the target database table must be of type BLOB. -** It is replaced by the result of applying the specified fossil delta to -** itself. -** -** If the target database table is a virtual table or a table with no PRIMARY -** KEY, the rbu_control value should not include a character corresponding -** to the rbu_rowid value. For example, this: -** -** INSERT INTO data_ft1(a, b, rbu_rowid, rbu_control) -** VALUES(NULL, 'usa', 12, '.x'); -** -** causes a result similar to: -** -** UPDATE ft1 SET b = 'usa' WHERE rowid = 12; -** -** The data_xxx tables themselves should have no PRIMARY KEY declarations. -** However, RBU is more efficient if reading the rows in from each data_xxx -** table in "rowid" order is roughly the same as reading them sorted by -** the PRIMARY KEY of the corresponding target database table. In other -** words, rows should be sorted using the destination table PRIMARY KEY -** fields before they are inserted into the data_xxx tables. -** -** USAGE -** -** The API declared below allows an application to apply an RBU update -** stored on disk to an existing target database. Essentially, the -** application: -** -** 1) Opens an RBU handle using the sqlite3rbu_open() function. -** -** 2) Registers any required virtual table modules with the database -** handle returned by sqlite3rbu_db(). Also, if required, register -** the rbu_delta() implementation. -** -** 3) Calls the sqlite3rbu_step() function one or more times on -** the new handle. Each call to sqlite3rbu_step() performs a single -** b-tree operation, so thousands of calls may be required to apply -** a complete update. -** -** 4) Calls sqlite3rbu_close() to close the RBU update handle. If -** sqlite3rbu_step() has been called enough times to completely -** apply the update to the target database, then the RBU database -** is marked as fully applied. Otherwise, the state of the RBU -** update application is saved in the RBU database for later -** resumption. -** -** See comments below for more detail on APIs. -** -** If an update is only partially applied to the target database by the -** time sqlite3rbu_close() is called, various state information is saved -** within the RBU database. This allows subsequent processes to automatically -** resume the RBU update from where it left off. -** -** To remove all RBU extension state information, returning an RBU database -** to its original contents, it is sufficient to drop all tables that begin -** with the prefix "rbu_" -** -** DATABASE LOCKING -** -** An RBU update may not be applied to a database in WAL mode. Attempting -** to do so is an error (SQLITE_ERROR). +** Return a copy of input string zInput enclosed in double-quotes (") and +** with all double quote characters escaped. For example: ** -** While an RBU handle is open, a SHARED lock may be held on the target -** database file. This means it is possible for other clients to read the -** database, but not to write it. +** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" ** -** If an RBU update is started and then suspended before it is completed, -** then an external client writes to the database, then attempting to resume -** the suspended RBU update is also an error (SQLITE_BUSY). +** The pointer returned points to memory obtained from sqlite3_malloc(). It +** is the callers responsibility to call sqlite3_free() to release this +** memory. */ - -#ifndef _SQLITE3RBU_H -#define _SQLITE3RBU_H - -/* #include "sqlite3.h" ** Required for error code definitions ** */ - -#if 0 -extern "C" { -#endif - -typedef struct sqlite3rbu sqlite3rbu; +static char *fts3QuoteId(char const *zInput){ + int nRet; + char *zRet; + nRet = 2 + (int)strlen(zInput)*2 + 1; + zRet = sqlite3_malloc(nRet); + if( zRet ){ + int i; + char *z = zRet; + *(z++) = '"'; + for(i=0; zInput[i]; i++){ + if( zInput[i]=='"' ) *(z++) = '"'; + *(z++) = zInput[i]; + } + *(z++) = '"'; + *(z++) = '\0'; + } + return zRet; +} /* -** Open an RBU handle. -** -** Argument zTarget is the path to the target database. Argument zRbu is -** the path to the RBU database. Each call to this function must be matched -** by a call to sqlite3rbu_close(). When opening the databases, RBU passes -** the SQLITE_CONFIG_URI flag to sqlite3_open_v2(). So if either zTarget -** or zRbu begin with "file:", it will be interpreted as an SQLite -** database URI, not a regular file name. +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: ** -** If the zState argument is passed a NULL value, the RBU extension stores -** the current state of the update (how many rows have been updated, which -** indexes are yet to be updated etc.) within the RBU database itself. This -** can be convenient, as it means that the RBU application does not need to -** organize removing a separate state file after the update is concluded. -** Or, if zState is non-NULL, it must be a path to a database file in which -** the RBU extension can store the state of the update. +** SELECT FROM %_content AS x ... ** -** When resuming an RBU update, the zState argument must be passed the same -** value as when the RBU update was started. +** to return the docid, followed by each column of text data in order +** from left to write. If parameter zFunc is not NULL, then instead of +** being returned directly each column of text data is passed to an SQL +** function named zFunc first. For example, if zFunc is "unzip" and the +** table has the three user-defined columns "a", "b", and "c", the following +** string is returned: ** -** Once the RBU update is finished, the RBU extension does not -** automatically remove any zState database file, even if it created it. +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" ** -** By default, RBU uses the default VFS to access the files on disk. To -** use a VFS other than the default, an SQLite "file:" URI containing a -** "vfs=..." option may be passed as the zTarget option. +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. ** -** IMPORTANT NOTE FOR ZIPVFS USERS: The RBU extension works with all of -** SQLite's built-in VFSs, including the multiplexor VFS. However it does -** not work out of the box with zipvfs. Refer to the comment describing -** the zipvfs_create_vfs() API below for details on using RBU with zipvfs. +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. */ -SQLITE_API sqlite3rbu *sqlite3rbu_open( - const char *zTarget, - const char *zRbu, - const char *zState -); +static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; -/* -** Open an RBU handle to perform an RBU vacuum on database file zTarget. -** An RBU vacuum is similar to SQLite's built-in VACUUM command, except -** that it can be suspended and resumed like an RBU update. -** -** The second argument to this function identifies a database in which -** to store the state of the RBU vacuum operation if it is suspended. The -** first time sqlite3rbu_vacuum() is called, to start an RBU vacuum -** operation, the state database should either not exist or be empty -** (contain no tables). If an RBU vacuum is suspended by calling -** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has -** returned SQLITE_DONE, the vacuum state is stored in the state database. -** The vacuum can be resumed by calling this function to open a new RBU -** handle specifying the same target and state databases. -** -** If the second argument passed to this function is NULL, then the -** name of the state database is "-vacuum", where -** is the name of the target database file. In this case, on UNIX, if the -** state database is not already present in the file-system, it is created -** with the same permissions as the target db is made. -** -** This function does not delete the state database after an RBU vacuum -** is completed, even if it created it. However, if the call to -** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents -** of the state tables within the state database are zeroed. This way, -** the next call to sqlite3rbu_vacuum() opens a handle that starts a -** new RBU vacuum operation. -** -** As with sqlite3rbu_open(), Zipvfs users should rever to the comment -** describing the sqlite3rbu_create_vfs() API function below for -** a description of the complications associated with using RBU with -** zipvfs databases. -*/ -SQLITE_API sqlite3rbu *sqlite3rbu_vacuum( - const char *zTarget, - const char *zState -); + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } + sqlite3_free(zFree); + }else{ + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + } + } + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); + return zRet; +} /* -** Internally, each RBU connection uses a separate SQLite database -** connection to access the target and rbu update databases. This -** API allows the application direct access to these database handles. -** -** The first argument passed to this function must be a valid, open, RBU -** handle. The second argument should be passed zero to access the target -** database handle, or non-zero to access the rbu update database handle. -** Accessing the underlying database handles may be useful in the -** following scenarios: -** -** * If any target tables are virtual tables, it may be necessary to -** call sqlite3_create_module() on the target database handle to -** register the required virtual table implementations. -** -** * If the data_xxx tables in the RBU source database are virtual -** tables, the application may need to call sqlite3_create_module() on -** the rbu update db handle to any required virtual table -** implementations. -** -** * If the application uses the "rbu_delta()" feature described above, -** it must use sqlite3_create_function() or similar to register the -** rbu_delta() implementation with the target database handle. +** Return a list of N comma separated question marks, where N is the number +** of columns in the %_content table (one for the docid plus one for each +** user-defined text column). ** -** If an error has occurred, either while opening or stepping the RBU object, -** this function may return NULL. The error code and message may be collected -** when sqlite3rbu_close() is called. +** If argument zFunc is not NULL, then all but the first question mark +** is preceded by zFunc and an open bracket, and followed by a closed +** bracket. For example, if zFunc is "zip" and the FTS3 table has three +** user-defined text columns, the following string is returned: ** -** Database handles returned by this function remain valid until the next -** call to any sqlite3rbu_xxx() function other than sqlite3rbu_db(). -*/ -SQLITE_API sqlite3 *sqlite3rbu_db(sqlite3rbu*, int bRbu); - -/* -** Do some work towards applying the RBU update to the target db. +** "?, zip(?), zip(?), zip(?)" ** -** Return SQLITE_DONE if the update has been completely applied, or -** SQLITE_OK if no error occurs but there remains work to do to apply -** the RBU update. If an error does occur, some other error code is -** returned. +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. ** -** Once a call to sqlite3rbu_step() has returned a value other than -** SQLITE_OK, all subsequent calls on the same RBU handle are no-ops -** that immediately return the same value. +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. */ -SQLITE_API int sqlite3rbu_step(sqlite3rbu *pRbu); +static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; -/* -** Force RBU to save its state to disk. -** -** If a power failure or application crash occurs during an update, following -** system recovery RBU may resume the update from the point at which the state -** was last saved. In other words, from the most recent successful call to -** sqlite3rbu_close() or this function. -** -** SQLITE_OK is returned if successful, or an SQLite error code otherwise. -*/ -SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *pRbu); + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "?"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", ?"); + } + sqlite3_free(zFree); + return zRet; +} /* -** Close an RBU handle. +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. ** -** If the RBU update has been completely applied, mark the RBU database -** as fully applied. Otherwise, assuming no error has occurred, save the -** current state of the RBU update appliation to the RBU database. +** Only decimal digits ('0'..'9') may be part of an integer value. ** -** If an error has already occurred as part of an sqlite3rbu_step() -** or sqlite3rbu_open() call, or if one occurs within this function, an -** SQLite error code is returned. Additionally, *pzErrmsg may be set to -** point to a buffer containing a utf-8 formatted English language error -** message. It is the responsibility of the caller to eventually free any -** such buffer using sqlite3_free(). +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. ** -** Otherwise, if no error occurs, this function returns SQLITE_OK if the -** update has been partially applied, or SQLITE_DONE if it has been -** completely applied. +** This function is used when parsing the "prefix=" FTS4 parameter. */ -SQLITE_API int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); +static int fts3GobbleInt(const char **pp, int *pnOut){ + const int MAX_NPREFIX = 10000000; + const char *p; /* Iterator pointer */ + int nInt = 0; /* Output value */ -/* -** Return the total number of key-value operations (inserts, deletes or -** updates) that have been performed on the target database since the -** current RBU update was started. -*/ -SQLITE_API sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ + nInt = nInt * 10 + (p[0] - '0'); + if( nInt>MAX_NPREFIX ){ + nInt = 0; + break; + } + } + if( p==*pp ) return SQLITE_ERROR; + *pnOut = nInt; + *pp = p; + return SQLITE_OK; +} /* -** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) -** progress indications for the two stages of an RBU update. This API may -** be useful for driving GUI progress indicators and similar. -** -** An RBU update is divided into two stages: -** -** * Stage 1, in which changes are accumulated in an oal/wal file, and -** * Stage 2, in which the contents of the wal file are copied into the -** main database. -** -** The update is visible to non-RBU clients during stage 2. During stage 1 -** non-RBU reader clients may see the original database. -** -** If this API is called during stage 2 of the update, output variable -** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) -** to a value between 0 and 10000 to indicate the permyriadage progress of -** stage 2. A value of 5000 indicates that stage 2 is half finished, -** 9000 indicates that it is 90% finished, and so on. -** -** If this API is called during stage 1 of the update, output variable -** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The -** value to which (*pnOne) is set depends on whether or not the RBU -** database contains an "rbu_count" table. The rbu_count table, if it -** exists, must contain the same columns as the following: +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. ** -** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. ** -** There must be one row in the table for each source (data_xxx) table within -** the RBU database. The 'tbl' column should contain the name of the source -** table. The 'cnt' column should contain the number of rows within the -** source table. +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. ** -** If the rbu_count table is present and populated correctly and this -** API is called during stage 1, the *pnOne output variable is set to the -** permyriadage progress of the same stage. If the rbu_count table does -** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count -** table exists but is not correctly populated, the value of the *pnOne -** output variable during stage 1 is undefined. +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. */ -SQLITE_API void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ +){ + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ -/* -** Obtain an indication as to the current stage of an RBU update or vacuum. -** This function always returns one of the SQLITE_RBU_STATE_XXX constants -** defined in this file. Return values should be interpreted as follows: -** -** SQLITE_RBU_STATE_OAL: -** RBU is currently building a *-oal file. The next call to sqlite3rbu_step() -** may either add further data to the *-oal file, or compute data that will -** be added by a subsequent call. -** -** SQLITE_RBU_STATE_MOVE: -** RBU has finished building the *-oal file. The next call to sqlite3rbu_step() -** will move the *-oal file to the equivalent *-wal path. If the current -** operation is an RBU update, then the updated version of the database -** file will become visible to ordinary SQLite clients following the next -** call to sqlite3rbu_step(). -** -** SQLITE_RBU_STATE_CHECKPOINT: -** RBU is currently performing an incremental checkpoint. The next call to -** sqlite3rbu_step() will copy a page of data from the *-wal file into -** the target database file. -** -** SQLITE_RBU_STATE_DONE: -** The RBU operation has finished. Any subsequent calls to sqlite3rbu_step() -** will immediately return SQLITE_DONE. -** -** SQLITE_RBU_STATE_ERROR: -** An error has occurred. Any subsequent calls to sqlite3rbu_step() will -** immediately return the SQLite error code associated with the error. -*/ -#define SQLITE_RBU_STATE_OAL 1 -#define SQLITE_RBU_STATE_MOVE 2 -#define SQLITE_RBU_STATE_CHECKPOINT 3 -#define SQLITE_RBU_STATE_DONE 4 -#define SQLITE_RBU_STATE_ERROR 5 + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); + *apIndex = aIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i=0 ); + if( nPrefix==0 ){ + nIndex--; + i--; + }else{ + aIndex[i].nPrefix = nPrefix; + } + p++; + } + } -SQLITE_API int sqlite3rbu_state(sqlite3rbu *pRbu); + *pnIndex = nIndex; + return SQLITE_OK; +} /* -** Create an RBU VFS named zName that accesses the underlying file-system -** via existing VFS zParent. Or, if the zParent parameter is passed NULL, -** then the new RBU VFS uses the default system VFS to access the file-system. -** The new object is registered as a non-default VFS with SQLite before -** returning. -** -** Part of the RBU implementation uses a custom VFS object. Usually, this -** object is created and deleted automatically by RBU. -** -** The exception is for applications that also use zipvfs. In this case, -** the custom VFS must be explicitly created by the user before the RBU -** handle is opened. The RBU VFS should be installed so that the zipvfs -** VFS uses the RBU VFS, which in turn uses any other VFS layers in use -** (for example multiplexor) to access the file-system. For example, -** to assemble an RBU enabled VFS stack that uses both zipvfs and -** multiplexor (error checking omitted): -** -** // Create a VFS named "multiplex" (not the default). -** sqlite3_multiplex_initialize(0, 0); -** -** // Create an rbu VFS named "rbu" that uses multiplexor. If the -** // second argument were replaced with NULL, the "rbu" VFS would -** // access the file-system via the system default VFS, bypassing the -** // multiplexor. -** sqlite3rbu_create_vfs("rbu", "multiplex"); +** This function is called when initializing an FTS4 table that uses the +** content=xxx option. It determines the number of and names of the columns +** of the new FTS4 table. ** -** // Create a zipvfs VFS named "zipvfs" that uses rbu. -** zipvfs_create_vfs_v3("zipvfs", "rbu", 0, xCompressorAlgorithmDetector); +** The third argument passed to this function is the value passed to the +** config=xxx option (i.e. "xxx"). This function queries the database for +** a table of that name. If found, the output variables are populated +** as follows: ** -** // Make zipvfs the default VFS. -** sqlite3_vfs_register(sqlite3_vfs_find("zipvfs"), 1); +** *pnCol: Set to the number of columns table xxx has, ** -** Because the default VFS created above includes a RBU functionality, it -** may be used by RBU clients. Attempting to use RBU with a zipvfs VFS stack -** that does not include the RBU layer results in an error. +** *pnStr: Set to the total amount of space required to store a copy +** of each columns name, including the nul-terminator. ** -** The overhead of adding the "rbu" VFS to the system is negligible for -** non-RBU users. There is no harm in an application accessing the -** file-system via "rbu" all the time, even if it only uses RBU functionality -** occasionally. -*/ -SQLITE_API int sqlite3rbu_create_vfs(const char *zName, const char *zParent); - -/* -** Deregister and destroy an RBU vfs created by an earlier call to -** sqlite3rbu_create_vfs(). +** *pazCol: Set to point to an array of *pnCol strings. Each string is +** the name of the corresponding column in table xxx. The array +** and its contents are allocated using a single allocation. It +** is the responsibility of the caller to free this allocation +** by eventually passing the *pazCol value to sqlite3_free(). ** -** VFS objects are not reference counted. If a VFS object is destroyed -** before all database handles that use it have been closed, the results -** are undefined. +** If the table cannot be found, an error code is returned and the output +** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is +** returned (and the output variables are undefined). */ -SQLITE_API void sqlite3rbu_destroy_vfs(const char *zName); - -#if 0 -} /* end of the 'extern "C"' block */ -#endif - -#endif /* _SQLITE3RBU_H */ - -/************** End of sqlite3rbu.h ******************************************/ -/************** Continuing where we left off in sqlite3rbu.c *****************/ - -#if defined(_WIN32_WCE) -/* #include "windows.h" */ -#endif +static int fts3ContentColumns( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ + const char *zTbl, /* Name of content table */ + const char ***pazCol, /* OUT: Malloc'd array of column names */ + int *pnCol, /* OUT: Size of array *pazCol */ + int *pnStr, /* OUT: Bytes of string content */ + char **pzErr /* OUT: error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* "SELECT *" statement on zTbl */ + sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ -/* Maximum number of prepared UPDATE statements held by this module */ -#define SQLITE_RBU_UPDATE_CACHESIZE 16 + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); + } + } + sqlite3_free(zSql); -/* -** Swap two objects of type TYPE. -*/ -#if !defined(SQLITE_AMALGAMATION) -# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} -#endif - -/* -** The rbu_state table is used to save the state of a partially applied -** update so that it can be resumed later. The table consists of integer -** keys mapped to values as follows: -** -** RBU_STATE_STAGE: -** May be set to integer values 1, 2, 4 or 5. As follows: -** 1: the *-rbu file is currently under construction. -** 2: the *-rbu file has been constructed, but not yet moved -** to the *-wal path. -** 4: the checkpoint is underway. -** 5: the rbu update has been checkpointed. -** -** RBU_STATE_TBL: -** Only valid if STAGE==1. The target database name of the table -** currently being written. -** -** RBU_STATE_IDX: -** Only valid if STAGE==1. The target database name of the index -** currently being written, or NULL if the main table is currently being -** updated. -** -** RBU_STATE_ROW: -** Only valid if STAGE==1. Number of rows already processed for the current -** table/index. -** -** RBU_STATE_PROGRESS: -** Trbul number of sqlite3rbu_step() calls made so far as part of this -** rbu update. -** -** RBU_STATE_CKPT: -** Valid if STAGE==4. The 64-bit checksum associated with the wal-index -** header created by recovering the *-wal file. This is used to detect -** cases when another client appends frames to the *-wal file in the -** middle of an incremental checkpoint (an incremental checkpoint cannot -** be continued if this happens). -** -** RBU_STATE_COOKIE: -** Valid if STAGE==1. The current change-counter cookie value in the -** target db file. -** -** RBU_STATE_OALSZ: -** Valid if STAGE==1. The size in bytes of the *-oal file. -*/ -#define RBU_STATE_STAGE 1 -#define RBU_STATE_TBL 2 -#define RBU_STATE_IDX 3 -#define RBU_STATE_ROW 4 -#define RBU_STATE_PROGRESS 5 -#define RBU_STATE_CKPT 6 -#define RBU_STATE_COOKIE 7 -#define RBU_STATE_OALSZ 8 -#define RBU_STATE_PHASEONESTEP 9 - -#define RBU_STAGE_OAL 1 -#define RBU_STAGE_MOVE 2 -#define RBU_STAGE_CAPTURE 3 -#define RBU_STAGE_CKPT 4 -#define RBU_STAGE_DONE 5 - - -#define RBU_CREATE_STATE \ - "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)" - -typedef struct RbuFrame RbuFrame; -typedef struct RbuObjIter RbuObjIter; -typedef struct RbuState RbuState; -typedef struct rbu_vfs rbu_vfs; -typedef struct rbu_file rbu_file; -typedef struct RbuUpdateStmt RbuUpdateStmt; - -#if !defined(SQLITE_AMALGAMATION) -typedef unsigned int u32; -typedef unsigned short u16; -typedef unsigned char u8; -typedef sqlite3_int64 i64; -#endif + if( rc==SQLITE_OK ){ + const char **azCol; /* Output array */ + int nStr = 0; /* Size of all column names (incl. 0x00) */ + int nCol; /* Number of table columns */ + int i; /* Used to iterate through columns */ -/* -** These values must match the values defined in wal.c for the equivalent -** locks. These are not magic numbers as they are part of the SQLite file -** format. -*/ -#define WAL_LOCK_WRITE 0 -#define WAL_LOCK_CKPT 1 -#define WAL_LOCK_READ0 3 + /* Loop through the returned columns. Set nStr to the number of bytes of + ** space required to store a copy of each column name, including the + ** nul-terminator byte. */ + nCol = sqlite3_column_count(pStmt); + for(i=0; i module name ("fts3" or "fts4") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. */ -struct RbuObjIter { - sqlite3_stmt *pTblIter; /* Iterate through tables */ - sqlite3_stmt *pIdxIter; /* Index iterator */ - int nTblCol; /* Size of azTblCol[] array */ - char **azTblCol; /* Array of unquoted target column names */ - char **azTblType; /* Array of target column types */ - int *aiSrcOrder; /* src table col -> target table col */ - u8 *abTblPk; /* Array of flags, set on target PK columns */ - u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ - u8 *abIndexed; /* Array of flags, set on indexed & PK cols */ - int eType; /* Table type - an RBU_PK_XXX value */ - - /* Output variables. zTbl==0 implies EOF. */ - int bCleanup; /* True in "cleanup" state */ - const char *zTbl; /* Name of target db table */ - const char *zDataTbl; /* Name of rbu db table (or null) */ - const char *zIdx; /* Name of target db index (or null) */ - int iTnum; /* Root page of current object */ - int iPkTnum; /* If eType==EXTERNAL, root of PK index */ - int bUnique; /* Current index is unique */ - int nIndex; /* Number of aux. indexes on table zTbl */ - - /* Statements created by rbuObjIterPrepareAll() */ - int nCol; /* Number of columns in current object */ - sqlite3_stmt *pSelect; /* Source data */ - sqlite3_stmt *pInsert; /* Statement for INSERT operations */ - sqlite3_stmt *pDelete; /* Statement for DELETE ops */ - sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ - - /* Last UPDATE used (for PK b-tree updates only), or NULL. */ - RbuUpdateStmt *pRbuUpdate; -}; +static int fts3InitVtab( + int isCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Table *p = 0; /* Pointer to allocated vtab */ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + int nByte; /* Size of allocation used for *p */ + int iCol; /* Column index */ + int nString = 0; /* Bytes required to hold all column names */ + int nCol = 0; /* Number of columns in the FTS table */ + char *zCsr; /* Space for holding column names */ + int nDb; /* Bytes required to hold database name */ + int nName; /* Bytes required to hold table name */ + int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ + const char **aCol; /* Array of column names */ + sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ -/* -** Values for RbuObjIter.eType -** -** 0: Table does not exist (error) -** 1: Table has an implicit rowid. -** 2: Table has an explicit IPK column. -** 3: Table has an external PK index. -** 4: Table is WITHOUT ROWID. -** 5: Table is a virtual table. -*/ -#define RBU_PK_NOTABLE 0 -#define RBU_PK_NONE 1 -#define RBU_PK_IPK 2 -#define RBU_PK_EXTERNAL 3 -#define RBU_PK_WITHOUT_ROWID 4 -#define RBU_PK_VTAB 5 + int nIndex = 0; /* Size of aIndex[] array */ + struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ + /* The results of parsing supported FTS4 key=value options: */ + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ + char *zContent = 0; /* content=? parameter (or NULL) */ + char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ + char **azNotindexed = 0; /* The set of notindexed= columns */ + int nNotindexed = 0; /* Size of azNotindexed[] array */ -/* -** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs -** one of the following operations. -*/ -#define RBU_INSERT 1 /* Insert on a main table b-tree */ -#define RBU_DELETE 2 /* Delete a row from a main table b-tree */ -#define RBU_REPLACE 3 /* Delete and then insert a row */ -#define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */ -#define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */ + assert( strlen(argv[0])==4 ); + assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) + || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) + ); -#define RBU_UPDATE 6 /* Update a row in a main table b-tree */ + nDb = (int)strlen(argv[1]) + 1; + nName = (int)strlen(argv[2]) + 1; -/* -** A single step of an incremental checkpoint - frame iWalFrame of the wal -** file should be copied to page iDbPage of the database file. -*/ -struct RbuFrame { - u32 iDbPage; - u32 iWalFrame; -}; + nByte = sizeof(const char *) * (argc-2); + aCol = (const char **)sqlite3_malloc(nByte); + if( aCol ){ + memset((void*)aCol, 0, nByte); + azNotindexed = (char **)sqlite3_malloc(nByte); + } + if( azNotindexed ){ + memset(azNotindexed, 0, nByte); + } + if( !aCol || !azNotindexed ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } -/* -** RBU handle. -** -** nPhaseOneStep: -** If the RBU database contains an rbu_count table, this value is set to -** a running estimate of the number of b-tree operations required to -** finish populating the *-oal file. This allows the sqlite3_bp_progress() -** API to calculate the permyriadage progress of populating the *-oal file -** using the formula: -** -** permyriadage = (10000 * nProgress) / nPhaseOneStep -** -** nPhaseOneStep is initialized to the sum of: -** -** nRow * (nIndex + 1) -** -** for all source tables in the RBU database, where nRow is the number -** of rows in the source table and nIndex the number of indexes on the -** corresponding target database table. -** -** This estimate is accurate if the RBU update consists entirely of -** INSERT operations. However, it is inaccurate if: -** -** * the RBU update contains any UPDATE operations. If the PK specified -** for an UPDATE operation does not exist in the target table, then -** no b-tree operations are required on index b-trees. Or if the -** specified PK does exist, then (nIndex*2) such operations are -** required (one delete and one insert on each index b-tree). -** -** * the RBU update contains any DELETE operations for which the specified -** PK does not exist. In this case no operations are required on index -** b-trees. -** -** * the RBU update contains REPLACE operations. These are similar to -** UPDATE operations. -** -** nPhaseOneStep is updated to account for the conditions above during the -** first pass of each source table. The updated nPhaseOneStep value is -** stored in the rbu_state table if the RBU update is suspended. -*/ -struct sqlite3rbu { - int eStage; /* Value of RBU_STATE_STAGE field */ - sqlite3 *dbMain; /* target database handle */ - sqlite3 *dbRbu; /* rbu database handle */ - char *zTarget; /* Path to target db */ - char *zRbu; /* Path to rbu db */ - char *zState; /* Path to state db (or NULL if zRbu) */ - char zStateDb[5]; /* Db name for state ("stat" or "main") */ - int rc; /* Value returned by last rbu_step() call */ - char *zErrmsg; /* Error message if rc!=SQLITE_OK */ - int nStep; /* Rows processed for current object */ - int nProgress; /* Rows processed for all objects */ - RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ - const char *zVfsName; /* Name of automatically created rbu vfs */ - rbu_file *pTargetFd; /* File handle open on target db */ - i64 iOalSz; - i64 nPhaseOneStep; - - /* The following state variables are used as part of the incremental - ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding - ** function rbuSetupCheckpoint() for details. */ - u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ - u32 mLock; - int nFrame; /* Entries in aFrame[] array */ - int nFrameAlloc; /* Allocated size of aFrame[] array */ - RbuFrame *aFrame; - int pgsz; - u8 *aBuf; - i64 iWalCksum; - - /* Used in RBU vacuum mode only */ - int nRbu; /* Number of RBU VFS in the stack */ - rbu_file *pRbuFd; /* Fd for main db of dbRbu */ -}; + /* Loop through all of the arguments passed by the user to the FTS3/4 + ** module (i.e. all the column names and special arguments). This loop + ** does the following: + ** + ** + Figures out the number of columns the FTSX table will have, and + ** the number of bytes of space that must be allocated to store copies + ** of the column names. + ** + ** + If there is a tokenizer specification included in the arguments, + ** initializes the tokenizer pTokenizer. + */ + for(i=3; rc==SQLITE_OK && i8 + && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3Fts3IsIdChar(z[8]) + ){ + rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); + } -/* -** Each file opened by an rbu VFS is represented by an instance of -** the following structure. -*/ -struct rbu_file { - sqlite3_file base; /* sqlite3_file methods */ - sqlite3_file *pReal; /* Underlying file handle */ - rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ - sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ + /* Check if it is an FTS4 special argument. */ + else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + struct Fts4Option { + const char *zOpt; + int nOpt; + } aFts4Opt[] = { + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 }, /* 5 -> CONTENT */ + { "languageid", 10 }, /* 6 -> LANGUAGEID */ + { "notindexed", 10 } /* 7 -> NOTINDEXED */ + }; - int openFlags; /* Flags this file was opened with */ - u32 iCookie; /* Cookie value for main db files */ - u8 iWriteVer; /* "write-version" value for main db files */ - u8 bNolock; /* True to fail EXCLUSIVE locks */ + int iOpt; + if( !zVal ){ + rc = SQLITE_NOMEM; + }else{ + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + if( iOpt==SizeofArray(aFts4Opt) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + }else{ + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; - int nShm; /* Number of entries in apShm[] array */ - char **apShm; /* Array of mmap'd *-shm regions */ - char *zDel; /* Delete this when closing file */ + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; - const char *zWal; /* Wal filename for this main db file */ - rbu_file *pWalFd; /* Wal file descriptor for this main db */ - rbu_file *pMainNext; /* Next MAIN_DB file */ -}; + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; -/* -** True for an RBU vacuum handle, or false otherwise. -*/ -#define rbuIsVacuum(p) ((p)->zTarget==0) + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) + ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; -/************************************************************************* -** The following three functions, found below: -** -** rbuDeltaGetInt() -** rbuDeltaChecksum() -** rbuDeltaApply() -** -** are lifted from the fossil source code (http://fossil-scm.org). They -** are used to implement the scalar SQL function rbu_fossil_delta(). -*/ + case 5: /* CONTENT */ + sqlite3_free(zContent); + zContent = zVal; + zVal = 0; + break; -/* -** Read bytes from *pz and convert them into a positive integer. When -** finished, leave *pz pointing to the first character past the end of -** the integer. The *pLen parameter holds the length of the string -** in *pz and is decremented once for each character in the integer. -*/ -static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ - static const signed char zValue[] = { - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, - -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, - -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, - }; - unsigned int v = 0; - int c; - unsigned char *z = (unsigned char*)*pz; - unsigned char *zStart = z; - while( (c = zValue[0x7f&*(z++)])>=0 ){ - v = (v<<6) + c; - } - z--; - *pLen -= z - zStart; - *pz = (char*)z; - return v; -} + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; -/* -** Compute a 32-bit checksum on the N-byte buffer. Return the result. -*/ -static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ - const unsigned char *z = (const unsigned char *)zIn; - unsigned sum0 = 0; - unsigned sum1 = 0; - unsigned sum2 = 0; - unsigned sum3 = 0; - while(N >= 16){ - sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); - sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); - sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); - sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); - z += 16; - N -= 16; - } - while(N >= 4){ - sum0 += z[0]; - sum1 += z[1]; - sum2 += z[2]; - sum3 += z[3]; - z += 4; - N -= 4; - } - sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); - switch(N){ - case 3: sum3 += (z[2] << 8); - case 2: sum3 += (z[1] << 16); - case 1: sum3 += (z[0] << 24); - default: ; - } - return sum3; -} - -/* -** Apply a delta. -** -** The output buffer should be big enough to hold the whole output -** file and a NUL terminator at the end. The delta_output_size() -** routine will determine this size for you. -** -** The delta string should be null-terminated. But the delta string -** may contain embedded NUL characters (if the input and output are -** binary files) so we also have to pass in the length of the delta in -** the lenDelta parameter. -** -** This function returns the size of the output file in bytes (excluding -** the final NUL terminator character). Except, if the delta string is -** malformed or intended for use with a source file other than zSrc, -** then this routine returns -1. -** -** Refer to the delta_create() documentation above for a description -** of the delta file format. -*/ -static int rbuDeltaApply( - const char *zSrc, /* The source or pattern file */ - int lenSrc, /* Length of the source file */ - const char *zDelta, /* Delta to apply to the pattern */ - int lenDelta, /* Length of the delta */ - char *zOut /* Write the output into this preallocated buffer */ -){ - unsigned int limit; - unsigned int total = 0; -#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST - char *zOrigOut = zOut; -#endif - - limit = rbuDeltaGetInt(&zDelta, &lenDelta); - if( *zDelta!='\n' ){ - /* ERROR: size integer not terminated by "\n" */ - return -1; - } - zDelta++; lenDelta--; - while( *zDelta && lenDelta>0 ){ - unsigned int cnt, ofst; - cnt = rbuDeltaGetInt(&zDelta, &lenDelta); - switch( zDelta[0] ){ - case '@': { - zDelta++; lenDelta--; - ofst = rbuDeltaGetInt(&zDelta, &lenDelta); - if( lenDelta>0 && zDelta[0]!=',' ){ - /* ERROR: copy command not terminated by ',' */ - return -1; - } - zDelta++; lenDelta--; - total += cnt; - if( total>limit ){ - /* ERROR: copy exceeds output file size */ - return -1; - } - if( (int)(ofst+cnt) > lenSrc ){ - /* ERROR: copy extends past end of input */ - return -1; - } - memcpy(zOut, &zSrc[ofst], cnt); - zOut += cnt; - break; - } - case ':': { - zDelta++; lenDelta--; - total += cnt; - if( total>limit ){ - /* ERROR: insert command gives an output larger than predicted */ - return -1; - } - if( (int)cnt>lenDelta ){ - /* ERROR: insert count exceeds size of delta */ - return -1; - } - memcpy(zOut, zDelta, cnt); - zOut += cnt; - zDelta += cnt; - lenDelta -= cnt; - break; - } - case ';': { - zDelta++; lenDelta--; - zOut[0] = 0; -#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST - if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ - /* ERROR: bad checksum */ - return -1; - } -#endif - if( total!=limit ){ - /* ERROR: generated size does not match predicted size */ - return -1; + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; + } } - return total; + sqlite3_free(zVal); } - default: { - /* ERROR: unknown delta operator */ - return -1; + } + + /* Otherwise, the argument is a column name. */ + else { + nString += (int)(strlen(z) + 1); + aCol[nCol++] = z; + } + } + + /* If a content=xxx option was specified, the following: + ** + ** 1. Ignore any compress= and uncompress= options. + ** + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. + */ + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + if( nCol==0 ){ + sqlite3_free((void*)aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); + + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; jdb = db; + p->nColumn = nCol; + p->nPendingData = 0; + p->azColumn = (char **)&p[1]; + p->pTokenizer = pTokenizer; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; + p->bHasDocsize = (isFts4 && bNoDocsize==0); + p->bHasStat = isFts4; + p->bFts4 = isFts4; + p->bDescIdx = bDescIdx; + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->zContentTbl = zContent; + p->zLanguageid = zLanguageid; + zContent = 0; + zLanguageid = 0; + TESTONLY( p->inTransaction = -1 ); + TESTONLY( p->mxSavepoint = -1 ); - assert( argc==2 ); + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } + p->abNotindexed = (u8 *)&p->aIndex[nIndex]; - nOrig = sqlite3_value_bytes(argv[0]); - aOrig = (const char*)sqlite3_value_blob(argv[0]); - nDelta = sqlite3_value_bytes(argv[1]); - aDelta = (const char*)sqlite3_value_blob(argv[1]); + /* Fill in the zName and zDb fields of the vtab structure. */ + zCsr = (char *)&p->abNotindexed[nCol]; + p->zName = zCsr; + memcpy(zCsr, argv[2], nName); + zCsr += nName; + p->zDb = zCsr; + memcpy(zCsr, argv[1], nDb); + zCsr += nDb; - /* Figure out the size of the output */ - nOut = rbuDeltaOutputSize(aDelta, nDelta); - if( nOut<0 ){ - sqlite3_result_error(context, "corrupt fossil delta", -1); - return; + /* Fill in the azColumn array */ + for(iCol=0; iColazColumn[iCol] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); } - aOut = sqlite3_malloc(nOut+1); - if( aOut==0 ){ - sqlite3_result_error_nomem(context); - }else{ - nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut); - if( nOut2!=nOut ){ - sqlite3_result_error(context, "corrupt fossil delta", -1); - }else{ - sqlite3_result_blob(context, aOut, nOut, sqlite3_free); + /* Fill in the abNotindexed array */ + for(iCol=0; iColazColumn[iCol]); + for(i=0; iazColumn[iCol], zNot, n) + ){ + p->abNotindexed[iCol] = 1; + sqlite3_free(zNot); + azNotindexed[i] = 0; + } + } + } + for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); + p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); + if( rc!=SQLITE_OK ) goto fts3_init_out; -/* -** Prepare the SQL statement in buffer zSql against database handle db. -** If successful, set *ppStmt to point to the new statement and return -** SQLITE_OK. -** -** Otherwise, if an error does occur, set *ppStmt to NULL and return -** an SQLite error code. Additionally, set output variable *pzErrmsg to -** point to a buffer containing an error message. It is the responsibility -** of the caller to (eventually) free this buffer using sqlite3_free(). -*/ -static int prepareAndCollectError( - sqlite3 *db, - sqlite3_stmt **ppStmt, - char **pzErrmsg, - const char *zSql -){ - int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); - if( rc!=SQLITE_OK ){ - *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - *ppStmt = 0; + /* If this is an xCreate call, create the underlying tables in the + ** database. TODO: For xConnect(), it could verify that said tables exist. + */ + if( isCreate ){ + rc = fts3CreateTables(p); } - return rc; -} -/* -** Reset the SQL statement passed as the first argument. Return a copy -** of the value returned by sqlite3_reset(). -** -** If an error has occurred, then set *pzErrmsg to point to a buffer -** containing an error message. It is the responsibility of the caller -** to eventually free this buffer using sqlite3_free(). -*/ -static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ - int rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ){ - *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); + /* Check to see if a legacy fts3 table has been "upgraded" by the + ** addition of a %_stat table so that it can use incremental merge. + */ + if( !isFts4 && !isCreate ){ + p->bHasStat = 2; } - return rc; -} -/* -** Unless it is NULL, argument zSql points to a buffer allocated using -** sqlite3_malloc containing an SQL statement. This function prepares the SQL -** statement against database db and frees the buffer. If statement -** compilation is successful, *ppStmt is set to point to the new statement -** handle and SQLITE_OK is returned. -** -** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code -** returned. In this case, *pzErrmsg may also be set to point to an error -** message. It is the responsibility of the caller to free this error message -** buffer using sqlite3_free(). -** -** If argument zSql is NULL, this function assumes that an OOM has occurred. -** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. -*/ -static int prepareFreeAndCollectError( - sqlite3 *db, - sqlite3_stmt **ppStmt, - char **pzErrmsg, - char *zSql -){ - int rc; - assert( *pzErrmsg==0 ); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - *ppStmt = 0; + /* Figure out the page-size for the database. This is required in order to + ** estimate the cost of loading large doclists from the database. */ + fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; + + /* Declare the table schema to SQLite. */ + fts3DeclareVtab(&rc, p); + +fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aIndex); + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + sqlite3_free(zContent); + sqlite3_free(zLanguageid); + for(i=0; ipModule->xDestroy(pTokenizer); + } }else{ - rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql); - sqlite3_free(zSql); + assert( p->pSegments==0 ); + *ppVTab = &p->base; } return rc; } /* -** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated -** by an earlier call to rbuObjIterCacheTableInfo(). +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts3InitVtab(). */ -static void rbuObjIterFreeCols(RbuObjIter *pIter){ - int i; - for(i=0; inTblCol; i++){ - sqlite3_free(pIter->azTblCol[i]); - sqlite3_free(pIter->azTblType[i]); - } - sqlite3_free(pIter->azTblCol); - pIter->azTblCol = 0; - pIter->azTblType = 0; - pIter->aiSrcOrder = 0; - pIter->abTblPk = 0; - pIter->abNotNull = 0; - pIter->nTblCol = 0; - pIter->eType = 0; /* Invalid value */ -} - -/* -** Finalize all statements and free all allocations that are specific to -** the current object (table/index pair). -*/ -static void rbuObjIterClearStatements(RbuObjIter *pIter){ - RbuUpdateStmt *pUp; - - sqlite3_finalize(pIter->pSelect); - sqlite3_finalize(pIter->pInsert); - sqlite3_finalize(pIter->pDelete); - sqlite3_finalize(pIter->pTmpInsert); - pUp = pIter->pRbuUpdate; - while( pUp ){ - RbuUpdateStmt *pTmp = pUp->pNext; - sqlite3_finalize(pUp->pUpdate); - sqlite3_free(pUp); - pUp = pTmp; - } - - pIter->pSelect = 0; - pIter->pInsert = 0; - pIter->pDelete = 0; - pIter->pRbuUpdate = 0; - pIter->pTmpInsert = 0; - pIter->nCol = 0; +static int fts3ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts3CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); } /* -** Clean up any resources allocated as part of the iterator object passed -** as the only argument. +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. */ -static void rbuObjIterFinalize(RbuObjIter *pIter){ - rbuObjIterClearStatements(pIter); - sqlite3_finalize(pIter->pTblIter); - sqlite3_finalize(pIter->pIdxIter); - rbuObjIterFreeCols(pIter); - memset(pIter, 0, sizeof(RbuObjIter)); +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif } -/* -** Advance the iterator to the next position. +/* +** Implementation of the xBestIndex method for FTS3 tables. There +** are three possible strategies, in order of preference: ** -** If no error occurs, SQLITE_OK is returned and the iterator is left -** pointing to the next entry. Otherwise, an error code and message is -** left in the RBU handle passed as the first argument. A copy of the -** error code is returned. +** 1. Direct lookup by rowid or docid. +** 2. Full-text search using a MATCH operator on a non-docid column. +** 3. Linear scan of %_content table. */ -static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ - int rc = p->rc; - if( rc==SQLITE_OK ){ +static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts3Table *p = (Fts3Table *)pVTab; + int i; /* Iterator variable */ + int iCons = -1; /* Index of constraint to use */ - /* Free any SQLite statements used while processing the previous object */ - rbuObjIterClearStatements(pIter); - if( pIter->zIdx==0 ){ - rc = sqlite3_exec(p->dbMain, - "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;" - "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;" - "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;" - "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;" - , 0, 0, &p->zErrmsg - ); - } + int iLangidCons = -1; /* Index of langid=x constraint, if present */ + int iDocidGe = -1; /* Index of docid>=x constraint, if present */ + int iDocidLe = -1; /* Index of docid<=x constraint, if present */ + int iIdx; - if( rc==SQLITE_OK ){ - if( pIter->bCleanup ){ - rbuObjIterFreeCols(pIter); - pIter->bCleanup = 0; - rc = sqlite3_step(pIter->pTblIter); - if( rc!=SQLITE_ROW ){ - rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); - pIter->zTbl = 0; - }else{ - pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); - pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); - rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; - } - }else{ - if( pIter->zIdx==0 ){ - sqlite3_stmt *pIdx = pIter->pIdxIter; - rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_step(pIter->pIdxIter); - if( rc!=SQLITE_ROW ){ - rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg); - pIter->bCleanup = 1; - pIter->zIdx = 0; - }else{ - pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); - pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1); - pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); - rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM; - } - } + /* By default use a full table scan. This is an expensive option, + ** so search through the constraints to see if a more efficient + ** strategy is possible. + */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 5000000; + for(i=0; inConstraint; i++){ + int bDocid; /* True if this constraint is on docid */ + struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; } + continue; } - } - - if( rc!=SQLITE_OK ){ - rbuObjIterFinalize(pIter); - p->rc = rc; - } - return rc; -} + bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); -/* -** The implementation of the rbu_target_name() SQL function. This function -** accepts one or two arguments. The first argument is the name of a table - -** the name of a table in the RBU database. The second, if it is present, is 1 -** for a view or 0 for a table. -** -** For a non-vacuum RBU handle, if the table name matches the pattern: -** -** data[0-9]_ -** -** where is any sequence of 1 or more characters, is returned. -** Otherwise, if the only argument does not match the above pattern, an SQL -** NULL is returned. -** -** "data_t1" -> "t1" -** "data0123_t2" -> "t2" -** "dataAB_t3" -> NULL -** -** For an rbu vacuum handle, a copy of the first argument is returned if -** the second argument is either missing or 0 (not a view). -*/ -static void rbuTargetNameFunc( - sqlite3_context *pCtx, - int argc, - sqlite3_value **argv -){ - sqlite3rbu *p = sqlite3_user_data(pCtx); - const char *zIn; - assert( argc==1 || argc==2 ); + /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ + if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ + pInfo->idxNum = FTS3_DOCID_SEARCH; + pInfo->estimatedCost = 1.0; + iCons = i; + } - zIn = (const char*)sqlite3_value_text(argv[0]); - if( zIn ){ - if( rbuIsVacuum(p) ){ - if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ - sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); - } - }else{ - if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ - int i; - for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); - if( zIn[i]=='_' && zIn[i+1] ){ - sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); - } - } + /* A MATCH constraint. Use a full-text search. + ** + ** If there is more than one MATCH constraint available, use the first + ** one encountered. If there is both a MATCH constraint and a direct + ** rowid/docid lookup, prefer the MATCH strategy. This is done even + ** though the rowid/docid lookup is faster than a MATCH query, selecting + ** it would lead to an "unable to use function MATCH in the requested + ** context" error. + */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn + ){ + pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; + pInfo->estimatedCost = 2.0; + iCons = i; } - } -} -/* -** Initialize the iterator structure passed as the second argument. -** -** If no error occurs, SQLITE_OK is returned and the iterator is left -** pointing to the first entry. Otherwise, an error code and message is -** left in the RBU handle passed as the first argument. A copy of the -** error code is returned. -*/ -static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ - int rc; - memset(pIter, 0, sizeof(RbuObjIter)); + /* Equality constraint on the langid column */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && pCons->iColumn==p->nColumn + 2 + ){ + iLangidCons = i; + } - rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, - sqlite3_mprintf( - "SELECT rbu_target_name(name, type='view') AS target, name " - "FROM sqlite_master " - "WHERE type IN ('table', 'view') AND target IS NOT NULL " - " %s " - "ORDER BY name" - , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : "")); + if( bDocid ){ + switch( pCons->op ){ + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: + iDocidGe = i; + break; - if( rc==SQLITE_OK ){ - rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, - "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " - " FROM main.sqlite_master " - " WHERE type='index' AND tbl_name = ?" - ); + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: + iDocidLe = i; + break; + } + } } - pIter->bCleanup = 1; - p->rc = rc; - return rbuObjIterNext(p, pIter); -} - -/* -** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs, -** an error code is stored in the RBU handle passed as the first argument. -** -** If an error has already occurred (p->rc is already set to something other -** than SQLITE_OK), then this function returns NULL without modifying the -** stored error code. In this case it still calls sqlite3_free() on any -** printf() parameters associated with %z conversions. -*/ -static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){ - char *zSql = 0; - va_list ap; - va_start(ap, zFmt); - zSql = sqlite3_vmprintf(zFmt, ap); - if( p->rc==SQLITE_OK ){ - if( zSql==0 ) p->rc = SQLITE_NOMEM; - }else{ - sqlite3_free(zSql); - zSql = 0; - } - va_end(ap); - return zSql; -} + iIdx = 1; + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; + pInfo->aConstraintUsage[iCons].omit = 1; + } + if( iLangidCons>=0 ){ + pInfo->idxNum |= FTS3_HAVE_LANGID; + pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; + } + if( iDocidGe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_GE; + pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; + } + if( iDocidLe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_LE; + pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; + } -/* -** Argument zFmt is a sqlite3_mprintf() style format string. The trailing -** arguments are the usual subsitution values. This function performs -** the printf() style substitutions and executes the result as an SQL -** statement on the RBU handles database. -** -** If an error occurs, an error code and error message is stored in the -** RBU handle. If an error has already occurred when this function is -** called, it is a no-op. -*/ -static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){ - va_list ap; - char *zSql; - va_start(ap, zFmt); - zSql = sqlite3_vmprintf(zFmt, ap); - if( p->rc==SQLITE_OK ){ - if( zSql==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg); + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. + */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + pInfo->orderByConsumed = 1; } } - sqlite3_free(zSql); - va_end(ap); - return p->rc; -} -/* -** Attempt to allocate and return a pointer to a zeroed block of nByte -** bytes. -** -** If an error (i.e. an OOM condition) occurs, return NULL and leave an -** error code in the rbu handle passed as the first argument. Or, if an -** error has already occurred when this function is called, return NULL -** immediately without attempting the allocation or modifying the stored -** error code. -*/ -static void *rbuMalloc(sqlite3rbu *p, int nByte){ - void *pRet = 0; - if( p->rc==SQLITE_OK ){ - assert( nByte>0 ); - pRet = sqlite3_malloc64(nByte); - if( pRet==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - memset(pRet, 0, nByte); - } - } - return pRet; + assert( p->pSegments==0 ); + return SQLITE_OK; } - /* -** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that -** there is room for at least nCol elements. If an OOM occurs, store an -** error code in the RBU handle passed as the first argument. +** Implementation of xOpen method. */ -static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ - int nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; - char **azNew; +static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ - azNew = (char**)rbuMalloc(p, nByte); - if( azNew ){ - pIter->azTblCol = azNew; - pIter->azTblType = &azNew[nCol]; - pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; - pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol]; - pIter->abNotNull = (u8*)&pIter->abTblPk[nCol]; - pIter->abIndexed = (u8*)&pIter->abNotNull[nCol]; + UNUSED_PARAMETER(pVTab); + + /* Allocate a buffer large enough for an Fts3Cursor structure. If the + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** if the allocation fails, return SQLITE_NOMEM. + */ + *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); + if( !pCsr ){ + return SQLITE_NOMEM; } + memset(pCsr, 0, sizeof(Fts3Cursor)); + return SQLITE_OK; } /* -** The first argument must be a nul-terminated string. This function -** returns a copy of the string in memory obtained from sqlite3_malloc(). -** It is the responsibility of the caller to eventually free this memory -** using sqlite3_free(). -** -** If an OOM condition is encountered when attempting to allocate memory, -** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise, -** if the allocation succeeds, (*pRc) is left unchanged. +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. */ -static char *rbuStrndup(const char *zStr, int *pRc){ - char *zRet = 0; - - assert( *pRc==SQLITE_OK ); - if( zStr ){ - size_t nCopy = strlen(zStr) + 1; - zRet = (char*)sqlite3_malloc64(nCopy); - if( zRet ){ - memcpy(zRet, zStr, nCopy); - }else{ - *pRc = SQLITE_NOMEM; - } - } - - return zRet; +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_finalize(pCsr->pStmt); + sqlite3Fts3ExprFree(pCsr->pExpr); + sqlite3Fts3FreeDeferredTokens(pCsr); + sqlite3_free(pCsr->aDoclist); + sqlite3_free(pCsr->aMatchinfo); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_free(pCsr); + return SQLITE_OK; } /* -** Finalize the statement passed as the second argument. +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: ** -** If the sqlite3_finalize() call indicates that an error occurs, and the -** rbu handle error code is not already set, set the error code and error -** message accordingly. -*/ -static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){ - sqlite3 *db = sqlite3_db_handle(pStmt); - int rc = sqlite3_finalize(pStmt); - if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ - p->rc = rc; - p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - } -} - -/* Determine the type of a table. -** -** peType is of type (int*), a pointer to an output parameter of type -** (int). This call sets the output parameter as follows, depending -** on the type of the table specified by parameters dbName and zTbl. -** -** RBU_PK_NOTABLE: No such table. -** RBU_PK_NONE: Table has an implicit rowid. -** RBU_PK_IPK: Table has an explicit IPK column. -** RBU_PK_EXTERNAL: Table has an external PK index. -** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID. -** RBU_PK_VTAB: Table is a virtual table. -** -** Argument *piPk is also of type (int*), and also points to an output -** parameter. Unless the table has an external primary key index -** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, -** if the table does have an external primary key index, then *piPk -** is set to the root page number of the primary key index before -** returning. -** -** ALGORITHM: -** -** if( no entry exists in sqlite_master ){ -** return RBU_PK_NOTABLE -** }else if( sql for the entry starts with "CREATE VIRTUAL" ){ -** return RBU_PK_VTAB -** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){ -** if( the index that is the pk exists in sqlite_master ){ -** *piPK = rootpage of that index. -** return RBU_PK_EXTERNAL -** }else{ -** return RBU_PK_WITHOUT_ROWID -** } -** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){ -** return RBU_PK_IPK -** }else{ -** return RBU_PK_NONE -** } +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. +** +** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. */ -static void rbuTableType( - sqlite3rbu *p, - const char *zTab, - int *peType, - int *piTnum, - int *piPk -){ - /* - ** 0) SELECT count(*) FROM sqlite_master where name=%Q AND IsVirtual(%Q) - ** 1) PRAGMA index_list = ? - ** 2) SELECT count(*) FROM sqlite_master where name=%Q - ** 3) PRAGMA table_info = ? - */ - sqlite3_stmt *aStmt[4] = {0, 0, 0, 0}; - - *peType = RBU_PK_NOTABLE; - *piPk = 0; - - assert( p->rc==SQLITE_OK ); - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg, - sqlite3_mprintf( - "SELECT (sql LIKE 'create virtual%%'), rootpage" - " FROM sqlite_master" - " WHERE name=%Q", zTab - )); - if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){ - /* Either an error, or no such table. */ - goto rbuTableType_end; - } - if( sqlite3_column_int(aStmt[0], 0) ){ - *peType = RBU_PK_VTAB; /* virtual table */ - goto rbuTableType_end; - } - *piTnum = sqlite3_column_int(aStmt[0], 1); - - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg, - sqlite3_mprintf("PRAGMA index_list=%Q",zTab) - ); - if( p->rc ) goto rbuTableType_end; - while( sqlite3_step(aStmt[1])==SQLITE_ROW ){ - const u8 *zOrig = sqlite3_column_text(aStmt[1], 3); - const u8 *zIdx = sqlite3_column_text(aStmt[1], 1); - if( zOrig && zIdx && zOrig[0]=='p' ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg, - sqlite3_mprintf( - "SELECT rootpage FROM sqlite_master WHERE name = %Q", zIdx - )); - if( p->rc==SQLITE_OK ){ - if( sqlite3_step(aStmt[2])==SQLITE_ROW ){ - *piPk = sqlite3_column_int(aStmt[2], 0); - *peType = RBU_PK_EXTERNAL; - }else{ - *peType = RBU_PK_WITHOUT_ROWID; - } - } - goto rbuTableType_end; - } - } - - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg, - sqlite3_mprintf("PRAGMA table_info=%Q",zTab) - ); - if( p->rc==SQLITE_OK ){ - while( sqlite3_step(aStmt[3])==SQLITE_ROW ){ - if( sqlite3_column_int(aStmt[3],5)>0 ){ - *peType = RBU_PK_IPK; /* explicit IPK column */ - goto rbuTableType_end; - } - } - *peType = RBU_PK_NONE; - } - -rbuTableType_end: { - unsigned int i; - for(i=0; ipStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); } + *ppStmt = pCsr->pStmt; + return rc; } /* -** This is a helper function for rbuObjIterCacheTableInfo(). It populates -** the pIter->abIndexed[] array. +** Position the pCsr->pStmt statement so that it is on the row +** of the %_content table that contains the last match. Return +** SQLITE_OK on success. */ -static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ - sqlite3_stmt *pList = 0; - int bIndex = 0; - - if( p->rc==SQLITE_OK ){ - memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol); - p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) - ); - } +static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->isRequireSeek ){ + sqlite3_stmt *pStmt = 0; - pIter->nIndex = 0; - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ - const char *zIdx = (const char*)sqlite3_column_text(pList, 1); - sqlite3_stmt *pXInfo = 0; - if( zIdx==0 ) break; - p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) - ); - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ - int iCid = sqlite3_column_int(pXInfo, 1); - if( iCid>=0 ) pIter->abIndexed[iCid] = 1; + rc = fts3CursorSeekStmt(pCsr, &pStmt); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + return SQLITE_OK; + }else{ + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ + /* If no row was found and no error has occurred, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. */ + rc = FTS_CORRUPT_VTAB; + pCsr->isEof = 1; + } + } } - rbuFinalize(p, pXInfo); - bIndex = 1; - pIter->nIndex++; } - if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ - /* "PRAGMA index_list" includes the main PK b-tree */ - pIter->nIndex--; + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); } - - rbuFinalize(p, pList); - if( bIndex==0 ) pIter->abIndexed = 0; + return rc; } - /* -** If they are not already populated, populate the pIter->azTblCol[], -** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to -** the table (not index) that the iterator currently points to. +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. +** +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. +** +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. ** -** Return SQLITE_OK if successful, or an SQLite error code otherwise. If -** an error does occur, an error code and error message are also left in -** the RBU handle. +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. */ -static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ - if( pIter->azTblCol==0 ){ - sqlite3_stmt *pStmt = 0; - int nCol = 0; - int i; /* for() loop iterator variable */ - int bRbuRowid = 0; /* If input table has column "rbu_rowid" */ - int iOrder = 0; - int iTnum = 0; - - /* Figure out the type of table this step will deal with. */ - assert( pIter->eType==0 ); - rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum); - if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl); - } - if( p->rc ) return p->rc; - if( pIter->zIdx==0 ) pIter->iTnum = iTnum; - - assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK - || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID - || pIter->eType==RBU_PK_VTAB - ); +static int fts3ScanInteriorNode( + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + int nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ - /* Populate the azTblCol[] and nTblCol variables based on the columns - ** of the input table. Ignore any input table columns that begin with - ** "rbu_". */ - p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) - ); - if( p->rc==SQLITE_OK ){ - nCol = sqlite3_column_count(pStmt); - rbuAllocateIterArrays(p, pIter, nCol); + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. + ** + ** Even if the data structure on disk is corrupted, this (reading two + ** varints from the buffer) does not risk an overread. If zNode is a + ** root node, then the buffer comes from a SELECT statement. SQLite does + ** not make this guarantee explicitly, but in practice there are always + ** either more than 20 bytes of allocated space following the nNode bytes of + ** contents, or two zero bytes. Or, if the node is read from the %_segments + ** table, then there are always 20 bytes of zeroed padding following the + ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + */ + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + if( zCsr>zEnd ){ + return FTS_CORRUPT_VTAB; + } + + while( zCsrrc==SQLITE_OK && irc); - pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol; - pIter->azTblCol[pIter->nTblCol++] = zCopy; - } - else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){ - bRbuRowid = 1; + isFirstTerm = 0; + zCsr += fts3GetVarint32(zCsr, &nSuffix); + + if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + if( nPrefix+nSuffix>nAlloc ){ + char *zNew; + nAlloc = (nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); + if( !zNew ){ + rc = SQLITE_NOMEM; + goto finish_scan; } + zBuffer = zNew; } - sqlite3_finalize(pStmt); - pStmt = 0; + assert( zBuffer ); + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; - if( p->rc==SQLITE_OK - && rbuIsVacuum(p)==0 - && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) - ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf( - "table %q %s rbu_rowid column", pIter->zDataTbl, - (bRbuRowid ? "may not have" : "requires") - ); + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = iChild; + piFirst = 0; } - /* Check that all non-HIDDEN columns in the destination table are also - ** present in the input table. Populate the abTblPk[], azTblType[] and - ** aiTblOrder[] arrays at the same time. */ - if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, - sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl) - ); + if( piLast && cmp<0 ){ + *piLast = iChild; + piLast = 0; } - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zName = (const char*)sqlite3_column_text(pStmt, 1); - if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */ - for(i=iOrder; inTblCol; i++){ - if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; - } - if( i==pIter->nTblCol ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("column missing from %q: %s", - pIter->zDataTbl, zName - ); - }else{ - int iPk = sqlite3_column_int(pStmt, 5); - int bNotNull = sqlite3_column_int(pStmt, 3); - const char *zType = (const char*)sqlite3_column_text(pStmt, 2); - - if( i!=iOrder ){ - SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]); - SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]); - } - pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc); - pIter->abTblPk[iOrder] = (iPk!=0); - pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0); - iOrder++; - } - } + iChild++; + }; - rbuFinalize(p, pStmt); - rbuObjIterCacheIndexedCols(p, pIter); - assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); - assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 ); - } + if( piFirst ) *piFirst = iChild; + if( piLast ) *piLast = iChild; - return p->rc; + finish_scan: + sqlite3_free(zBuffer); + return rc; } -/* -** This function constructs and returns a pointer to a nul-terminated -** string containing some SQL clause or list based on one or more of the -** column names currently stored in the pIter->azTblCol[] array. -*/ -static char *rbuObjIterGetCollist( - sqlite3rbu *p, /* RBU object */ - RbuObjIter *pIter /* Object iterator for column names */ -){ - char *zList = 0; - const char *zSep = ""; - int i; - for(i=0; inTblCol; i++){ - const char *z = pIter->azTblCol[i]; - zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z); - zSep = ", "; - } - return zList; -} /* -** This function is used to create a SELECT list (the list of SQL -** expressions that follows a SELECT keyword) for a SELECT statement -** used to read from an data_xxx or rbu_tmp_xxx table while updating the -** index object currently indicated by the iterator object passed as the -** second argument. A "PRAGMA index_xinfo = " statement is used -** to obtain the required information. -** -** If the index is of the following form: -** -** CREATE INDEX i1 ON t1(c, b COLLATE nocase); -** -** and "t1" is a table with an explicit INTEGER PRIMARY KEY column -** "ipk", the returned string is: +** The buffer pointed to by argument zNode (size nNode bytes) contains an +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) +** contains a term. This function searches the sub-tree headed by the zNode +** node for the range of leaf nodes that may contain the specified term +** or terms for which the specified term is a prefix. ** -** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'" +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** left-most leaf node in the tree that may contain the specified term. +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the +** right-most leaf node that may contain a term for which the specified +** term is a prefix. ** -** As well as the returned string, three other malloc'd strings are -** returned via output parameters. As follows: +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the +** segment does contain any such terms, they are stored within the identified +** range. Because this function only inspects interior segment nodes (and +** never loads leaf nodes into memory), it is not possible to be sure. ** -** pzImposterCols: ... -** pzImposterPk: ... -** pzWhere: ... -*/ -static char *rbuObjIterGetIndexCols( - sqlite3rbu *p, /* RBU object */ - RbuObjIter *pIter, /* Object iterator for column names */ - char **pzImposterCols, /* OUT: Columns for imposter table */ - char **pzImposterPk, /* OUT: Imposter PK clause */ - char **pzWhere, /* OUT: WHERE clause */ - int *pnBind /* OUT: Trbul number of columns */ +** If an error occurs, an error code other than SQLITE_OK is returned. +*/ +static int fts3SelectLeaf( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ ){ - int rc = p->rc; /* Error code */ - int rc2; /* sqlite3_finalize() return code */ - char *zRet = 0; /* String to return */ - char *zImpCols = 0; /* String to return via *pzImposterCols */ - char *zImpPK = 0; /* String to return via *pzImposterPK */ - char *zWhere = 0; /* String to return via *pzWhere */ - int nBind = 0; /* Value to return via *pnBind */ - const char *zCom = ""; /* Set to ", " later on */ - const char *zAnd = ""; /* Set to " AND " later on */ - sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */ + int rc = SQLITE_OK; /* Return code */ + int iHeight; /* Height of this node in tree */ - if( rc==SQLITE_OK ){ - assert( p->zErrmsg==0 ); - rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) - ); - } + assert( piLeaf || piLeaf2 ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ - int iCid = sqlite3_column_int(pXInfo, 1); - int bDesc = sqlite3_column_int(pXInfo, 3); - const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); - const char *zCol; - const char *zType; + fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); - if( iCid<0 ){ - /* An integer primary key. If the table has an explicit IPK, use - ** its name. Otherwise, use "rbu_rowid". */ - if( pIter->eType==RBU_PK_IPK ){ - int i; - for(i=0; pIter->abTblPk[i]==0; i++); - assert( inTblCol ); - zCol = pIter->azTblCol[i]; - }else if( rbuIsVacuum(p) ){ - zCol = "_rowid_"; - }else{ - zCol = "rbu_rowid"; + if( rc==SQLITE_OK && iHeight>1 ){ + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob = 0; /* Size of zBlob in bytes */ + + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); } - zType = "INTEGER"; - }else{ - zCol = pIter->azTblCol[iCid]; - zType = pIter->azTblType[iCid]; + sqlite3_free(zBlob); + piLeaf = 0; + zBlob = 0; } - zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate); - if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ - const char *zOrder = (bDesc ? " DESC" : ""); - zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s", - zImpPK, zCom, nBind, zCol, zOrder - ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); } - zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q", - zImpCols, zCom, nBind, zCol, zType, zCollate - ); - zWhere = sqlite3_mprintf( - "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol - ); - if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM; - zCom = ", "; - zAnd = " AND "; - nBind++; - } - - rc2 = sqlite3_finalize(pXInfo); - if( rc==SQLITE_OK ) rc = rc2; - - if( rc!=SQLITE_OK ){ - sqlite3_free(zRet); - sqlite3_free(zImpCols); - sqlite3_free(zImpPK); - sqlite3_free(zWhere); - zRet = 0; - zImpCols = 0; - zImpPK = 0; - zWhere = 0; - p->rc = rc; + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } + sqlite3_free(zBlob); } - *pzImposterCols = zImpCols; - *pzImposterPk = zImpPK; - *pzWhere = zWhere; - *pnBind = nBind; - return zRet; + return rc; } /* -** Assuming the current table columns are "a", "b" and "c", and the zObj -** paramter is passed "old", return a string of the form: -** -** "old.a, old.b, old.b" -** -** With the column names escaped. -** -** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append -** the text ", old._rowid_" to the returned value. +** This function is used to create delta-encoded serialized lists of FTS3 +** varints. Each call to this function appends a single varint to a list. */ -static char *rbuObjIterGetOldlist( - sqlite3rbu *p, - RbuObjIter *pIter, - const char *zObj +static void fts3PutDeltaVarint( + char **pp, /* IN/OUT: Output pointer */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + sqlite3_int64 iVal /* Write this value to the list */ ){ - char *zList = 0; - if( p->rc==SQLITE_OK && pIter->abIndexed ){ - const char *zS = ""; - int i; - for(i=0; inTblCol; i++){ - if( pIter->abIndexed[i] ){ - const char *zCol = pIter->azTblCol[i]; - zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol); - }else{ - zList = sqlite3_mprintf("%z%sNULL", zList, zS); - } - zS = ", "; - if( zList==0 ){ - p->rc = SQLITE_NOMEM; - break; - } - } - - /* For a table with implicit rowids, append "old._rowid_" to the list. */ - if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ - zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj); - } - } - return zList; + assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); + *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); + *piPrev = iVal; } /* -** Return an expression that can be used in a WHERE clause to match the -** primary key of the current table. For example, if the table is: -** -** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)); +** When this function is called, *ppPoslist is assumed to point to the +** start of a position-list. After it returns, *ppPoslist points to the +** first byte after the position-list. ** -** Return the string: +** A position list is list of positions (delta encoded) and columns for +** a single document record of a doclist. So, in other words, this +** routine advances *ppPoslist so that it points to the next docid in +** the doclist, or to the first byte past the end of the doclist. ** -** "b = ?1 AND c = ?2" +** If pp is not NULL, then the contents of the position list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. */ -static char *rbuObjIterGetWhere( - sqlite3rbu *p, - RbuObjIter *pIter -){ - char *zList = 0; - if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){ - zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1); - }else if( pIter->eType==RBU_PK_EXTERNAL ){ - const char *zSep = ""; - int i; - for(i=0; inTblCol; i++){ - if( pIter->abTblPk[i] ){ - zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1); - zSep = " AND "; - } - } - zList = rbuMPrintf(p, - "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList - ); +static void fts3PoslistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; - }else{ - const char *zSep = ""; - int i; - for(i=0; inTblCol; i++){ - if( pIter->abTblPk[i] ){ - const char *zCol = pIter->azTblCol[i]; - zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1); - zSep = " AND "; - } - } + /* The end of a position list is marked by a zero encoded as an FTS3 + ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by + ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail + ** of some other, multi-byte, value. + ** + ** The following while-loop moves pEnd to point to the first byte that is not + ** immediately preceded by a byte with the 0x80 bit set. Then increments + ** pEnd once more so that it points to the byte immediately following the + ** last byte in the position-list. + */ + while( *pEnd | c ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && (*pEnd)==0 ); } - return zList; -} + pEnd++; /* Advance past the POS_END terminator byte */ -/* -** The SELECT statement iterating through the keys for the current object -** (p->objiter.pSelect) currently points to a valid row. However, there -** is something wrong with the rbu_control value in the rbu_control value -** stored in the (p->nCol+1)'th column. Set the error code and error message -** of the RBU handle to something reflecting this. -*/ -static void rbuBadControlError(sqlite3rbu *p){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("invalid rbu_control value"); + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; } - /* -** Return a nul-terminated string containing the comma separated list of -** assignments that should be included following the "SET" keyword of -** an UPDATE statement used to update the table object that the iterator -** passed as the second argument currently points to if the rbu_control -** column of the data_xxx table entry is set to zMask. +** When this function is called, *ppPoslist is assumed to point to the +** start of a column-list. After it returns, *ppPoslist points to the +** to the terminator (POS_COLUMN or POS_END) byte of the column-list. ** -** The memory for the returned string is obtained from sqlite3_malloc(). -** It is the responsibility of the caller to eventually free it using -** sqlite3_free(). +** A column-list is list of delta-encoded positions for a single column +** within a single document within a doclist. +** +** The column-list is terminated either by a POS_COLUMN varint (1) or +** a POS_END varint (0). This routine leaves *ppPoslist pointing to +** the POS_COLUMN or POS_END that terminates the column-list. ** -** If an OOM error is encountered when allocating space for the new -** string, an error code is left in the rbu handle passed as the first -** argument and NULL is returned. Or, if an error has already occurred -** when this function is called, NULL is returned immediately, without -** attempting the allocation or modifying the stored error code. +** If pp is not NULL, then the contents of the column-list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. The POS_COLUMN or POS_END terminator +** is not copied into *pp. */ -static char *rbuObjIterGetSetlist( - sqlite3rbu *p, - RbuObjIter *pIter, - const char *zMask -){ - char *zList = 0; - if( p->rc==SQLITE_OK ){ - int i; +static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; - if( (int)strlen(zMask)!=pIter->nTblCol ){ - rbuBadControlError(p); - }else{ - const char *zSep = ""; - for(i=0; inTblCol; i++){ - char c = zMask[pIter->aiSrcOrder[i]]; - if( c=='x' ){ - zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", - zList, zSep, pIter->azTblCol[i], i+1 - ); - zSep = ", "; - } - else if( c=='d' ){ - zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", - zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 - ); - zSep = ", "; - } - else if( c=='f' ){ - zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", - zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 - ); - zSep = ", "; - } - } - } + /* A column-list is terminated by either a 0x01 or 0x00 byte that is + ** not part of a multi-byte varint. + */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && ((*pEnd)&0xfe)==0 ); + } + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; } - return zList; + *ppPoslist = pEnd; } /* -** Return a nul-terminated string consisting of nByte comma separated -** "?" expressions. For example, if nByte is 3, return a pointer to -** a buffer containing the string "?,?,?". -** -** The memory for the returned string is obtained from sqlite3_malloc(). -** It is the responsibility of the caller to eventually free it using -** sqlite3_free(). -** -** If an OOM error is encountered when allocating space for the new -** string, an error code is left in the rbu handle passed as the first -** argument and NULL is returned. Or, if an error has already occurred -** when this function is called, NULL is returned immediately, without -** attempting the allocation or modifying the stored error code. +** Value used to signify the end of an position-list. This is safe because +** it is not possible to have a document with 2^31 terms. */ -static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ - char *zRet = 0; - int nByte = nBind*2 + 1; - - zRet = (char*)rbuMalloc(p, nByte); - if( zRet ){ - int i; - for(i=0; izIdx==0 ); - if( p->rc==SQLITE_OK ){ - const char *zSep = "PRIMARY KEY("; - sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */ - sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = */ - - p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) - ); - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){ - const char *zOrig = (const char*)sqlite3_column_text(pXList,3); - if( zOrig && strcmp(zOrig, "pk")==0 ){ - const char *zIdx = (const char*)sqlite3_column_text(pXList,1); - if( zIdx ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) - ); - } - break; - } - } - rbuFinalize(p, pXList); - - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ - if( sqlite3_column_int(pXInfo, 5) ){ - /* int iCid = sqlite3_column_int(pXInfo, 0); */ - const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2); - const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : ""; - z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc); - zSep = ", "; - } - } - z = rbuMPrintf(p, "%z)", z); - rbuFinalize(p, pXInfo); +static void fts3ReadNextPos( + char **pp, /* IN/OUT: Pointer into position-list buffer */ + sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ +){ + if( (**pp)&0xFE ){ + fts3GetDeltaVarint(pp, pi); + *pi -= 2; + }else{ + *pi = POSITION_LIST_END; } - return z; } /* -** This function creates the second imposter table used when writing to -** a table b-tree where the table has an external primary key. If the -** iterator passed as the second argument does not currently point to -** a table (not index) with an external primary key, this function is a -** no-op. -** -** Assuming the iterator does point to a table with an external PK, this -** function creates a WITHOUT ROWID imposter table named "rbu_imposter2" -** used to access that PK index. For example, if the target table is -** declared as follows: -** -** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c)); -** -** then the imposter table schema is: -** -** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID; +** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by +** the value of iCol encoded as a varint to *pp. This will start a new +** column list. ** +** Set *pp to point to the byte just after the last byte written before +** returning (do not modify it if iCol==0). Return the total number of bytes +** written (0 if iCol==0). */ -static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ - if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){ - int tnum = pIter->iPkTnum; /* Root page of PK index */ - sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */ - const char *zIdx = 0; /* Name of PK index */ - sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */ - const char *zComma = ""; - char *zCols = 0; /* Used to build up list of table cols */ - char *zPk = 0; /* Used to build up table PK declaration */ - - /* Figure out the name of the primary key index for the current table. - ** This is needed for the argument to "PRAGMA index_xinfo". Set - ** zIdx to point to a nul-terminated string containing this name. */ - p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg, - "SELECT name FROM sqlite_master WHERE rootpage = ?" - ); - if( p->rc==SQLITE_OK ){ - sqlite3_bind_int(pQuery, 1, tnum); - if( SQLITE_ROW==sqlite3_step(pQuery) ){ - zIdx = (const char*)sqlite3_column_text(pQuery, 0); - } - } - if( zIdx ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) - ); - } - rbuFinalize(p, pQuery); - - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ - int bKey = sqlite3_column_int(pXInfo, 5); - if( bKey ){ - int iCid = sqlite3_column_int(pXInfo, 1); - int bDesc = sqlite3_column_int(pXInfo, 3); - const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); - zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %s", zCols, zComma, - iCid, pIter->azTblType[iCid], zCollate - ); - zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":""); - zComma = ", "; - } - } - zCols = rbuMPrintf(p, "%z, id INTEGER", zCols); - rbuFinalize(p, pXInfo); - - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); - rbuMPrintfExec(p, p->dbMain, - "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID", - zCols, zPk - ); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); +static int fts3PutColNumber(char **pp, int iCol){ + int n = 0; /* Number of bytes written */ + if( iCol ){ + char *p = *pp; /* Output pointer */ + n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); + *p = 0x01; + *pp = &p[n]; } + return n; } /* -** If an error has already occurred when this function is called, it -** immediately returns zero (without doing any work). Or, if an error -** occurs during the execution of this function, it sets the error code -** in the sqlite3rbu object indicated by the first argument and returns -** zero. -** -** The iterator passed as the second argument is guaranteed to point to -** a table (not an index) when this function is called. This function -** attempts to create any imposter table required to write to the main -** table b-tree of the table before returning. Non-zero is returned if -** an imposter table are created, or zero otherwise. -** -** An imposter table is required in all cases except RBU_PK_VTAB. Only -** virtual tables are written to directly. The imposter table has the -** same schema as the actual target table (less any UNIQUE constraints). -** More precisely, the "same schema" means the same columns, types, -** collation sequences. For tables that do not have an external PRIMARY -** KEY, it also means the same PRIMARY KEY declaration. -*/ -static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){ - if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){ - int tnum = pIter->iTnum; - const char *zComma = ""; - char *zSql = 0; - int iCol; - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); +** Compute the union of two position lists. The output written +** into *pp contains all positions of both *pp1 and *pp2 in sorted +** order and with any duplicates removed. All pointers are +** updated appropriately. The caller is responsible for insuring +** that there is enough space in *pp to hold the complete output. +*/ +static void fts3PoslistMerge( + char **pp, /* Output buffer */ + char **pp1, /* Left input list */ + char **pp2 /* Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; - for(iCol=0; p->rc==SQLITE_OK && iColnTblCol; iCol++){ - const char *zPk = ""; - const char *zCol = pIter->azTblCol[iCol]; - const char *zColl = 0; + while( *p1 || *p2 ){ + int iCol1; /* The current column index in pp1 */ + int iCol2; /* The current column index in pp2 */ - p->rc = sqlite3_table_column_metadata( - p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0 - ); + if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); + else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; + else iCol1 = 0; - if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){ - /* If the target table column is an "INTEGER PRIMARY KEY", add - ** "PRIMARY KEY" to the imposter table column declaration. */ - zPk = "PRIMARY KEY "; - } - zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s%s", - zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl, - (pIter->abNotNull[iCol] ? " NOT NULL" : "") - ); - zComma = ", "; - } + if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); + else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; + else iCol2 = 0; - if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ - char *zPk = rbuWithoutRowidPK(p, pIter); - if( zPk ){ - zSql = rbuMPrintf(p, "%z, %z", zSql, zPk); - } - } + if( iCol1==iCol2 ){ + sqlite3_int64 i1 = 0; /* Last position from pp1 */ + sqlite3_int64 i2 = 0; /* Last position from pp2 */ + sqlite3_int64 iPrev = 0; + int n = fts3PutColNumber(&p, iCol1); + p1 += n; + p2 += n; - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); - rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s", - pIter->zTbl, zSql, - (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") - ); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + /* At this point, both p1 and p2 point to the start of column-lists + ** for the same column (the column with index iCol1 and iCol2). + ** A column-list is a list of non-negative delta-encoded varints, each + ** incremented by 2 before being stored. Each list is terminated by a + ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists + ** and writes the results to buffer p. p is left pointing to the byte + ** after the list written. No terminator (POS_END or POS_COLUMN) is + ** written to the output. + */ + fts3GetDeltaVarint(&p1, &i1); + fts3GetDeltaVarint(&p2, &i2); + do { + fts3PutDeltaVarint(&p, &iPrev, (i1pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. +** when the *pp1 token appears before the *pp2 token, but not more than nToken +** slots before it. +** +** e.g. nToken==1 searches for adjacent positions. */ -static void rbuObjIterPrepareTmpInsert( - sqlite3rbu *p, - RbuObjIter *pIter, - const char *zCollist, - const char *zRbuRowid -){ - int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE); - char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid); - if( zBind ){ - assert( pIter->pTmpInsert==0 ); - p->rc = prepareFreeAndCollectError( - p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( - "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", - p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind - )); - } -} - -static void rbuTmpInsertFunc( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal +static int fts3PoslistPhraseMerge( + char **pp, /* IN/OUT: Preallocated output buffer */ + int nToken, /* Maximum difference in token positions */ + int isSaveLeft, /* Save the left position */ + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ ){ - sqlite3rbu *p = sqlite3_user_data(pCtx); - int rc = SQLITE_OK; - int i; + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + int iCol1 = 0; + int iCol2 = 0; - assert( sqlite3_value_int(apVal[0])!=0 - || p->objiter.eType==RBU_PK_EXTERNAL - || p->objiter.eType==RBU_PK_NONE - ); - if( sqlite3_value_int(apVal[0])!=0 ){ - p->nPhaseOneStep += p->objiter.nIndex; - } + /* Never set both isSaveLeft and isExact for the same invocation. */ + assert( isSaveLeft==0 || isExact==0 ); - for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); - } - if( rc==SQLITE_OK ){ - sqlite3_step(p->objiter.pTmpInsert); - rc = sqlite3_reset(p->objiter.pTmpInsert); + assert( p!=0 && *p1!=0 && *p2!=0 ); + if( *p1==POS_COLUMN ){ + p1++; + p1 += fts3GetVarint32(p1, &iCol1); } - - if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pCtx, rc); + if( *p2==POS_COLUMN ){ + p2++; + p2 += fts3GetVarint32(p2, &iCol2); } -} - -/* -** Ensure that the SQLite statement handles required to update the -** target database object currently indicated by the iterator passed -** as the second argument are available. -*/ -static int rbuObjIterPrepareAll( - sqlite3rbu *p, - RbuObjIter *pIter, - int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */ -){ - assert( pIter->bCleanup==0 ); - if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){ - const int tnum = pIter->iTnum; - char *zCollist = 0; /* List of indexed columns */ - char **pz = &p->zErrmsg; - const char *zIdx = pIter->zIdx; - char *zLimit = 0; - - if( nOffset ){ - zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset); - if( !zLimit ) p->rc = SQLITE_NOMEM; - } - - if( zIdx ){ - const char *zTbl = pIter->zTbl; - char *zImposterCols = 0; /* Columns for imposter table */ - char *zImposterPK = 0; /* Primary key declaration for imposter */ - char *zWhere = 0; /* WHERE clause on PK columns */ - char *zBind = 0; - int nBind = 0; - - assert( pIter->eType!=RBU_PK_VTAB ); - zCollist = rbuObjIterGetIndexCols( - p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind - ); - zBind = rbuObjIterGetBindlist(p, nBind); - - /* Create the imposter table used to write to this index. */ - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum); - rbuMPrintfExec(p, p->dbMain, - "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID", - zTbl, zImposterCols, zImposterPK - ); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); - - /* Create the statement to insert index entries */ - pIter->nCol = nBind; - if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError( - p->dbMain, &pIter->pInsert, &p->zErrmsg, - sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind) - ); - } - - /* And to delete index entries */ - if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError( - p->dbMain, &pIter->pDelete, &p->zErrmsg, - sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) - ); - } - - /* Create the SELECT statement to read keys in sorted order */ - if( p->rc==SQLITE_OK ){ - char *zSql; - if( rbuIsVacuum(p) ){ - zSql = sqlite3_mprintf( - "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s", - zCollist, - pIter->zDataTbl, - zCollist, zLimit - ); - }else - if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ - zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", - zCollist, p->zStateDb, pIter->zDataTbl, - zCollist, zLimit - ); - }else{ - zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " - "UNION ALL " - "SELECT %s, rbu_control FROM '%q' " - "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " - "ORDER BY %s%s", - zCollist, p->zStateDb, pIter->zDataTbl, - zCollist, pIter->zDataTbl, - zCollist, zLimit - ); - } - p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql); - } + while( 1 ){ + if( iCol1==iCol2 ){ + char *pSave = p; + sqlite3_int64 iPrev = 0; + sqlite3_int64 iPos1 = 0; + sqlite3_int64 iPos2 = 0; - sqlite3_free(zImposterCols); - sqlite3_free(zImposterPK); - sqlite3_free(zWhere); - sqlite3_free(zBind); - }else{ - int bRbuRowid = (pIter->eType==RBU_PK_VTAB) - ||(pIter->eType==RBU_PK_NONE) - ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); - const char *zTbl = pIter->zTbl; /* Table this step applies to */ - const char *zWrite; /* Imposter table name */ - - char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid); - char *zWhere = rbuObjIterGetWhere(p, pIter); - char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old"); - char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new"); - - zCollist = rbuObjIterGetCollist(p, pIter); - pIter->nCol = pIter->nTblCol; - - /* Create the imposter table or tables (if required). */ - rbuCreateImposterTable(p, pIter); - rbuCreateImposterTable2(p, pIter); - zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_"); - - /* Create the INSERT statement to write to the target PK b-tree */ - if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz, - sqlite3_mprintf( - "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", - zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings - ) - ); + if( iCol1 ){ + *p++ = POS_COLUMN; + p += sqlite3Fts3PutVarint(p, iCol1); } - /* Create the DELETE statement to write to the target PK b-tree. - ** Because it only performs INSERT operations, this is not required for - ** an rbu vacuum handle. */ - if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, - sqlite3_mprintf( - "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere - ) - ); - } + assert( *p1!=POS_END && *p1!=POS_COLUMN ); + assert( *p2!=POS_END && *p2!=POS_COLUMN ); + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; - if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ - const char *zRbuRowid = ""; - if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ - zRbuRowid = ", rbu_rowid"; + while( 1 ){ + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + ){ + sqlite3_int64 iSave; + iSave = isSaveLeft ? iPos1 : iPos2; + fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; + pSave = 0; + assert( p ); } - - /* Create the rbu_tmp_xxx table and the triggers to populate it. */ - rbuMPrintfExec(p, p->dbRbu, - "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " - "SELECT *%s FROM '%q' WHERE 0;" - , p->zStateDb, pIter->zDataTbl - , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") - , pIter->zDataTbl - ); - - rbuMPrintfExec(p, p->dbMain, - "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" " - "BEGIN " - " SELECT rbu_tmp_insert(3, %s);" - "END;" - - "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" " - "BEGIN " - " SELECT rbu_tmp_insert(3, %s);" - "END;" - - "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" " - "BEGIN " - " SELECT rbu_tmp_insert(4, %s);" - "END;", - zWrite, zTbl, zOldlist, - zWrite, zTbl, zOldlist, - zWrite, zTbl, zNewlist - ); - - if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ - rbuMPrintfExec(p, p->dbMain, - "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" " - "BEGIN " - " SELECT rbu_tmp_insert(0, %s);" - "END;", - zWrite, zTbl, zNewlist - ); + if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ + if( (*p2&0xFE)==0 ) break; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else{ + if( (*p1&0xFE)==0 ) break; + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; } - - rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid); } - /* Create the SELECT statement to read keys from data_xxx */ - if( p->rc==SQLITE_OK ){ - const char *zRbuRowid = ""; - if( bRbuRowid ){ - zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; - } - p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, - sqlite3_mprintf( - "SELECT %s,%s rbu_control%s FROM '%q'%s", - zCollist, - (rbuIsVacuum(p) ? "0 AS " : ""), - zRbuRowid, - pIter->zDataTbl, zLimit - ) - ); + if( pSave ){ + assert( pp && p ); + p = pSave; } - sqlite3_free(zWhere); - sqlite3_free(zOldlist); - sqlite3_free(zNewlist); - sqlite3_free(zBindings); - } - sqlite3_free(zCollist); - sqlite3_free(zLimit); - } - - return p->rc; -} - -/* -** Set output variable *ppStmt to point to an UPDATE statement that may -** be used to update the imposter table for the main table b-tree of the -** table object that pIter currently points to, assuming that the -** rbu_control column of the data_xyz table contains zMask. -** -** If the zMask string does not specify any columns to update, then this -** is not an error. Output variable *ppStmt is set to NULL in this case. -*/ -static int rbuGetUpdateStmt( - sqlite3rbu *p, /* RBU handle */ - RbuObjIter *pIter, /* Object iterator */ - const char *zMask, /* rbu_control value ('x.x.') */ - sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */ -){ - RbuUpdateStmt **pp; - RbuUpdateStmt *pUp = 0; - int nUp = 0; - - /* In case an error occurs */ - *ppStmt = 0; - - /* Search for an existing statement. If one is found, shift it to the front - ** of the LRU queue and return immediately. Otherwise, leave nUp pointing - ** to the number of statements currently in the cache and pUp to the - ** last object in the list. */ - for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){ - pUp = *pp; - if( strcmp(pUp->zMask, zMask)==0 ){ - *pp = pUp->pNext; - pUp->pNext = pIter->pRbuUpdate; - pIter->pRbuUpdate = pUp; - *ppStmt = pUp->pUpdate; - return SQLITE_OK; - } - nUp++; - } - assert( pUp==0 || pUp->pNext==0 ); - - if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){ - for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext)); - *pp = 0; - sqlite3_finalize(pUp->pUpdate); - pUp->pUpdate = 0; - }else{ - pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1); - } - - if( pUp ){ - char *zWhere = rbuObjIterGetWhere(p, pIter); - char *zSet = rbuObjIterGetSetlist(p, pIter, zMask); - char *zUpdate = 0; - - pUp->zMask = (char*)&pUp[1]; - memcpy(pUp->zMask, zMask, pIter->nTblCol); - pUp->pNext = pIter->pRbuUpdate; - pIter->pRbuUpdate = pUp; - - if( zSet ){ - const char *zPrefix = ""; + fts3ColumnlistCopy(0, &p1); + fts3ColumnlistCopy(0, &p2); + assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); + if( 0==*p1 || 0==*p2 ) break; - if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_"; - zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s", - zPrefix, pIter->zTbl, zSet, zWhere - ); - p->rc = prepareFreeAndCollectError( - p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate - ); - *ppStmt = pUp->pUpdate; + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + p2++; + p2 += fts3GetVarint32(p2, &iCol2); } - sqlite3_free(zWhere); - sqlite3_free(zSet); - } - return p->rc; -} - -static sqlite3 *rbuOpenDbhandle( - sqlite3rbu *p, - const char *zName, - int bUseVfs -){ - sqlite3 *db = 0; - if( p->rc==SQLITE_OK ){ - const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; - p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0); - if( p->rc ){ - p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - sqlite3_close(db); - db = 0; + /* Advance pointer p1 or p2 (whichever corresponds to the smaller of + ** iCol1 and iCol2) so that it points to either the 0x00 that marks the + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. + */ + else if( iCol1zTbl); - sqlite3_free(p->zIdx); - sqlite3_free(p); + fts3PoslistCopy(0, &p2); + fts3PoslistCopy(0, &p1); + *pp1 = p1; + *pp2 = p2; + if( *pp==p ){ + return 0; } + *p++ = 0x00; + *pp = p; + return 1; } /* -** Allocate an RbuState object and load the contents of the rbu_state -** table into it. Return a pointer to the new object. It is the -** responsibility of the caller to eventually free the object using -** sqlite3_free(). +** Merge two position-lists as required by the NEAR operator. The argument +** position lists correspond to the left and right phrases of an expression +** like: ** -** If an error occurs, leave an error code and message in the rbu handle -** and return NULL. +** "phrase 1" NEAR "phrase number 2" +** +** Position list *pp1 corresponds to the left-hand side of the NEAR +** expression and *pp2 to the right. As usual, the indexes in the position +** lists are the offsets of the last token in each phrase (tokens "1" and "2" +** in the example above). +** +** The output position list - written to *pp - is a copy of *pp2 with those +** entries that are not sufficiently NEAR entries in *pp1 removed. */ -static RbuState *rbuLoadState(sqlite3rbu *p){ - RbuState *pRet = 0; - sqlite3_stmt *pStmt = 0; - int rc; - int rc2; - - pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); - if( pRet==0 ) return 0; - - rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - switch( sqlite3_column_int(pStmt, 0) ){ - case RBU_STATE_STAGE: - pRet->eStage = sqlite3_column_int(pStmt, 1); - if( pRet->eStage!=RBU_STAGE_OAL - && pRet->eStage!=RBU_STAGE_MOVE - && pRet->eStage!=RBU_STAGE_CKPT - ){ - p->rc = SQLITE_CORRUPT; - } - break; - - case RBU_STATE_TBL: - pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_IDX: - pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_ROW: - pRet->nRow = sqlite3_column_int(pStmt, 1); - break; - - case RBU_STATE_PROGRESS: - pRet->nProgress = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_CKPT: - pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_COOKIE: - pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_OALSZ: - pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); - break; +static int fts3PoslistNearMerge( + char **pp, /* Output buffer */ + char *aTmp, /* Temporary buffer space */ + int nRight, /* Maximum difference in token positions */ + int nLeft, /* Maximum difference in token positions */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ +){ + char *p1 = *pp1; + char *p2 = *pp2; - case RBU_STATE_PHASEONESTEP: - pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); - break; + char *pTmp1 = aTmp; + char *pTmp2; + char *aTmp2; + int res = 1; - default: - rc = SQLITE_CORRUPT; - break; - } + fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); + aTmp2 = pTmp2 = pTmp1; + *pp1 = p1; + *pp2 = p2; + fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); + if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ + fts3PoslistMerge(pp, &aTmp, &aTmp2); + }else if( pTmp1!=aTmp ){ + fts3PoslistCopy(pp, &aTmp); + }else if( pTmp2!=aTmp2 ){ + fts3PoslistCopy(pp, &aTmp2); + }else{ + res = 0; } - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - p->rc = rc; - return pRet; + return res; } +/* +** An instance of this function is used to merge together the (potentially +** large number of) doclists for each term that matches a prefix query. +** See function fts3TermSelectMerge() for details. +*/ +typedef struct TermSelect TermSelect; +struct TermSelect { + char *aaOutput[16]; /* Malloc'd output buffers */ + int anOutput[16]; /* Size each output buffer in bytes */ +}; /* -** Open the database handle and attach the RBU database as "rbu". If an -** error occurs, leave an error code and message in the RBU handle. +** This function is used to read a single varint from a buffer. Parameter +** pEnd points 1 byte past the end of the buffer. When this function is +** called, if *pp points to pEnd or greater, then the end of the buffer +** has been reached. In this case *pp is set to 0 and the function returns. +** +** If *pp does not point to or past pEnd, then a single varint is read +** from *pp. *pp is then set to point 1 byte past the end of the read varint. +** +** If bDescIdx is false, the value read is added to *pVal before returning. +** If it is true, the value read is subtracted from *pVal before this +** function returns. */ -static void rbuOpenDatabase(sqlite3rbu *p){ - assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); - assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); - - /* Open the RBU database */ - p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); - - if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ - sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); - if( p->zState==0 ){ - const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); - p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile); - } - } - - /* If using separate RBU and state databases, attach the state database to - ** the RBU db handle now. */ - if( p->zState ){ - rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState); - memcpy(p->zStateDb, "stat", 4); +static void fts3GetDeltaVarint3( + char **pp, /* IN/OUT: Point to read varint from */ + char *pEnd, /* End of buffer */ + int bDescIdx, /* True if docids are descending */ + sqlite3_int64 *pVal /* IN/OUT: Integer value */ +){ + if( *pp>=pEnd ){ + *pp = 0; }else{ - memcpy(p->zStateDb, "main", 4); - } - -#if 0 - if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ - p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0); - } -#endif - - /* If it has not already been created, create the rbu_state table */ - rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); - -#if 0 - if( rbuIsVacuum(p) ){ - if( p->rc==SQLITE_OK ){ - int rc2; - int bOk = 0; - sqlite3_stmt *pCnt = 0; - p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, - "SELECT count(*) FROM stat.sqlite_master" - ); - if( p->rc==SQLITE_OK - && sqlite3_step(pCnt)==SQLITE_ROW - && 1==sqlite3_column_int(pCnt, 0) - ){ - bOk = 1; - } - rc2 = sqlite3_finalize(pCnt); - if( p->rc==SQLITE_OK ) p->rc = rc2; - - if( p->rc==SQLITE_OK && bOk==0 ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("invalid state database"); - } - - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); - } - } - } -#endif - - if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ - int bOpen = 0; - int rc; - p->nRbu = 0; - p->pRbuFd = 0; - rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); - if( rc!=SQLITE_NOTFOUND ) p->rc = rc; - if( p->eStage>=RBU_STAGE_MOVE ){ - bOpen = 1; - }else{ - RbuState *pState = rbuLoadState(p); - if( pState ){ - bOpen = (pState->eStage>RBU_STAGE_MOVE); - rbuFreeState(pState); - } - } - if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); - } - - p->eStage = 0; - if( p->rc==SQLITE_OK && p->dbMain==0 ){ - if( !rbuIsVacuum(p) ){ - p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); - }else if( p->pRbuFd->pWalFd ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + if( bDescIdx ){ + *pVal -= iVal; }else{ - char *zTarget; - char *zExtra = 0; - if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){ - zExtra = &p->zRbu[5]; - while( *zExtra ){ - if( *zExtra++=='?' ) break; - } - if( *zExtra=='\0' ) zExtra = 0; - } - - zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", - sqlite3_db_filename(p->dbRbu, "main"), - (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) - ); - - if( zTarget==0 ){ - p->rc = SQLITE_NOMEM; - return; - } - p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); - sqlite3_free(zTarget); + *pVal += iVal; } } - - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_create_function(p->dbMain, - "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 - ); - } - - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_create_function(p->dbMain, - "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0 - ); - } - - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_create_function(p->dbRbu, - "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 - ); - } - - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); - } - rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master"); - - /* Mark the database file just opened as an RBU target database. If - ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use. - ** This is an error. */ - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); - } - - if( p->rc==SQLITE_NOTFOUND ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("rbu vfs not found"); - } } /* -** This routine is a copy of the sqlite3FileSuffix3() routine from the core. -** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. -** -** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database -** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and -** if filename in z[] has a suffix (a.k.a. "extension") that is longer than -** three characters, then shorten the suffix on z[] to be the last three -** characters of the original suffix. +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. ** -** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always -** do the suffix shortening regardless of URI parameter. +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. ** -** Examples: +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). ** -** test.db-journal => test.nal -** test.db-wal => test.wal -** test.db-shm => test.shm -** test.db-mj7f3319fa => test.9fa +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. */ -static void rbuFileSuffix3(const char *zBase, char *z){ -#ifdef SQLITE_ENABLE_8_3_NAMES -#if SQLITE_ENABLE_8_3_NAMES<2 - if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) -#endif - { - int i, sz; - sz = (int)strlen(z)&0xffffff; - for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} - if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4); +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_int64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + iWrite = iVal - *piPrev; + }else{ + iWrite = *piPrev - iVal; } -#endif + assert( *pbFirst || *piPrev==0 ); + assert( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; } + /* -** Return the current wal-index header checksum for the target database -** as a 64-bit integer. +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). ** -** The checksum is store in the first page of xShmMap memory as an 8-byte -** blob starting at byte offset 40. +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. */ -static i64 rbuShmChecksum(sqlite3rbu *p){ - i64 iRet = 0; - if( p->rc==SQLITE_OK ){ - sqlite3_file *pDb = p->pTargetFd->pReal; - u32 volatile *ptr; - p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); - if( p->rc==SQLITE_OK ){ - iRet = ((i64)ptr[10] << 32) + ptr[11]; - } - } - return iRet; -} +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) /* -** This function is called as part of initializing or reinitializing an -** incremental checkpoint. +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. ** -** It populates the sqlite3rbu.aFrame[] array with the set of -** (wal frame -> db page) copy operations required to checkpoint the -** current wal file, and obtains the set of shm locks required to safely -** perform the copy operations directly on the file-system. +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. ** -** If argument pState is not NULL, then the incremental checkpoint is -** being resumed. In this case, if the checksum of the wal-index-header -** following recovery is not the same as the checksum saved in the RbuState -** object, then the rbu handle is set to DONE state. This occurs if some -** other client appends a transaction to the wal file in the middle of -** an incremental checkpoint. +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. */ -static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ +static int fts3DoclistOrMerge( + int bDescDoclist, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; - /* If pState is NULL, then the wal file may not have been opened and - ** recovered. Running a read-statement here to ensure that doing so - ** does not interfere with the "capture" process below. */ - if( pState==0 ){ - p->eStage = 0; - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_master", 0, 0, 0); - } - } + *paOut = 0; + *pnOut = 0; - /* Assuming no error has occurred, run a "restart" checkpoint with the - ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following - ** special behaviour in the rbu VFS: - ** - ** * If the exclusive shm WRITER or READ0 lock cannot be obtained, - ** the checkpoint fails with SQLITE_BUSY (normally SQLite would - ** proceed with running a passive checkpoint instead of failing). - ** - ** * Attempts to read from the *-wal file or write to the database file - ** do not perform any IO. Instead, the frame/page combinations that - ** would be read/written are recorded in the sqlite3rbu.aFrame[] - ** array. - ** - ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, - ** READ0 and CHECKPOINT locks taken as part of the checkpoint are - ** no-ops. These locks will not be released until the connection - ** is closed. + /* Allocate space for the output. Both the input and output doclists + ** are delta encoded. If they are in ascending order (bDescDoclist==0), + ** then the first docid in each list is simply encoded as a varint. For + ** each subsequent docid, the varint stored is the difference between the + ** current and previous docid (a positive number - since the list is in + ** ascending order). ** - ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL - ** error. + ** The first docid written to the output is therefore encoded using the + ** same number of bytes as it is in whichever of the input lists it is + ** read from. And each subsequent docid read from the same input list + ** consumes either the same or less bytes as it did in the input (since + ** the difference between it and the previous value in the output must + ** be a positive value less than or equal to the delta value read from + ** the input list). The same argument applies to all but the first docid + ** read from the 'other' list. And to the contents of all position lists + ** that will be copied and merged from the input to the output. ** - ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the - ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] - ** array populated with a set of (frame -> page) mappings. Because the - ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy - ** data from the wal file into the database file according to the - ** contents of aFrame[]. - */ - if( p->rc==SQLITE_OK ){ - int rc2; - p->eStage = RBU_STAGE_CAPTURE; - rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); - if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; - } - - if( p->rc==SQLITE_OK ){ - p->eStage = RBU_STAGE_CKPT; - p->nStep = (pState ? pState->nRow : 0); - p->aBuf = rbuMalloc(p, p->pgsz); - p->iWalCksum = rbuShmChecksum(p); - } - - if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){ - p->rc = SQLITE_DONE; - p->eStage = RBU_STAGE_DONE; - } -} - -/* -** Called when iAmt bytes are read from offset iOff of the wal file while -** the rbu object is in capture mode. Record the frame number of the frame -** being read in the aFrame[] array. -*/ -static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ - const u32 mReq = (1<mLock!=mReq ){ - pRbu->rc = SQLITE_BUSY; - return SQLITE_INTERNAL; - } - - pRbu->pgsz = iAmt; - if( pRbu->nFrame==pRbu->nFrameAlloc ){ - int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; - RbuFrame *aNew; - aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame)); - if( aNew==0 ) return SQLITE_NOMEM; - pRbu->aFrame = aNew; - pRbu->nFrameAlloc = nNew; - } - - iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1; - if( pRbu->iMaxFrame