diff --git a/.github/workflows/cross.sh b/.github/workflows/cross.sh index 883663bb..264c0f1b 100755 --- a/.github/workflows/cross.sh +++ b/.github/workflows/cross.sh @@ -17,7 +17,9 @@ echo aix ; GOOS=aix GOARCH=ppc64 go build . echo js ; GOOS=js GOARCH=wasm go build . echo wasip1 ; GOOS=wasip1 GOARCH=wasm go build . echo darwin-flock ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_flock . +echo darwin-noshm ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_noshm . echo darwin-nosys ; GOOS=darwin GOARCH=amd64 go build -tags sqlite3_nosys . +echo linux-noshm ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_noshm . echo linux-nosys ; GOOS=linux GOARCH=amd64 go build -tags sqlite3_nosys . echo windows-nosys ; GOOS=windows GOARCH=amd64 go build -tags sqlite3_nosys . echo freebsd-nosys ; GOOS=freebsd GOARCH=amd64 go build -tags sqlite3_nosys . \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39128f8e..1238661a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,10 @@ jobs: run: go test -v -tags sqlite3_flock ./... if: matrix.os == 'macos-latest' + - name: Test no shared memory + run: go test -v -tags sqlite3_noshm ./... + if: matrix.os == 'ubuntu-latest' + - name: Test no locks run: go test -v -tags sqlite3_nosys ./tests -run TestDB_nolock diff --git a/embed/exports.txt b/embed/exports.txt index 73c680fe..4af10dcc 100644 --- a/embed/exports.txt +++ b/embed/exports.txt @@ -1,6 +1,7 @@ free malloc malloc_destructor +aligned_alloc sqlite3_anycollseq_init sqlite3_backup_finish sqlite3_backup_init diff --git a/embed/sqlite3.wasm b/embed/sqlite3.wasm index 0490c164..c36b3c82 100755 Binary files a/embed/sqlite3.wasm and b/embed/sqlite3.wasm differ diff --git a/go.mod b/go.mod index 70590af1..e05ce31c 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,10 @@ go 1.21 require ( github.com/ncruces/julianday v1.0.0 github.com/psanford/httpreadat v0.1.0 - github.com/tetratelabs/wazero v1.7.0 - golang.org/x/crypto v0.21.0 - golang.org/x/sync v0.6.0 - golang.org/x/sys v0.18.0 + github.com/tetratelabs/wazero v1.7.1-0.20240410111357-a0fbb185447f + golang.org/x/crypto v0.22.0 + golang.org/x/sync v0.7.0 + golang.org/x/sys v0.19.0 golang.org/x/text v0.14.0 ) diff --git a/go.sum b/go.sum index 8c046809..596c223d 100644 --- a/go.sum +++ b/go.sum @@ -2,13 +2,13 @@ github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIwZLUE= github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE= -github.com/tetratelabs/wazero v1.7.0 h1:jg5qPydno59wqjpGrHph81lbtHzTrWzwwtD4cD88+hQ= -github.com/tetratelabs/wazero v1.7.0/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +github.com/tetratelabs/wazero v1.7.1-0.20240410111357-a0fbb185447f h1:xJ6F/f7fM1OvnPFSn7Ggf9icswSXoYOYLZbu7aJVQbA= +github.com/tetratelabs/wazero v1.7.1-0.20240410111357-a0fbb185447f/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= diff --git a/go.work.sum b/go.work.sum index 342066cb..5a6f9035 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,4 +1,8 @@ +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= diff --git a/gormlite/go.mod b/gormlite/go.mod index 13e1a4d3..6c7e94c1 100644 --- a/gormlite/go.mod +++ b/gormlite/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/ncruces/go-sqlite3 v0.13.0 - gorm.io/gorm v1.25.8 + gorm.io/gorm v1.25.9 ) require ( @@ -12,5 +12,5 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/ncruces/julianday v1.0.0 // indirect github.com/tetratelabs/wazero v1.7.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/sys v0.19.0 // indirect ) diff --git a/gormlite/go.sum b/gormlite/go.sum index bcf1a320..9fe5f3b3 100644 --- a/gormlite/go.sum +++ b/gormlite/go.sum @@ -8,9 +8,9 @@ github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/tetratelabs/wazero v1.7.0 h1:jg5qPydno59wqjpGrHph81lbtHzTrWzwwtD4cD88+hQ= github.com/tetratelabs/wazero v1.7.0/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= -gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.9 h1:wct0gxZIELDk8+ZqF/MVnHLkA1rvYlBWUMv2EdsK1g8= +gorm.io/gorm v1.25.9/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= diff --git a/gormlite/test.sh b/gormlite/test.sh index 68594ce4..f5ab285f 100755 --- a/gormlite/test.sh +++ b/gormlite/test.sh @@ -7,7 +7,7 @@ rm -rf gorm/ tests/ go work use -r . go test -git clone --branch v1.25.8 --filter=blob:none https://github.com/go-gorm/gorm.git +git clone --branch v1.25.9 --filter=blob:none https://github.com/go-gorm/gorm.git mv gorm/tests tests rm -rf gorm/ diff --git a/internal/util/alloc.go b/internal/util/alloc.go new file mode 100644 index 00000000..8e3e83f9 --- /dev/null +++ b/internal/util/alloc.go @@ -0,0 +1,81 @@ +//go:build unix + +package util + +import ( + "math" + + "github.com/tetratelabs/wazero/experimental" + "golang.org/x/sys/unix" +) + +func mmappedAllocator(min, cap, max uint64) experimental.MemoryBuffer { + // Round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + max = (max + rnd) &^ rnd + cap = (cap + rnd) &^ rnd + + if max > math.MaxInt { + // This ensures int(max) overflows to a negative value, + // and unix.Mmap returns EINVAL. + max = math.MaxUint64 + } + // Reserve max bytes of address space, to ensure we won't need to move it. + // A protected, private, anonymous mapping should not commit memory. + b, err := unix.Mmap(-1, 0, int(max), unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON) + if err != nil { + panic(err) + } + // Commit the initial cap bytes of memory. + err = unix.Mprotect(b[:cap], unix.PROT_READ|unix.PROT_WRITE) + if err != nil { + unix.Munmap(b) + panic(err) + } + return &mmappedBuffer{ + buf: b[:cap], + cur: min, + } +} + +// The slice covers the entire mmapped memory: +// - len(buf) is the already committed memory, +// - cap(buf) is the reserved address space, +// - cur is the already requested size. +type mmappedBuffer struct { + buf []byte + cur uint64 +} + +func (m *mmappedBuffer) Buffer() []byte { + // Limit capacity because bytes beyond len(m.buf) + // have not yet been committed. + return m.buf[:m.cur:len(m.buf)] +} + +func (m *mmappedBuffer) Grow(size uint64) []byte { + if com := uint64(len(m.buf)); com < size { + // Round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + new := (size + rnd) &^ rnd + + // Commit additional memory up to new bytes. + err := unix.Mprotect(m.buf[com:new], unix.PROT_READ|unix.PROT_WRITE) + if err != nil { + panic(err) + } + + // Update commited memory. + m.buf = m.buf[:new] + } + m.cur = size + return m.Buffer() +} + +func (m *mmappedBuffer) Free() { + err := unix.Munmap(m.buf[:cap(m.buf)]) + if err != nil { + panic(err) + } + m.buf = nil +} diff --git a/internal/util/handle.go b/internal/util/handle.go index 2309ed47..4584324c 100644 --- a/internal/util/handle.go +++ b/internal/util/handle.go @@ -3,21 +3,11 @@ package util import ( "context" "io" - - "github.com/tetratelabs/wazero/experimental" ) -type handleKey struct{} type handleState struct { handles []any - empty int -} - -func NewContext(ctx context.Context) context.Context { - state := new(handleState) - ctx = experimental.WithCloseNotifier(ctx, state) - ctx = context.WithValue(ctx, handleKey{}, state) - return ctx + holes int } func (s *handleState) CloseNotify(ctx context.Context, exitCode uint32) { @@ -27,14 +17,14 @@ func (s *handleState) CloseNotify(ctx context.Context, exitCode uint32) { } } s.handles = nil - s.empty = 0 + s.holes = 0 } func GetHandle(ctx context.Context, id uint32) any { if id == 0 { return nil } - s := ctx.Value(handleKey{}).(*handleState) + s := ctx.Value(moduleKey{}).(*moduleState) return s.handles[^id] } @@ -42,10 +32,10 @@ func DelHandle(ctx context.Context, id uint32) error { if id == 0 { return nil } - s := ctx.Value(handleKey{}).(*handleState) + s := ctx.Value(moduleKey{}).(*moduleState) a := s.handles[^id] s.handles[^id] = nil - s.empty++ + s.holes++ if c, ok := a.(io.Closer); ok { return c.Close() } @@ -56,13 +46,13 @@ func AddHandle(ctx context.Context, a any) (id uint32) { if a == nil { panic(NilErr) } - s := ctx.Value(handleKey{}).(*handleState) + s := ctx.Value(moduleKey{}).(*moduleState) // Find an empty slot. - if s.empty > cap(s.handles)-len(s.handles) { + if s.holes > cap(s.handles)-len(s.handles) { for id, h := range s.handles { if h == nil { - s.empty-- + s.holes-- s.handles[id] = a return ^uint32(id) } diff --git a/internal/util/mmap.go b/internal/util/mmap.go new file mode 100644 index 00000000..a2067b94 --- /dev/null +++ b/internal/util/mmap.go @@ -0,0 +1,105 @@ +//go:build (linux || darwin) && (amd64 || arm64) && !sqlite3_flock && !sqlite3_noshm && !sqlite3_nosys + +package util + +import ( + "context" + "os" + "unsafe" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "golang.org/x/sys/unix" +) + +type mmapState struct { + regions []*MappedRegion + enabled bool +} + +func (s *mmapState) init(ctx context.Context, enabled bool) context.Context { + if s.enabled = enabled; enabled { + return experimental.WithMemoryAllocator(ctx, mmappedAllocator) + } + return ctx +} + +func CanMap(ctx context.Context) bool { + s := ctx.Value(moduleKey{}).(*moduleState) + return s.mmapState.enabled +} + +func (s *mmapState) new(ctx context.Context, mod api.Module, size uint32) *MappedRegion { + // Find unused region. + for _, r := range s.regions { + if !r.used && r.size == size { + return r + } + } + + // Allocate page aligned memmory. + alloc := mod.ExportedFunction("aligned_alloc") + stack := [2]uint64{ + uint64(unix.Getpagesize()), + uint64(size), + } + if err := alloc.CallWithStack(ctx, stack[:]); err != nil { + panic(err) + } + if stack[0] == 0 { + panic(OOMErr) + } + + // Save the newly allocated region. + ptr := uint32(stack[0]) + buf := View(mod, ptr, uint64(size)) + addr := uintptr(unsafe.Pointer(&buf[0])) + s.regions = append(s.regions, &MappedRegion{ + Ptr: ptr, + addr: addr, + size: size, + }) + return s.regions[len(s.regions)-1] +} + +type MappedRegion struct { + addr uintptr + Ptr uint32 + size uint32 + used bool +} + +func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size uint32) (*MappedRegion, error) { + s := ctx.Value(moduleKey{}).(*moduleState) + r := s.new(ctx, mod, size) + err := r.mmap(f, offset) + if err != nil { + return nil, err + } + return r, nil +} + +func (r *MappedRegion) Unmap() error { + // We can't munmap the region, otherwise it could be remaped. + // Instead, convert it to a protected, private, anonymous mapping. + // If successful, it can be reused for a subsequent mmap. + _, err := mmap(r.addr, uintptr(r.size), + unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON|unix.MAP_FIXED, + -1, 0) + r.used = err != nil + return err +} + +func (r *MappedRegion) mmap(f *os.File, offset int64) error { + _, err := mmap(r.addr, uintptr(r.size), + unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED|unix.MAP_FIXED, + int(f.Fd()), offset) + r.used = err == nil + return err +} + +//go:linkname mmap syscall.mmap +func mmap(addr, length uintptr, prot, flag, fd int, pos int64) (*byte, error) + +//go:linkname munmap syscall.munmap +func munmap(addr, length uintptr) error diff --git a/internal/util/mmap_other.go b/internal/util/mmap_other.go new file mode 100644 index 00000000..d585340f --- /dev/null +++ b/internal/util/mmap_other.go @@ -0,0 +1,15 @@ +//go:build !(linux || darwin) || !(amd64 || arm64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys + +package util + +import "context" + +type mmapState struct{} + +func (s *mmapState) init(ctx context.Context, _ bool) context.Context { + return ctx +} + +func CanMap(ctx context.Context) bool { + return false +} diff --git a/internal/util/module.go b/internal/util/module.go new file mode 100644 index 00000000..0db93bbb --- /dev/null +++ b/internal/util/module.go @@ -0,0 +1,21 @@ +package util + +import ( + "context" + + "github.com/tetratelabs/wazero/experimental" +) + +type moduleKey struct{} +type moduleState struct { + handleState + mmapState +} + +func NewContext(ctx context.Context, mappableMemory bool) context.Context { + state := new(moduleState) + ctx = context.WithValue(ctx, moduleKey{}, state) + ctx = experimental.WithCloseNotifier(ctx, state) + ctx = state.mmapState.init(ctx, mappableMemory) + return ctx +} diff --git a/sqlite.go b/sqlite.go index 56ea68b8..a7f3f3b0 100644 --- a/sqlite.go +++ b/sqlite.go @@ -85,7 +85,7 @@ func instantiateSQLite() (sqlt *sqlite, err error) { } sqlt = new(sqlite) - sqlt.ctx = util.NewContext(context.Background()) + sqlt.ctx = util.NewContext(context.Background(), vfs.SupportsSharedMemory) sqlt.mod, err = instance.runtime.InstantiateModule(sqlt.ctx, instance.compiled, wazero.NewModuleConfig()) diff --git a/sqlite3/vfs.c b/sqlite3/vfs.c index 31558be6..d2e79fb6 100644 --- a/sqlite3/vfs.c +++ b/sqlite3/vfs.c @@ -13,7 +13,7 @@ int go_sleep(sqlite3_vfs *, int microseconds); int go_current_time_64(sqlite3_vfs *, sqlite3_int64 *); int go_open(sqlite3_vfs *, sqlite3_filename zName, sqlite3_file *, int flags, - int *pOutFlags); + int *pOutFlags, int *pOutVFS); int go_delete(sqlite3_vfs *, const char *zName, int syncDir); int go_access(sqlite3_vfs *, const char *zName, int flags, int *pResOut); int go_full_pathname(sqlite3_vfs *, const char *zName, int nOut, char *zOut); @@ -32,29 +32,55 @@ int go_lock(sqlite3_file *, int eLock); int go_unlock(sqlite3_file *, int eLock); int go_check_reserved_lock(sqlite3_file *, int *pResOut); +int go_shm_map(sqlite3_file *, int iPg, int pgsz, int, void volatile **); +int go_shm_lock(sqlite3_file *, int offset, int n, int flags); +int go_shm_unmap(sqlite3_file *, int deleteFlag); +void go_shm_barrier(sqlite3_file *); + static int go_open_wrapper(sqlite3_vfs *vfs, sqlite3_filename zName, sqlite3_file *file, int flags, int *pOutFlags) { - static const sqlite3_io_methods os_io = { - .iVersion = 1, - .xClose = go_close, - .xRead = go_read, - .xWrite = go_write, - .xTruncate = go_truncate, - .xSync = go_sync, - .xFileSize = go_file_size, - .xLock = go_lock, - .xUnlock = go_unlock, - .xCheckReservedLock = go_check_reserved_lock, - .xFileControl = go_file_control, - .xSectorSize = go_sector_size, - .xDeviceCharacteristics = go_device_characteristics, - }; + static const sqlite3_io_methods go_io[2] = { + { + .iVersion = 1, + .xClose = go_close, + .xRead = go_read, + .xWrite = go_write, + .xTruncate = go_truncate, + .xSync = go_sync, + .xFileSize = go_file_size, + .xLock = go_lock, + .xUnlock = go_unlock, + .xCheckReservedLock = go_check_reserved_lock, + .xFileControl = go_file_control, + .xSectorSize = go_sector_size, + .xDeviceCharacteristics = go_device_characteristics, + }, + { + .iVersion = 2, + .xClose = go_close, + .xRead = go_read, + .xWrite = go_write, + .xTruncate = go_truncate, + .xSync = go_sync, + .xFileSize = go_file_size, + .xLock = go_lock, + .xUnlock = go_unlock, + .xCheckReservedLock = go_check_reserved_lock, + .xFileControl = go_file_control, + .xSectorSize = go_sector_size, + .xDeviceCharacteristics = go_device_characteristics, + .xShmMap = go_shm_map, + .xShmLock = go_shm_lock, + .xShmBarrier = go_shm_barrier, + .xShmUnmap = go_shm_unmap, + }}; + int vfsID = 0; memset(file, 0, vfs->szOsFile); - int rc = go_open(vfs, zName, file, flags, pOutFlags); + int rc = go_open(vfs, zName, file, flags, pOutFlags, &vfsID); if (rc) { return rc; } - file->pMethods = &os_io; + file->pMethods = &go_io[vfsID]; return SQLITE_OK; } diff --git a/tests/parallel/parallel_test.go b/tests/parallel/parallel_test.go index 7c2f6726..b5bdb263 100644 --- a/tests/parallel/parallel_test.go +++ b/tests/parallel/parallel_test.go @@ -12,9 +12,16 @@ import ( "github.com/ncruces/go-sqlite3" _ "github.com/ncruces/go-sqlite3/embed" + "github.com/ncruces/go-sqlite3/vfs" "github.com/ncruces/go-sqlite3/vfs/memdb" + "github.com/tetratelabs/wazero" ) +func TestMain(m *testing.M) { + sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(1024) + os.Exit(m.Run()) +} + func TestParallel(t *testing.T) { var iter int if testing.Short() { @@ -32,6 +39,20 @@ func TestParallel(t *testing.T) { testIntegrity(t, name) } +func TestWAL(t *testing.T) { + if !vfs.SupportsSharedMemory { + t.Skip("skipping without shared memory") + } + + name := "file:" + + filepath.Join(t.TempDir(), "test.db") + + "?_pragma=busy_timeout(10000)" + + "&_pragma=journal_mode(wal)" + + "&_pragma=synchronous(off)" + testParallel(t, name, 1000) + testIntegrity(t, name) +} + func TestMemory(t *testing.T) { var iter int if testing.Short() { diff --git a/tests/wal_test.go b/tests/wal_test.go new file mode 100644 index 00000000..3076b64a --- /dev/null +++ b/tests/wal_test.go @@ -0,0 +1,33 @@ +package tests + +import ( + "path/filepath" + "testing" + + "github.com/ncruces/go-sqlite3" +) + +func TestWAL_enter_exit(t *testing.T) { + t.Parallel() + + file := filepath.Join(t.TempDir(), "test.db") + + db, err := sqlite3.Open(file) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + err = db.Exec(` + CREATE TABLE test (col); + PRAGMA journal_mode=WAL; + SELECT * FROM test; + PRAGMA journal_mode=DELETE; + SELECT * FROM test; + PRAGMA journal_mode=WAL; + SELECT * FROM test; + `) + if err != nil { + t.Fatal(err) + } +} diff --git a/vfs/const.go b/vfs/const.go index 0a3c3aba..6405d618 100644 --- a/vfs/const.go +++ b/vfs/const.go @@ -37,6 +37,10 @@ const ( _IOERR_CHECKRESERVEDLOCK _ErrorCode = util.IOERR_CHECKRESERVEDLOCK _IOERR_LOCK _ErrorCode = util.IOERR_LOCK _IOERR_CLOSE _ErrorCode = util.IOERR_CLOSE + _IOERR_SHMOPEN _ErrorCode = util.IOERR_SHMOPEN + _IOERR_SHMSIZE _ErrorCode = util.IOERR_SHMSIZE + _IOERR_SHMLOCK _ErrorCode = util.IOERR_SHMLOCK + _IOERR_SHMMAP _ErrorCode = util.IOERR_SHMMAP _IOERR_SEEK _ErrorCode = util.IOERR_SEEK _IOERR_DELETE_NOENT _ErrorCode = util.IOERR_DELETE_NOENT _IOERR_BEGIN_ATOMIC _ErrorCode = util.IOERR_BEGIN_ATOMIC @@ -44,6 +48,7 @@ const ( _IOERR_ROLLBACK_ATOMIC _ErrorCode = util.IOERR_ROLLBACK_ATOMIC _CANTOPEN_FULLPATH _ErrorCode = util.CANTOPEN_FULLPATH _CANTOPEN_ISDIR _ErrorCode = util.CANTOPEN_ISDIR + _READONLY_CANTINIT _ErrorCode = util.READONLY_CANTINIT _OK_SYMLINK _ErrorCode = util.OK_SYMLINK ) @@ -213,3 +218,13 @@ const ( _FCNTL_CKSM_FILE _FcntlOpcode = 41 _FCNTL_RESET_CACHE _FcntlOpcode = 42 ) + +// https://sqlite.org/c3ref/c_shm_exclusive.html +type _ShmFlag uint32 + +const ( + _SHM_UNLOCK _ShmFlag = 1 + _SHM_LOCK _ShmFlag = 2 + _SHM_SHARED _ShmFlag = 4 + _SHM_EXCLUSIVE _ShmFlag = 8 +) diff --git a/vfs/file.go b/vfs/file.go index d589d377..42d03ed6 100644 --- a/vfs/file.go +++ b/vfs/file.go @@ -130,6 +130,7 @@ type vfsFile struct { keepWAL bool syncDir bool psow bool + shm vfsShm } var ( @@ -141,6 +142,11 @@ var ( _ FilePowersafeOverwrite = &vfsFile{} ) +func (f *vfsFile) Close() error { + f.shm.Close() + return f.File.Close() +} + func (f *vfsFile) Sync(flags SyncFlag) error { dataonly := (flags & SYNC_DATAONLY) != 0 fullsync := (flags & 0x0f) == SYNC_FULL diff --git a/vfs/lock.go b/vfs/lock.go index 1b705499..9332c9f5 100644 --- a/vfs/lock.go +++ b/vfs/lock.go @@ -1,7 +1,17 @@ +//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys + package vfs import "github.com/ncruces/go-sqlite3/internal/util" +// SupportsFileLocking is false on platforms that do not support file locking. +// To open a database file on those platforms, +// you need to use the [nolock] or [immutable] URI parameters. +// +// [nolock]: https://sqlite.org/uri.html#urinolock +// [immutable]: https://sqlite.org/uri.html#uriimmutable +const SupportsFileLocking = true + const ( _PENDING_BYTE = 0x40000000 _RESERVED_BYTE = (_PENDING_BYTE + 1) diff --git a/vfs/lock_other.go b/vfs/lock_other.go new file mode 100644 index 00000000..a4563af4 --- /dev/null +++ b/vfs/lock_other.go @@ -0,0 +1,23 @@ +//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) || sqlite3_nosys + +package vfs + +// SupportsFileLocking is false on platforms that do not support file locking. +// To open a database file on those platforms, +// you need to use the [nolock] or [immutable] URI parameters. +// +// [nolock]: https://sqlite.org/uri.html#urinolock +// [immutable]: https://sqlite.org/uri.html#uriimmutable +const SupportsFileLocking = false + +func (f *vfsFile) Lock(LockLevel) error { + return _IOERR_LOCK +} + +func (f *vfsFile) Unlock(LockLevel) error { + return _IOERR_UNLOCK +} + +func (f *vfsFile) CheckReservedLock() (bool, error) { + return false, _IOERR_CHECKRESERVEDLOCK +} diff --git a/vfs/lock_test.go b/vfs/lock_test.go index db1c4df8..6cf359c1 100644 --- a/vfs/lock_test.go +++ b/vfs/lock_test.go @@ -33,7 +33,7 @@ func Test_vfsLock(t *testing.T) { pOutput = 32 ) mod := wazerotest.NewModule(wazerotest.NewMemory(wazerotest.PageSize)) - ctx := util.NewContext(context.TODO()) + ctx := util.NewContext(context.TODO(), false) vfsFileRegister(ctx, mod, pFile1, &vfsFile{File: file1}) vfsFileRegister(ctx, mod, pFile2, &vfsFile{File: file2}) diff --git a/vfs/os_bsd.go b/vfs/os_bsd.go index 441d3e9b..3506ec9e 100644 --- a/vfs/os_bsd.go +++ b/vfs/os_bsd.go @@ -32,14 +32,14 @@ func osWriteLock(file *os.File, _ /*start*/, _ /*len*/ int64, _ /*timeout*/ time return osLock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK) } -func osCheckLock(file *os.File, start, len int64) (bool, _ErrorCode) { +func osGetLock(file *os.File, start, len int64) (int16, _ErrorCode) { lock := unix.Flock_t{ - Type: unix.F_RDLCK, + Type: unix.F_WRLCK, Start: start, Len: len, } if unix.FcntlFlock(file.Fd(), unix.F_GETLK, &lock) != nil { - return false, _IOERR_CHECKRESERVEDLOCK + return 0, _IOERR_CHECKRESERVEDLOCK } - return lock.Type != unix.F_UNLCK, _OK + return lock.Type, _OK } diff --git a/vfs/os_darwin.go b/vfs/os_darwin.go index 7c1ed959..4269445e 100644 --- a/vfs/os_darwin.go +++ b/vfs/os_darwin.go @@ -95,14 +95,14 @@ func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorC return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK) } -func osCheckLock(file *os.File, start, len int64) (bool, _ErrorCode) { +func osGetLock(file *os.File, start, len int64) (int16, _ErrorCode) { lock := unix.Flock_t{ - Type: unix.F_RDLCK, + Type: unix.F_WRLCK, Start: start, Len: len, } if unix.FcntlFlock(file.Fd(), _F_OFD_GETLK, &lock) != nil { - return false, _IOERR_CHECKRESERVEDLOCK + return 0, _IOERR_CHECKRESERVEDLOCK } - return lock.Type != unix.F_UNLCK, _OK + return lock.Type, _OK } diff --git a/vfs/os_nolock.go b/vfs/os_nolock.go deleted file mode 100644 index 4bceefed..00000000 --- a/vfs/os_nolock.go +++ /dev/null @@ -1,41 +0,0 @@ -//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) || sqlite3_nosys - -package vfs - -import "os" - -// SupportsFileLocking is false on platforms that do not support file locking. -// To open a database file in one such platform, -// you need to use the [nolock] or [immutable] URI parameters. -// -// [nolock]: https://sqlite.org/uri.html#urinolock -// [immutable]: https://sqlite.org/uri.html#uriimmutable -const SupportsFileLocking = false - -func osGetSharedLock(_ *os.File) _ErrorCode { - return _IOERR_RDLOCK -} - -func osGetReservedLock(_ *os.File) _ErrorCode { - return _IOERR_LOCK -} - -func osGetPendingLock(_ *os.File, _ bool) _ErrorCode { - return _IOERR_LOCK -} - -func osGetExclusiveLock(_ *os.File, _ bool) _ErrorCode { - return _IOERR_LOCK -} - -func osDowngradeLock(_ *os.File, _ LockLevel) _ErrorCode { - return _IOERR_RDLOCK -} - -func osReleaseLock(_ *os.File, _ LockLevel) _ErrorCode { - return _IOERR_UNLOCK -} - -func osCheckReservedLock(_ *os.File) (bool, _ErrorCode) { - return false, _IOERR_CHECKRESERVEDLOCK -} diff --git a/vfs/os_ofd.go b/vfs/os_ofd.go index ced941eb..f94ece47 100644 --- a/vfs/os_ofd.go +++ b/vfs/os_ofd.go @@ -57,14 +57,14 @@ func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorC return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK) } -func osCheckLock(file *os.File, start, len int64) (bool, _ErrorCode) { +func osGetLock(file *os.File, start, len int64) (int16, _ErrorCode) { lock := unix.Flock_t{ - Type: unix.F_RDLCK, + Type: unix.F_WRLCK, Start: start, Len: len, } if unix.FcntlFlock(file.Fd(), unix.F_OFD_GETLK, &lock) != nil { - return false, _IOERR_CHECKRESERVEDLOCK + return 0, _IOERR_CHECKRESERVEDLOCK } - return lock.Type != unix.F_UNLCK, _OK + return lock.Type, _OK } diff --git a/vfs/os_unix_lock.go b/vfs/os_unix_lock.go index 15db23f0..fc4309d4 100644 --- a/vfs/os_unix_lock.go +++ b/vfs/os_unix_lock.go @@ -9,17 +9,9 @@ import ( "golang.org/x/sys/unix" ) -// SupportsFileLocking is false on platforms that do not support file locking. -// To open a database file in one such platform, -// you need to use the [nolock] or [immutable] URI parameters. -// -// [nolock]: https://sqlite.org/uri.html#urinolock -// [immutable]: https://sqlite.org/uri.html#uriimmutable -const SupportsFileLocking = true - func osGetSharedLock(file *os.File) _ErrorCode { // Test the PENDING lock before acquiring a new SHARED lock. - if pending, _ := osCheckLock(file, _PENDING_BYTE, 1); pending { + if lock, _ := osGetLock(file, _PENDING_BYTE, 1); lock == unix.F_WRLCK { return _BUSY } // Acquire the SHARED lock. @@ -72,7 +64,8 @@ func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode { func osCheckReservedLock(file *os.File) (bool, _ErrorCode) { // Test the RESERVED lock. - return osCheckLock(file, _RESERVED_BYTE, 1) + lock, rc := osGetLock(file, _RESERVED_BYTE, 1) + return lock == unix.F_WRLCK, rc } func osLockErrorCode(err error, def _ErrorCode) _ErrorCode { diff --git a/vfs/os_windows.go b/vfs/os_windows.go index abc6c4b7..0fee2f17 100644 --- a/vfs/os_windows.go +++ b/vfs/os_windows.go @@ -9,18 +9,9 @@ import ( "golang.org/x/sys/windows" ) -// SupportsFileLocking is false on platforms that do not support file locking. -// To open a database file in one such platform, -// you need to use the [nolock] or [immutable] URI parameters. -// -// [nolock]: https://sqlite.org/uri.html#urinolock -// [immutable]: https://sqlite.org/uri.html#uriimmutable -const SupportsFileLocking = true - func osGetSharedLock(file *os.File) _ErrorCode { // Acquire the PENDING lock temporarily before acquiring a new SHARED lock. rc := osReadLock(file, _PENDING_BYTE, 1, 0) - if rc == _OK { // Acquire the SHARED lock. rc = osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0) @@ -104,7 +95,15 @@ func osReleaseLock(file *os.File, state LockLevel) _ErrorCode { func osCheckReservedLock(file *os.File) (bool, _ErrorCode) { // Test the RESERVED lock. - return osCheckLock(file, _RESERVED_BYTE, 1) + rc := osLock(file, 0, _RESERVED_BYTE, 1, 0, _IOERR_CHECKRESERVEDLOCK) + if rc == _BUSY { + return true, _OK + } + if rc == _OK { + // Release the RESERVED lock. + osUnlock(file, _RESERVED_BYTE, 1) + } + return false, rc } func osUnlock(file *os.File, start, len uint32) _ErrorCode { @@ -155,17 +154,6 @@ func osWriteLock(file *os.File, start, len uint32, timeout time.Duration) _Error return osLock(file, windows.LOCKFILE_EXCLUSIVE_LOCK, start, len, timeout, _IOERR_LOCK) } -func osCheckLock(file *os.File, start, len uint32) (bool, _ErrorCode) { - rc := osLock(file, 0, start, len, 0, _IOERR_CHECKRESERVEDLOCK) - if rc == _BUSY { - return true, _OK - } - if rc == _OK { - osUnlock(file, start, len) - } - return false, rc -} - func osLockErrorCode(err error, def _ErrorCode) _ErrorCode { if err == nil { return _OK diff --git a/vfs/shm.go b/vfs/shm.go new file mode 100644 index 00000000..1dda1b5c --- /dev/null +++ b/vfs/shm.go @@ -0,0 +1,144 @@ +//go:build (linux || darwin) && (amd64 || arm64) && !sqlite3_flock && !sqlite3_noshm && !sqlite3_nosys + +package vfs + +import ( + "context" + "io" + "os" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/tetratelabs/wazero/api" + "golang.org/x/sys/unix" +) + +// SupportsSharedMemory is true on platforms that support shared memory. +// To enable shared memory support on those platforms, +// you need to set the appropriate [wazero.RuntimeConfig]; +// otherwise, [EXCLUSIVE locking mode] is activated automatically +// to use [WAL without shared-memory]. +// +// [WAL without shared-memory]: https://sqlite.org/wal.html#noshm +// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode +const SupportsSharedMemory = true + +type vfsShm struct { + *os.File + regions []*util.MappedRegion +} + +const ( + _SHM_NLOCK = 8 + _SHM_BASE = 120 + _SHM_DMS = _SHM_BASE + _SHM_NLOCK +) + +func (f *vfsFile) shmMap(ctx context.Context, mod api.Module, id, size uint32, extend bool) (uint32, error) { + // Ensure size is a multiple of the OS page size. + if int(size)&(unix.Getpagesize()-1) != 0 { + return 0, _IOERR_SHMMAP + } + + if f.shm.File == nil { + var flag int + if f.readOnly { + flag = unix.O_RDONLY | unix.O_NOFOLLOW + } else { + flag = unix.O_RDWR | unix.O_CREAT | unix.O_NOFOLLOW + } + s, err := os.OpenFile(f.Name()+"-shm", flag, 0666) + if err != nil { + return 0, _CANTOPEN + } + f.shm.File = s + } + + // Dead man's switch. + if lock, rc := osGetLock(f.shm.File, _SHM_DMS, 1); rc != _OK { + return 0, _IOERR_LOCK + } else if lock == unix.F_WRLCK { + return 0, _BUSY + } else if lock == unix.F_UNLCK { + if f.readOnly { + return 0, _READONLY_CANTINIT + } + if rc := osWriteLock(f.shm.File, _SHM_DMS, 1, 0); rc != _OK { + return 0, rc + } + if err := f.shm.Truncate(0); err != nil { + return 0, _IOERR_SHMOPEN + } + } + if rc := osReadLock(f.shm.File, _SHM_DMS, 1, 0); rc != _OK { + return 0, rc + } + + // Check if file is big enough. + s, err := f.shm.Seek(0, io.SeekEnd) + if err != nil { + return 0, _IOERR_SHMSIZE + } + if n := (int64(id) + 1) * int64(size); n > s { + if !extend { + return 0, nil + } + err := osAllocate(f.shm.File, n) + if err != nil { + return 0, _IOERR_SHMSIZE + } + } + + r, err := util.MapRegion(ctx, mod, f.shm.File, int64(id)*int64(size), size) + if err != nil { + return 0, err + } + f.shm.regions = append(f.shm.regions, r) + return r.Ptr, nil +} + +func (f *vfsFile) shmLock(offset, n uint32, flags _ShmFlag) error { + // Argument check. + if n == 0 || offset+n > _SHM_NLOCK { + panic(util.AssertErr()) + } + switch flags { + case + _SHM_LOCK | _SHM_SHARED, + _SHM_LOCK | _SHM_EXCLUSIVE, + _SHM_UNLOCK | _SHM_SHARED, + _SHM_UNLOCK | _SHM_EXCLUSIVE: + // + default: + panic(util.AssertErr()) + } + if n != 1 && flags&_SHM_EXCLUSIVE == 0 { + panic(util.AssertErr()) + } + + switch { + case flags&_SHM_UNLOCK != 0: + return osUnlock(f.shm.File, _SHM_BASE+int64(offset), int64(n)) + case flags&_SHM_SHARED != 0: + return osReadLock(f.shm.File, _SHM_BASE+int64(offset), int64(n), 0) + case flags&_SHM_EXCLUSIVE != 0: + return osWriteLock(f.shm.File, _SHM_BASE+int64(offset), int64(n), 0) + default: + panic(util.AssertErr()) + } +} + +func (f *vfsFile) shmUnmap(delete bool) { + // Unmap regions. + for _, r := range f.shm.regions { + r.Unmap() + } + clear(f.shm.regions) + f.shm.regions = f.shm.regions[:0] + + // Close the file. + if delete && f.shm.File != nil { + os.Remove(f.shm.Name()) + } + f.shm.Close() + f.shm.File = nil +} diff --git a/vfs/shm_other.go b/vfs/shm_other.go new file mode 100644 index 00000000..922e1475 --- /dev/null +++ b/vfs/shm_other.go @@ -0,0 +1,17 @@ +//go:build !(linux || darwin) || !(amd64 || arm64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys + +package vfs + +// SupportsSharedMemory is true on platforms that support shared memory. +// To enable shared memory support on those platforms, +// you need to set the appropriate [wazero.RuntimeConfig]; +// otherwise, [EXCLUSIVE locking mode] is activated automatically +// to use [WAL without shared-memory]. +// +// [WAL without shared-memory]: https://sqlite.org/wal.html#noshm +// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode +const SupportsSharedMemory = false + +type vfsShm struct{} + +func (vfsShm) Close() error { return nil } diff --git a/vfs/tests/mptest/mptest_test.go b/vfs/tests/mptest/mptest_test.go index f2e61280..6ef2af96 100644 --- a/vfs/tests/mptest/mptest_test.go +++ b/vfs/tests/mptest/mptest_test.go @@ -92,7 +92,7 @@ func system(ctx context.Context, mod api.Module, ptr uint32) uint32 { cfg := config(ctx).WithArgs(args...) go func() { - ctx := util.NewContext(ctx) + ctx := util.NewContext(ctx, true) mod, _ := rt.InstantiateModule(ctx, module, cfg) mod.Close(ctx) }() @@ -100,7 +100,7 @@ func system(ctx context.Context, mod api.Module, ptr uint32) uint32 { } func Test_config01(t *testing.T) { - ctx := util.NewContext(newContext(t)) + ctx := util.NewContext(newContext(t), false) name := filepath.Join(t.TempDir(), "test.db") cfg := config(ctx).WithArgs("mptest", name, "config01.test") mod, err := rt.InstantiateModule(ctx, module, cfg) @@ -118,7 +118,7 @@ func Test_config02(t *testing.T) { t.Skip("skipping in CI") } - ctx := util.NewContext(newContext(t)) + ctx := util.NewContext(newContext(t), false) name := filepath.Join(t.TempDir(), "test.db") cfg := config(ctx).WithArgs("mptest", name, "config02.test") mod, err := rt.InstantiateModule(ctx, module, cfg) @@ -136,7 +136,7 @@ func Test_crash01(t *testing.T) { t.Skip("skipping in CI") } - ctx := util.NewContext(newContext(t)) + ctx := util.NewContext(newContext(t), false) name := filepath.Join(t.TempDir(), "test.db") cfg := config(ctx).WithArgs("mptest", name, "crash01.test") mod, err := rt.InstantiateModule(ctx, module, cfg) @@ -151,7 +151,7 @@ func Test_multiwrite01(t *testing.T) { t.Skip("skipping in short mode") } - ctx := util.NewContext(newContext(t)) + ctx := util.NewContext(newContext(t), false) name := filepath.Join(t.TempDir(), "test.db") cfg := config(ctx).WithArgs("mptest", name, "multiwrite01.test") mod, err := rt.InstantiateModule(ctx, module, cfg) @@ -162,7 +162,7 @@ func Test_multiwrite01(t *testing.T) { } func Test_config01_memory(t *testing.T) { - ctx := util.NewContext(newContext(t)) + ctx := util.NewContext(newContext(t), false) cfg := config(ctx).WithArgs("mptest", "/test.db", "config01.test", "--vfs", "memdb") mod, err := rt.InstantiateModule(ctx, module, cfg) @@ -177,7 +177,7 @@ func Test_multiwrite01_memory(t *testing.T) { t.Skip("skipping in short mode") } - ctx := util.NewContext(newContext(t)) + ctx := util.NewContext(newContext(t), false) cfg := config(ctx).WithArgs("mptest", "/test.db", "multiwrite01.test", "--vfs", "memdb") mod, err := rt.InstantiateModule(ctx, module, cfg) @@ -187,6 +187,59 @@ func Test_multiwrite01_memory(t *testing.T) { mod.Close(ctx) } +func Test_config01_wal(t *testing.T) { + ctx := util.NewContext(newContext(t), true) + name := filepath.Join(t.TempDir(), "test.db") + cfg := config(ctx).WithArgs("mptest", name, "config01.test", + "--journalmode", "wal") + mod, err := rt.InstantiateModule(ctx, module, cfg) + if err != nil { + t.Fatal(err) + } + mod.Close(ctx) +} + +func Test_crash01_wal(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + if os.Getenv("CI") != "" { + t.Skip("skipping in CI") + } + if !vfs.SupportsSharedMemory { + t.Skip("skipping without shared memory") + } + + ctx := util.NewContext(newContext(t), true) + name := filepath.Join(t.TempDir(), "test.db") + cfg := config(ctx).WithArgs("mptest", name, "crash01.test", + "--journalmode", "wal") + mod, err := rt.InstantiateModule(ctx, module, cfg) + if err != nil { + t.Fatal(err) + } + mod.Close(ctx) +} + +func Test_multiwrite01_wal(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + if !vfs.SupportsSharedMemory { + t.Skip("skipping without shared memory") + } + + ctx := util.NewContext(newContext(t), true) + name := filepath.Join(t.TempDir(), "test.db") + cfg := config(ctx).WithArgs("mptest", name, "multiwrite01.test", + "--journalmode", "wal") + mod, err := rt.InstantiateModule(ctx, module, cfg) + if err != nil { + t.Fatal(err) + } + mod.Close(ctx) +} + func newContext(t *testing.T) context.Context { return context.WithValue(context.Background(), logger{}, &testWriter{T: t}) } diff --git a/vfs/tests/mptest/testdata/build.sh b/vfs/tests/mptest/testdata/build.sh index 4b9ce887..dc81aa02 100755 --- a/vfs/tests/mptest/testdata/build.sh +++ b/vfs/tests/mptest/testdata/build.sh @@ -21,7 +21,8 @@ WASI_SDK="$ROOT/tools/wasi-sdk-21.0/bin" -DSQLITE_DEFAULT_LOCKING_MODE=0 \ -DHAVE_USLEEP -DSQLITE_NO_SYNC \ -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ - -D_WASI_EMULATED_GETPID -lwasi-emulated-getpid + -D_WASI_EMULATED_GETPID -lwasi-emulated-getpid \ + -Wl,--export=aligned_alloc "$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \ mptest.wasm -o mptest.tmp \ diff --git a/vfs/tests/mptest/testdata/mptest.wasm.bz2 b/vfs/tests/mptest/testdata/mptest.wasm.bz2 index c233e589..0b753efa 100644 --- a/vfs/tests/mptest/testdata/mptest.wasm.bz2 +++ b/vfs/tests/mptest/testdata/mptest.wasm.bz2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b5f4778b49a6b99a1be11db5cb07e3c5b52f91a932a97e33c895f52c6f54bf57 -size 469149 +oid sha256:523b3640c6de9accf3f6b01e7da2098faba12eec0168b67a78c56fef0716c7ae +size 469261 diff --git a/vfs/tests/speedtest1/speedtest1_test.go b/vfs/tests/speedtest1/speedtest1_test.go index a0e59af3..786d6b95 100644 --- a/vfs/tests/speedtest1/speedtest1_test.go +++ b/vfs/tests/speedtest1/speedtest1_test.go @@ -84,7 +84,7 @@ func initFlags() { func Benchmark_speedtest1(b *testing.B) { output.Reset() - ctx := util.NewContext(context.Background()) + ctx := util.NewContext(context.Background(), false) name := filepath.Join(b.TempDir(), "test.db") args := append(options, "--size", strconv.Itoa(b.N), name) cfg := wazero.NewModuleConfig(). diff --git a/vfs/tests/speedtest1/testdata/build.sh b/vfs/tests/speedtest1/testdata/build.sh index 34f8ab87..36727930 100755 --- a/vfs/tests/speedtest1/testdata/build.sh +++ b/vfs/tests/speedtest1/testdata/build.sh @@ -16,7 +16,8 @@ WASI_SDK="$ROOT/tools/wasi-sdk-21.0/bin" -fno-stack-protector -fno-stack-clash-protection \ -Wl,--stack-first \ -Wl,--import-undefined \ - -D_HAVE_SQLITE_CONFIG_H + -D_HAVE_SQLITE_CONFIG_H \ + -Wl,--export=aligned_alloc "$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \ speedtest1.wasm -o speedtest1.tmp \ diff --git a/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 b/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 index 154992b2..e178f88b 100644 --- a/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 +++ b/vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2 @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:06882576fdea2c8e164dd2d97dce394dc825dca3cee30c9efc9601b84de92865 -size 483191 +oid sha256:855720cce2881c98d09c15eddf9cab0d5974a2a82f7f67987a28b97414629345 +size 483463 diff --git a/vfs/vfs.go b/vfs/vfs.go index 1e8761b4..d7577ed9 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -6,6 +6,7 @@ import ( "io" "net/url" "reflect" + "sync" "time" "github.com/ncruces/go-sqlite3/internal/util" @@ -27,7 +28,7 @@ func ExportHostFunctions(env wazero.HostModuleBuilder) wazero.HostModuleBuilder util.ExportFuncIIIII(env, "go_full_pathname", vfsFullPathname) util.ExportFuncIIII(env, "go_delete", vfsDelete) util.ExportFuncIIIII(env, "go_access", vfsAccess) - util.ExportFuncIIIIII(env, "go_open", vfsOpen) + util.ExportFuncIIIIIII(env, "go_open", vfsOpen) util.ExportFuncII(env, "go_close", vfsClose) util.ExportFuncIIIIJ(env, "go_read", vfsRead) util.ExportFuncIIIIJ(env, "go_write", vfsWrite) @@ -40,6 +41,10 @@ func ExportHostFunctions(env wazero.HostModuleBuilder) wazero.HostModuleBuilder util.ExportFuncIII(env, "go_lock", vfsLock) util.ExportFuncIII(env, "go_unlock", vfsUnlock) util.ExportFuncIII(env, "go_check_reserved_lock", vfsCheckReservedLock) + util.ExportFuncIIIIII(env, "go_shm_map", vfsShmMap) + util.ExportFuncIIIII(env, "go_shm_lock", vfsShmLock) + util.ExportFuncIII(env, "go_shm_unmap", vfsShmUnmap) + util.ExportFuncVI(env, "go_shm_barrier", vfsShmBarrier) return env } @@ -129,7 +134,7 @@ func vfsAccess(ctx context.Context, mod api.Module, pVfs, zPath uint32, flags Ac return vfsErrorCode(err, _IOERR_ACCESS) } -func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, flags OpenFlag, pOutFlags uint32) _ErrorCode { +func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, flags OpenFlag, pOutFlags, pOutVFS uint32) _ErrorCode { vfs := vfsGet(mod, pVfs) var path string @@ -165,6 +170,9 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla if pOutFlags != 0 { util.WriteUint32(mod, pOutFlags, uint32(flags)) } + if pOutVFS != 0 && util.CanMap(ctx) { + util.WriteUint32(mod, pOutVFS, 1) + } vfsFileRegister(ctx, mod, pFile, file) return _OK } @@ -348,6 +356,35 @@ func vfsDeviceCharacteristics(ctx context.Context, mod api.Module, pFile uint32) return file.DeviceCharacteristics() } +var shmBarrier sync.Mutex + +func vfsShmMap(ctx context.Context, mod api.Module, pFile, iRegion, szRegion, bExtend, pp uint32) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(fileShm) + p, err := file.shmMap(ctx, mod, iRegion, szRegion, bExtend != 0) + if err != nil { + return vfsErrorCode(err, _IOERR_SHMMAP) + } + util.WriteUint32(mod, pp, p) + return _OK +} + +func vfsShmLock(ctx context.Context, mod api.Module, pFile, offset, n uint32, flags _ShmFlag) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(fileShm) + err := file.shmLock(offset, n, flags) + return vfsErrorCode(err, _IOERR_SHMLOCK) +} + +func vfsShmUnmap(ctx context.Context, mod api.Module, pFile, bDelete uint32) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(fileShm) + file.shmUnmap(bDelete != 0) + return _OK +} + +func vfsShmBarrier(ctx context.Context, mod api.Module, pFile uint32) { + shmBarrier.Lock() + defer shmBarrier.Unlock() +} + func vfsURIParameters(ctx context.Context, mod api.Module, zPath uint32, flags OpenFlag) url.Values { if flags&OPEN_URI == 0 { return nil @@ -428,3 +465,9 @@ func vfsErrorCode(err error, def _ErrorCode) _ErrorCode { } return def } + +type fileShm interface { + shmMap(context.Context, api.Module, uint32, uint32, bool) (uint32, error) + shmLock(uint32, uint32, _ShmFlag) error + shmUnmap(bool) +} diff --git a/vfs/vfs_test.go b/vfs/vfs_test.go index 0729a62d..872ca07e 100644 --- a/vfs/vfs_test.go +++ b/vfs/vfs_test.go @@ -209,10 +209,10 @@ func Test_vfsAccess(t *testing.T) { func Test_vfsFile(t *testing.T) { mod := wazerotest.NewModule(wazerotest.NewMemory(wazerotest.PageSize)) - ctx := util.NewContext(context.TODO()) + ctx := util.NewContext(context.TODO(), false) // Open a temporary file. - rc := vfsOpen(ctx, mod, 0, 0, 4, OPEN_CREATE|OPEN_EXCLUSIVE|OPEN_READWRITE|OPEN_DELETEONCLOSE, 0) + rc := vfsOpen(ctx, mod, 0, 0, 4, OPEN_CREATE|OPEN_EXCLUSIVE|OPEN_READWRITE|OPEN_DELETEONCLOSE, 0, 0) if rc != _OK { t.Fatal("returned", rc) } @@ -281,10 +281,10 @@ func Test_vfsFile(t *testing.T) { func Test_vfsFile_psow(t *testing.T) { mod := wazerotest.NewModule(wazerotest.NewMemory(wazerotest.PageSize)) - ctx := util.NewContext(context.TODO()) + ctx := util.NewContext(context.TODO(), false) // Open a temporary file. - rc := vfsOpen(ctx, mod, 0, 0, 4, OPEN_CREATE|OPEN_EXCLUSIVE|OPEN_READWRITE|OPEN_DELETEONCLOSE, 0) + rc := vfsOpen(ctx, mod, 0, 0, 4, OPEN_CREATE|OPEN_EXCLUSIVE|OPEN_READWRITE|OPEN_DELETEONCLOSE, 0, 0) if rc != _OK { t.Fatal("returned", rc) }