Skip to content

Commit 07241d0

Browse files
ncrucesncruces
authored
Adiantum encrypting VFS improvements. (#80)
Encrypt temporary files.
1 parent 2c30bc9 commit 07241d0

File tree

17 files changed

+112
-59
lines changed

17 files changed

+112
-59
lines changed

Diff for: ‎embed/exports.txt

+1-3
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ sqlite3_create_collation_go
5151
sqlite3_create_function_go
5252
sqlite3_create_module_go
5353
sqlite3_create_window_function_go
54+
sqlite3_database_file_object
5455
sqlite3_db_config
5556
sqlite3_db_name
5657
sqlite3_db_readonly
@@ -61,9 +62,6 @@ sqlite3_errmsg
6162
sqlite3_error_offset
6263
sqlite3_errstr
6364
sqlite3_exec
64-
sqlite3_filename_database
65-
sqlite3_filename_journal
66-
sqlite3_filename_wal
6765
sqlite3_finalize
6866
sqlite3_get_autocommit
6967
sqlite3_get_auxdata

Diff for: ‎embed/sqlite3.wasm

-490 Bytes
Binary file not shown.

Diff for: ‎vfs/adiantum/README.md

+15-5
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,24 @@ In general, any HBSH construction can be used to wrap any VFS.
1616

1717
The default Adiantum construction uses XChaCha12 for its stream cipher,
1818
AES for its block cipher, and NH and Poly1305 for hashing.
19-
It uses Argon2id to derive 256-bit keys from plain text.
19+
Additionally, we use Argon2id to derive 256-bit keys from plain text.
2020

21-
The VFS encrypts database files, rollback and statement journals, and WAL files.
21+
The VFS encrypts all files _except_
22+
[super journals](https://sqlite.org/tempfiles.html#super_journal_files):
23+
these _never_ contain database data, only filenames,
24+
and padding them to the block size is problematic.
25+
26+
Temporary files _are_ encrypted with **random** keys,
27+
as they _may_ contain database data.
28+
To avoid the overhead of encrypting temporary files,
29+
keep them in memory:
30+
31+
PRAGMA temp_store = memory;
2232

2333
> [!IMPORTANT]
2434
> Adiantum is typically used for disk encryption.
2535
> The standard threat model for disk encryption considers an adversary
2636
> that can read multiple snapshots of a disk.
27-
> The security property that disk encryption provides is that
28-
> the only information such an adversary can determine is
29-
> whether the data in a sector has or has not changed over time.
37+
> The only security property that disk encryption (and this package)
38+
> provides is that the only information such an adversary can determine
39+
> is whether the data in a sector has or has not changed over time.

Diff for: ‎vfs/adiantum/adiantum.go

+7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package adiantum
22

33
import (
4+
"crypto/rand"
45
"sync"
56

67
"golang.org/x/crypto/argon2"
@@ -20,6 +21,12 @@ func (adiantumCreator) HBSH(key []byte) *hbsh.HBSH {
2021
}
2122

2223
func (adiantumCreator) KDF(text string) []byte {
24+
if text == "" {
25+
key := make([]byte, 32)
26+
n, _ := rand.Read(key)
27+
return key[:n]
28+
}
29+
2330
if key := keyCacheGet(text); key != nil {
2431
return key[:]
2532
}

Diff for: ‎vfs/adiantum/api.go

+7-5
Original file line numberDiff line numberDiff line change
@@ -33,18 +33,20 @@ func Register(name string, base vfs.VFS, cipher HBSHCreator) {
3333
if cipher == nil {
3434
cipher = adiantumCreator{}
3535
}
36-
vfs.Register("adiantum", &hbshVFS{
36+
vfs.Register(name, &hbshVFS{
3737
VFS: base,
3838
hbsh: cipher,
3939
})
4040
}
4141

42-
// HBSHCreator creates an [hbsh.HBSH] cipher,
42+
// HBSHCreator creates an [hbsh.HBSH] cipher
4343
// given key material.
4444
type HBSHCreator interface {
45-
// KDF maps a secret (text) to a key of the appropriate size.
46-
KDF(text string) (key []byte)
45+
// KDF derives an HBSH key from a secret.
46+
// If no secret is given, a random key is generated.
47+
KDF(secret string) (key []byte)
4748

48-
// HBSH creates an HBSH cipher given an appropriate key.
49+
// HBSH creates an HBSH cipher given a key.
50+
// If key is not appropriate, nil is returned.
4951
HBSH(key []byte) *hbsh.HBSH
5052
}

Diff for: ‎vfs/adiantum/hbsh.go

+7-10
Original file line numberDiff line numberDiff line change
@@ -18,25 +18,23 @@ type hbshVFS struct {
1818
}
1919

2020
func (h *hbshVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
21-
return h.OpenParams(name, flags, url.Values{})
21+
return h.OpenParams(name, flags, nil)
2222
}
2323

2424
func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values) (file vfs.File, _ vfs.OpenFlag, err error) {
25-
encrypt := flags&(0|
26-
vfs.OPEN_MAIN_DB|
27-
vfs.OPEN_MAIN_JOURNAL|
28-
vfs.OPEN_SUBJOURNAL|
29-
vfs.OPEN_WAL) != 0
30-
3125
var hbsh *hbsh.HBSH
32-
if encrypt {
26+
27+
// Encrypt everything except super journals.
28+
if flags&vfs.OPEN_SUPER_JOURNAL == 0 {
3329
var key []byte
3430
if t, ok := params["key"]; ok {
3531
key = []byte(t[0])
3632
} else if t, ok := params["hexkey"]; ok {
3733
key, _ = hex.DecodeString(t[0])
3834
} else if t, ok := params["textkey"]; ok {
3935
key = h.hbsh.KDF(t[0])
36+
} else if name == "" {
37+
key = h.hbsh.KDF("")
4038
}
4139

4240
if hbsh = h.hbsh.HBSH(key); hbsh == nil {
@@ -45,15 +43,14 @@ func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values)
4543
}
4644

4745
if h, ok := h.VFS.(vfs.VFSParams); ok {
48-
delete(params, "vfs")
4946
delete(params, "key")
5047
delete(params, "hexkey")
5148
delete(params, "textkey")
5249
file, flags, err = h.OpenParams(name, flags, params)
5350
} else {
5451
file, flags, err = h.Open(name, flags)
5552
}
56-
if err != nil || hbsh == nil {
53+
if err != nil || hbsh == nil || flags&vfs.OPEN_MEMORY != 0 {
5754
return file, flags, err
5855
}
5956
return &hbshFile{File: file, hbsh: hbsh}, flags, err

Diff for: ‎vfs/api.go

+9
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,15 @@ type VFSParams interface {
2424
OpenParams(name string, flags OpenFlag, params url.Values) (File, OpenFlag, error)
2525
}
2626

27+
// VFSJournal extends VFS with the ability to open journals
28+
// that need a reference to their corresponding database files.
29+
//
30+
// https://sqlite.org/c3ref/database_file_object.html
31+
type VFSJournal interface {
32+
VFS
33+
OpenJournal(name string, flags OpenFlag, db File) (File, OpenFlag, error)
34+
}
35+
2736
// A File represents an open file in the OS interface layer.
2837
//
2938
// Use sqlite3.ErrorCode or sqlite3.ExtendedErrorCode to return specific error codes to SQLite.

Diff for: ‎vfs/memdb/memdb.go

+12-5
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,22 @@ import (
1111
"github.com/ncruces/go-sqlite3/vfs"
1212
)
1313

14+
// Must be a multiple of 64K (the largest page size).
15+
const sectorSize = 65536
16+
1417
type memVFS struct{}
1518

1619
func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
17-
// Allowed file types:
20+
// For simplicity, we do not support reading or writing data
21+
// across "sector" boundaries.
22+
//
23+
// This is not a problem for most SQLite file types:
1824
// - databases, which only do page aligned reads/writes;
19-
// - temp journals, used by the sorter, which does the same.
25+
// - temp journals, as used by the sorter, which does the same:
26+
// https://sqlite.org/src/artifact/237840?ln=409-412
27+
//
28+
// We refuse to open all other file types,
29+
// but returning OPEN_MEMORY means SQLite won't ask us to.
2030
const types = vfs.OPEN_MAIN_DB |
2131
vfs.OPEN_TRANSIENT_DB |
2232
vfs.OPEN_TEMP_DB |
@@ -61,9 +71,6 @@ func (memVFS) FullPathname(name string) (string, error) {
6171
return name, nil
6272
}
6373

64-
// Must be a multiple of 64K (the largest page size).
65-
const sectorSize = 65536
66-
6774
type memDB struct {
6875
// +checklocks:lockMtx
6976
pending *memFile

Diff for: ‎vfs/registry.go

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ func Find(name string) VFS {
2323
}
2424

2525
// Register registers a VFS.
26+
// Empty and "os" are reserved names.
2627
//
2728
// https://sqlite.org/c3ref/vfs_find.html
2829
func Register(name string, vfs VFS) {

Diff for: ‎vfs/tests/mptest/testdata/build.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
1616
-fno-stack-protector -fno-stack-clash-protection \
1717
-Wl,--stack-first \
1818
-Wl,--import-undefined \
19-
-D_HAVE_SQLITE_CONFIG_H -DHAVE_USLEEP \
19+
-D_HAVE_SQLITE_CONFIG_H -DSQLITE_USE_URI \
2020
-DSQLITE_DEFAULT_SYNCHRONOUS=0 \
2121
-DSQLITE_DEFAULT_LOCKING_MODE=0 \
2222
-DSQLITE_NO_SYNC -DSQLITE_THREADSAFE=0 \
23-
-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_USE_URI \
23+
-DSQLITE_OMIT_LOAD_EXTENSION -DHAVE_USLEEP \
2424
-D_WASI_EMULATED_GETPID -lwasi-emulated-getpid \
2525
$(awk '{print "-Wl,--export="$0}' exports.txt)
2626

Diff for: ‎vfs/tests/mptest/testdata/exports.txt

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
aligned_alloc
22
free
33
malloc
4-
sqlite3_filename_database
5-
sqlite3_filename_journal
6-
sqlite3_filename_wal
4+
sqlite3_database_file_object
75
sqlite3_uri_key
86
sqlite3_uri_parameter

Diff for: ‎vfs/tests/mptest/testdata/mptest.wasm.bz2

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
version https://git-lfs.github.com/spec/v1
2-
oid sha256:35d27c1ec2c14d82ee6757d768435dde8dd0324ed6963fb00889a44364e19071
3-
size 470365
2+
oid sha256:afe1db5aea2a3ab996370fa85052cafaae10ab4b2f8154885da2c1d2a8503840
3+
size 470240

Diff for: ‎vfs/tests/speedtest1/speedtest1_test.go

+21-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
"github.com/ncruces/go-sqlite3/internal/util"
2323
"github.com/ncruces/go-sqlite3/vfs"
24+
_ "github.com/ncruces/go-sqlite3/vfs/adiantum"
2425
_ "github.com/ncruces/go-sqlite3/vfs/memdb"
2526
)
2627

@@ -38,8 +39,7 @@ func TestMain(m *testing.M) {
3839
initFlags()
3940

4041
ctx := context.Background()
41-
cfg := wazero.NewRuntimeConfig().WithMemoryLimitPages(1024)
42-
rt = wazero.NewRuntimeWithConfig(ctx, cfg)
42+
rt = wazero.NewRuntime(ctx)
4343
wasi_snapshot_preview1.MustInstantiate(ctx, rt)
4444
env := vfs.ExportHostFunctions(rt.NewHostModuleBuilder("env"))
4545
_, err := env.Instantiate(ctx)
@@ -100,3 +100,22 @@ func Benchmark_speedtest1(b *testing.B) {
100100
}
101101
mod.Close(ctx)
102102
}
103+
104+
func Benchmark_adiantum(b *testing.B) {
105+
output.Reset()
106+
ctx := util.NewContext(context.Background(), true)
107+
name := "file:" + filepath.Join(b.TempDir(), "test.db") +
108+
"?textkey=correct+horse+battery+staple"
109+
args := append(options, "--vfs", "adiantum", "--size", strconv.Itoa(b.N), name)
110+
cfg := wazero.NewModuleConfig().
111+
WithArgs(args...).WithName("speedtest1").
112+
WithStdout(&output).WithStderr(&output).
113+
WithSysWalltime().WithSysNanotime().WithSysNanosleep().
114+
WithOsyield(runtime.Gosched).
115+
WithRandSource(rand.Reader)
116+
mod, err := rt.InstantiateModule(ctx, module, cfg)
117+
if err != nil {
118+
b.Fatal(err)
119+
}
120+
mod.Close(ctx)
121+
}

Diff for: ‎vfs/tests/speedtest1/testdata/build.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
1616
-fno-stack-protector -fno-stack-clash-protection \
1717
-Wl,--stack-first \
1818
-Wl,--import-undefined \
19-
-D_HAVE_SQLITE_CONFIG_H \
20-
-Wl,--export=aligned_alloc
19+
-D_HAVE_SQLITE_CONFIG_H -DSQLITE_USE_URI \
20+
$(awk '{print "-Wl,--export="$0}' exports.txt)
2121

2222
"$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \
2323
speedtest1.wasm -o speedtest1.tmp \

Diff for: ‎vfs/tests/speedtest1/testdata/exports.txt

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
aligned_alloc
2+
free
3+
malloc
4+
sqlite3_database_file_object
5+
sqlite3_uri_key
6+
sqlite3_uri_parameter

Diff for: ‎vfs/tests/speedtest1/testdata/speedtest1.wasm.bz2

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
version https://git-lfs.github.com/spec/v1
2-
oid sha256:7f669eede3ba9c9a104d37c3a90ecec26086a87aea54c1f650df202923d75cb9
3-
size 483426
2+
oid sha256:e4f9ed81c9497a9d8b91517416d122ae04c2c517d3e0612d869e128dcee3fa81
3+
size 483519

Diff for: ‎vfs/vfs.go

+17-18
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,10 @@ func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, fla
143143
var err error
144144
var parsed bool
145145
var params url.Values
146-
if pfs, ok := vfs.(VFSParams); ok {
146+
if jfs, ok := vfs.(VFSJournal); ok && flags&(OPEN_WAL|OPEN_MAIN_JOURNAL) != 0 {
147+
db := vfsDatabaseFileObject(ctx, mod, zPath)
148+
file, flags, err = jfs.OpenJournal(path, flags, db)
149+
} else if pfs, ok := vfs.(VFSParams); ok {
147150
parsed = true
148151
params = vfsURIParameters(ctx, mod, zPath, flags)
149152
file, flags, err = pfs.OpenParams(path, flags, params)
@@ -387,30 +390,17 @@ func vfsShmUnmap(ctx context.Context, mod api.Module, pFile, bDelete uint32) _Er
387390
func vfsURIParameters(ctx context.Context, mod api.Module, zPath uint32, flags OpenFlag) url.Values {
388391
switch {
389392
case flags&(OPEN_URI|OPEN_MAIN_DB) == OPEN_URI|OPEN_MAIN_DB:
390-
// database file
391-
case flags&(OPEN_MAIN_JOURNAL|OPEN_SUBJOURNAL|OPEN_SUPER_JOURNAL|OPEN_WAL) != 0:
393+
// database file with URI
394+
case flags&(OPEN_WAL|OPEN_MAIN_JOURNAL) != 0:
392395
// journal or WAL file
393396
default:
394397
return nil
395398
}
396399

397-
nameDB := mod.ExportedFunction("sqlite3_filename_database")
398-
uriKey := mod.ExportedFunction("sqlite3_uri_key")
399-
uriParam := mod.ExportedFunction("sqlite3_uri_parameter")
400-
if nameDB == nil || uriKey == nil || uriParam == nil {
401-
return nil
402-
}
403-
404400
var stack [2]uint64
405401
var params url.Values
406-
407-
if flags&OPEN_MAIN_DB == 0 {
408-
stack[0] = uint64(zPath)
409-
if err := nameDB.CallWithStack(ctx, stack[:]); err != nil {
410-
panic(err)
411-
}
412-
zPath = uint32(stack[0])
413-
}
402+
uriKey := mod.ExportedFunction("sqlite3_uri_key")
403+
uriParam := mod.ExportedFunction("sqlite3_uri_parameter")
414404

415405
for i := 0; ; i++ {
416406
stack[1] = uint64(i)
@@ -438,6 +428,15 @@ func vfsURIParameters(ctx context.Context, mod api.Module, zPath uint32, flags O
438428
}
439429
}
440430

431+
func vfsDatabaseFileObject(ctx context.Context, mod api.Module, zPath uint32) File {
432+
stack := [...]uint64{uint64(zPath)}
433+
fn := mod.ExportedFunction("sqlite3_database_file_object")
434+
if err := fn.CallWithStack(ctx, stack[:]); err != nil {
435+
panic(err)
436+
}
437+
return vfsFileGet(ctx, mod, uint32(stack[0]))
438+
}
439+
441440
func vfsGet(mod api.Module, pVfs uint32) VFS {
442441
var name string
443442
if pVfs != 0 {

0 commit comments

Comments
 (0)