From 4b816470655a21ee0e5567f21b7f5d954d3d536b Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Wed, 5 Aug 2026 15:27:03 +0100 Subject: [PATCH 1/2] Add bloom filter to improve miss performance --- LICENSE | 208 ++++++++++++++++++++++++++++++++++++-- bench/.gitignore | 1 - bench/bench.go | 1 - bench/bench_test.go | 237 -------------------------------------------- bench/go.mod | 18 ---- bench/go.sum | 35 ------- bench_test.go | 60 +++++++++++ bloom.go | 123 +++++++++++++++++++++++ bloom_test.go | 55 ++++++++++ reader.go | 61 +++++++++--- reader_test.go | 72 +++++++++++++- sntable.go | 7 ++ sntable_test.go | 2 +- writer.go | 67 +++++++++++-- 14 files changed, 620 insertions(+), 327 deletions(-) delete mode 100644 bench/.gitignore delete mode 100644 bench/bench.go delete mode 100644 bench/bench_test.go delete mode 100644 bench/go.mod delete mode 100644 bench/go.sum create mode 100644 bench_test.go create mode 100644 bloom.go create mode 100644 bloom_test.go diff --git a/LICENSE b/LICENSE index 73d55a5..f0ad7a2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,201 @@ -Copyright 2017 Black Square Media Ltd + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - http://www.apache.org/licenses/LICENSE-2.0 + 1. Definitions. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative + Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, excluding + those notices that do not pertain to any part of the Derivative + Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and do + not modify the License. You may add Your own attribution notices + within Derivative Works that You distribute, alongside or as an + addendum to the NOTICE text from the Work, provided that such + additional attribution notices cannot be construed as modifying + the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Black Square Media Ltd + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bench/.gitignore b/bench/.gitignore deleted file mode 100644 index 8de83f2..0000000 --- a/bench/.gitignore +++ /dev/null @@ -1 +0,0 @@ -seed.* diff --git a/bench/bench.go b/bench/bench.go deleted file mode 100644 index 354df9d..0000000 --- a/bench/bench.go +++ /dev/null @@ -1 +0,0 @@ -package bench diff --git a/bench/bench_test.go b/bench/bench_test.go deleted file mode 100644 index 670b76e..0000000 --- a/bench/bench_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package bench_test - -import ( - "encoding/binary" - "fmt" - "math/rand" - "os" - "testing" - - "github.com/bsm/sntable" - "github.com/golang/leveldb/db" - leveldb "github.com/golang/leveldb/table" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" - goleveldb "github.com/syndtr/goleveldb/leveldb/table" - "github.com/syndtr/goleveldb/leveldb/util" -) - -func Benchmark(b *testing.B) { - b.Run("bsm/sntable 10M plain", func(b *testing.B) { - benchSnTable(b, 10e6, false) - }) - b.Run("golang/leveldb 10M plain", func(b *testing.B) { - benchLevelDB(b, 10e6, false) - }) - b.Run("syndtr/goleveldb 10M plain", func(b *testing.B) { - benchGoLevelDB(b, 10e6, false) - }) - - b.Run("bsm/sntable 10M snappy", func(b *testing.B) { - benchSnTable(b, 10e6, true) - }) - b.Run("golang/leveldb 10M snappy", func(b *testing.B) { - benchLevelDB(b, 10e6, true) - }) - b.Run("syndtr/goleveldb 10M snappy", func(b *testing.B) { - benchGoLevelDB(b, 10e6, true) - }) -} - -func benchSnTable(b *testing.B, numSeeds int, compress bool) { - fname := createSeedFile(b, "sntable", numSeeds, compress, func(f *os.File) error { - o := &sntable.WriterOptions{ - BlockSize: 8 * 1024, - BlockRestartInterval: 1024, - Compression: sntable.NoCompression, - } - if compress { - o.Compression = sntable.SnappyCompression - } - w := sntable.NewWriter(f, o) - defer w.Close() - - eachKVPair(b, numSeeds, func(num uint64, val []byte) error { - return w.Append(num, val) - }) - - return w.Close() - }) - - openSeedFile(b, fname, func(file *os.File, size int64) error { - read, err := sntable.NewReader(file, size) - if err != nil { - b.Fatal(err) - } - - sink := make([]byte, 0, 256) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - key := uint64(i % (2 * numSeeds)) - _, err := read.Append(sink[:0], key) - if err != nil && err != sntable.ErrNotFound { - b.Fatal(err) - } - } - return nil - }) -} - -func benchLevelDB(b *testing.B, numSeeds int, compress bool) { - fname := createSeedFile(b, "leveldb", numSeeds, compress, func(f *os.File) error { - o := &db.Options{ - BlockSize: 8 * 1024, - BlockRestartInterval: 1024, - Compression: db.NoCompression, - WriteBufferSize: 64 * 1024 * 1024, - } - if compress { - o.Compression = db.SnappyCompression - } - w := leveldb.NewWriter(f, o) - defer w.Close() - - eachKVPair(b, numSeeds, func(num uint64, val []byte) error { - key := make([]byte, 8) - binary.BigEndian.PutUint64(key, num) - return w.Set(key, val, nil) - }) - - return w.Close() - }) - - openSeedFile(b, fname, func(file *os.File, _ int64) error { - read := leveldb.NewReader(file, nil) - defer read.Close() - - key := make([]byte, 8) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - binary.BigEndian.PutUint64(key, uint64(i%(2*numSeeds))) - _, err := read.Get(key, nil) - if err != nil && err != db.ErrNotFound { - b.Fatal(err) - } - } - return nil - }) -} - -func benchGoLevelDB(b *testing.B, numSeeds int, compress bool) { - opts := opt.Options{ - DisableBlockCache: true, - BlockCacher: opt.NoCacher, - BlockSize: 8 * 1024, - BlockRestartInterval: 1024, - Compression: opt.NoCompression, - WriteBuffer: 64 * 1024 * 1024, - Strict: opt.NoStrict, - } - if compress { - opts.Compression = opt.SnappyCompression - } - - fname := createSeedFile(b, "goleveldb", numSeeds, compress, func(f *os.File) error { - w := goleveldb.NewWriter(f, &opts) - defer w.Close() - - eachKVPair(b, numSeeds, func(num uint64, val []byte) error { - key := make([]byte, 8) - binary.BigEndian.PutUint64(key, num) - return w.Append(key, val) - }) - - return w.Close() - }) - - openSeedFile(b, fname, func(file *os.File, size int64) error { - pool := util.NewBufferPool(opts.BlockSize) - defer pool.Close() - - read, err := goleveldb.NewReader(file, size, storage.FileDesc{}, nil, pool, &opts) - if err != nil { - b.Fatal(err) - } - defer read.Release() - - key := make([]byte, 8) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - binary.BigEndian.PutUint64(key, uint64(i%(2*numSeeds))) - val, err := read.Get(key, nil) - if err != nil && err != goleveldb.ErrNotFound { - b.Fatal(err) - } else if val != nil { - pool.Put(val) - } - } - return nil - }) -} - -// -------------------------------------------------------------------- - -func createSeedFile(b *testing.B, prefix string, numSeeds int, compress bool, cb func(*os.File) error) string { - b.Helper() - - suffix := "plain" - if compress { - suffix = "snappy" - } - fname := fmt.Sprintf("seed.%s.%d.%s", prefix, numSeeds, suffix) - if _, err := os.Stat(fname); err == nil { - return fname - } else if !os.IsNotExist(err) { - b.Fatal(err) - } - - f, err := os.Create(fname) - if err != nil { - b.Fatal(err) - } - defer f.Close() - - if err := cb(f); err != nil { - b.Fatal(err) - } - return fname -} - -func openSeedFile(b *testing.B, fname string, cb func(*os.File, int64) error) { - b.Helper() - - file, err := os.Open(fname) - if err != nil { - b.Fatal(err) - } - - stat, err := file.Stat() - if err != nil { - b.Fatal(err) - } - - if err := cb(file, stat.Size()); err != nil { - b.Fatal(err) - } - - b.StopTimer() -} - -func eachKVPair(b *testing.B, numSeeds int, cb func(uint64, []byte) error) { - b.Helper() - - rnd := rand.New(rand.NewSource(33)) - val := make([]byte, 128) - - for i := 0; i < numSeeds*2; i += 2 { - if _, err := rnd.Read(val); err != nil { - b.Fatal(err) - } - if err := cb(uint64(i), val); err != nil { - b.Fatal(err) - } - } -} diff --git a/bench/go.mod b/bench/go.mod deleted file mode 100644 index 91d0de5..0000000 --- a/bench/go.mod +++ /dev/null @@ -1,18 +0,0 @@ -module github.com/bsm/sntable/bench - -go 1.26 - -replace github.com/bsm/sntable => ../ - -require ( - github.com/bsm/sntable v0.1.1 - github.com/golang/leveldb v0.0.0-20170107010102-259d9253d719 - github.com/syndtr/goleveldb v1.0.0 -) - -require ( - github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect - github.com/klauspost/compress v1.19.0 // indirect - golang.org/x/net v0.0.0-20190607181551-461777fb6f67 // indirect - golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76 // indirect -) diff --git a/bench/go.sum b/bench/go.sum deleted file mode 100644 index 1b7be74..0000000 --- a/bench/go.sum +++ /dev/null @@ -1,35 +0,0 @@ -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/golang/leveldb v0.0.0-20170107010102-259d9253d719 h1:yahFtfWlyALYDkXw2ETowZqG4vi8hiE0yOEBOkpaXl0= -github.com/golang/leveldb v0.0.0-20170107010102-259d9253d719/go.mod h1:etEpE0xVqxA0N3WNUa5wic5HCNSsQvYm+PFNmOnx2iU= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190607181551-461777fb6f67 h1:rJJxsykSlULwd2P2+pg/rtnwN2FrWp4IuCxOSyS0V00= -golang.org/x/net v0.0.0-20190607181551-461777fb6f67/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76 h1:QSmW7Q3mFdAGjtAd0byXmFJ55inUydyZ4WQmiuItAIQ= -golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/bench_test.go b/bench_test.go new file mode 100644 index 0000000..88a8b27 --- /dev/null +++ b/bench_test.go @@ -0,0 +1,60 @@ +package sntable_test + +import ( + "bytes" + "testing" + + "github.com/bsm/sntable" +) + +// Benchmark_miss is the important one: it shows the bloom filter turning an +// absent-key lookup from a full block read+decompress into an in-memory bit +// check. Benchmark_hit shows the filter adds ~nothing on the hit path. +func Benchmark_bloom(b *testing.B) { + const n = 200_000 + + build := func(o *sntable.WriterOptions) *sntable.Reader { + var buf bytes.Buffer + w := sntable.NewWriter(&buf, o) + val := make([]byte, 16) + for i := range n { + if err := w.Append(uint64(i*2), val); err != nil { + b.Fatal(err) + } + } + if err := w.Close(); err != nil { + b.Fatal(err) + } + r, err := sntable.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + if err != nil { + b.Fatal(err) + } + return r + } + + variants := []struct { + name string + opts *sntable.WriterOptions + }{ + {"nobloom-snappy", &sntable.WriterOptions{Compression: sntable.SnappyCompression}}, + {"nobloom-plain", &sntable.WriterOptions{Compression: sntable.NoCompression}}, + {"bloom-snappy", &sntable.WriterOptions{Compression: sntable.SnappyCompression, BloomBitsPerKey: 10, BloomExpectedKeys: n}}, + {"bloom-plain", &sntable.WriterOptions{Compression: sntable.NoCompression, BloomBitsPerKey: 10, BloomExpectedKeys: n}}, + } + + for _, v := range variants { + r := build(v.opts) + var dst []byte + + b.Run(v.name+"/miss", func(b *testing.B) { + for i := 0; i < b.N; i++ { + dst, _ = r.Append(dst[:0], uint64((i%n)*2+1)) // odd = absent + } + }) + b.Run(v.name+"/hit", func(b *testing.B) { + for i := 0; i < b.N; i++ { + dst, _ = r.Append(dst[:0], uint64((i%n)*2)) // even = present + } + }) + } +} diff --git a/bloom.go b/bloom.go new file mode 100644 index 0000000..403e3e1 --- /dev/null +++ b/bloom.go @@ -0,0 +1,123 @@ +package sntable + +import ( + "encoding/binary" + "io" + "math" +) + +// bloomFilter is a classic bloom filter over uint64 keys. sntable keys are +// already integers, so there is no variable-length key to hash first - a fast +// integer bit-mix is enough to spread the bits. +// +// The filter is stored in the table file and loaded fully into memory when the +// reader opens. Its whole purpose is to answer "definitely absent" without +// touching the data blocks: on the miss-heavy lookup patterns this library is +// built for, most Get calls never read a block at all. +type bloomFilter struct { + bits []uint64 + k int // number of hash probes + nbits uint64 // len(bits)*64, cached +} + +// newBloomFilter sizes a filter for an expected number of keys at the given +// bits-per-key. 10 bits/key gives a ~1% false-positive rate. The estimate only +// affects accuracy: too low degrades the false-positive rate but never breaks +// correctness (a bloom filter has no false negatives). +func newBloomFilter(expectedKeys, bitsPerKey int) *bloomFilter { + if expectedKeys < 1 { + expectedKeys = 1 + } + if bitsPerKey < 1 { + bitsPerKey = 10 + } + + nwords := max((uint64(expectedKeys)*uint64(bitsPerKey)+63)/64, 1) + + k := int(float64(bitsPerKey)*math.Ln2 + 0.5) + if k < 1 { + k = 1 + } else if k > 30 { + k = 30 + } + + return &bloomFilter{bits: make([]uint64, nwords), k: k, nbits: nwords * 64} +} + +// bloomMix is splitmix64: a fast, well-spread integer hash. Keys that are +// already hashes still benefit, since low bits of a caller's hash may be weak. +func bloomMix(x uint64) uint64 { + x ^= x >> 30 + x *= 0xbf58476d1ce4e5b9 + x ^= x >> 27 + x *= 0x94d049bb133111eb + x ^= x >> 31 + return x +} + +// probe positions use double hashing: pos_i = h1 + i*h2. Both derive from one +// mix, so add() and has() must compute them identically. +func (f *bloomFilter) hashes(key uint64) (h1, h2 uint64) { + h1 = bloomMix(key) + h2 = bloomMix(key+0x9e3779b97f4a7c15) | 1 // odd, so it strides all words + return +} + +func (f *bloomFilter) add(key uint64) { + h1, h2 := f.hashes(key) + for i := 0; i < f.k; i++ { + p := (h1 + uint64(i)*h2) % f.nbits + f.bits[p>>6] |= 1 << (p & 63) + } +} + +func (f *bloomFilter) has(key uint64) bool { + h1, h2 := f.hashes(key) + for i := 0; i < f.k; i++ { + p := (h1 + uint64(i)*h2) % f.nbits + if f.bits[p>>6]&(1<<(p&63)) == 0 { + return false + } + } + return true +} + +// on-disk layout: [k uint32][nwords uint64][words... uint64 LE]. +const bloomHeaderLen = 12 + +func (f *bloomFilter) writeTo(w io.Writer) (int, error) { + hdr := make([]byte, bloomHeaderLen) + binary.LittleEndian.PutUint32(hdr[0:], uint32(f.k)) + binary.LittleEndian.PutUint64(hdr[4:], uint64(len(f.bits))) + + body := make([]byte, 8*len(f.bits)) + for i, word := range f.bits { + binary.LittleEndian.PutUint64(body[i*8:], word) + } + + n1, err := w.Write(hdr) + if err != nil { + return n1, err + } + n2, err := w.Write(body) + return n1 + n2, err +} + +func readBloomFilter(r io.ReaderAt, off int64) (*bloomFilter, error) { + hdr := make([]byte, bloomHeaderLen) + if _, err := r.ReadAt(hdr, off); err != nil { + return nil, err + } + k := int(binary.LittleEndian.Uint32(hdr[0:])) + nwords := binary.LittleEndian.Uint64(hdr[4:]) + + body := make([]byte, 8*nwords) + if _, err := r.ReadAt(body, off+bloomHeaderLen); err != nil { + return nil, err + } + bits := make([]uint64, nwords) + for i := range bits { + bits[i] = binary.LittleEndian.Uint64(body[i*8:]) + } + return &bloomFilter{bits: bits, k: k, nbits: nwords * 64}, nil +} diff --git a/bloom_test.go b/bloom_test.go new file mode 100644 index 0000000..cd9fb68 --- /dev/null +++ b/bloom_test.go @@ -0,0 +1,55 @@ +package sntable + +import ( + "bytes" + "testing" +) + +func TestBloomFilter_roundtrip(t *testing.T) { + f := newBloomFilter(1000, 10) + for i := range uint64(1000) { + f.add(i * 7) + } + for i := range uint64(1000) { + if !f.has(i * 7) { + t.Fatalf("false negative for %d", i*7) + } + } + + // false-positive rate should be in the right ballpark for 10 bits/key + var fp int + for i := range uint64(100_000) { + key := i*7 + 3 // never a multiple of 7 + if f.has(key) { + fp++ + } + } + if rate := float64(fp) / 100_000; rate > 0.03 { + t.Fatalf("false-positive rate too high: %.4f", rate) + } +} + +func TestBloomFilter_serialisation(t *testing.T) { + f := newBloomFilter(500, 12) + for i := range uint64(500) { + f.add(i) + } + + var buf bytes.Buffer + if _, err := f.writeTo(&buf); err != nil { + t.Fatal(err) + } + + got, err := readBloomFilter(bytes.NewReader(buf.Bytes()), 0) + if err != nil { + t.Fatal(err) + } + if got.k != f.k || got.nbits != f.nbits { + t.Fatalf("header mismatch: k=%d nbits=%d, want k=%d nbits=%d", got.k, got.nbits, f.k, f.nbits) + } + for i := range uint64(500) { + if !got.has(i) { + t.Fatalf("false negative after reload for %d", i) + } + } +} diff --git a/reader.go b/reader.go index 910d79b..a8a3f79 100644 --- a/reader.go +++ b/reader.go @@ -15,28 +15,32 @@ type Reader struct { r io.ReaderAt index []blockInfo - maxOffset int64 + maxOffset int64 // exclusive upper bound of the last data block + bloom *bloomFilter // nil unless the file carries one } // NewReader opens a reader. func NewReader(r io.ReaderAt, size int64) (*Reader, error) { - tmp := make([]byte, 16+binary.MaxVarintLen64) - - // read footer - footerOffset := size - 16 - if _, err := r.ReadAt(tmp[:16], footerOffset); err != nil { + indexOffset, bloomOffset, footerOffset, err := readFooter(r, size) + if err != nil { return nil, err } - // parse footer - if !bytes.Equal(tmp[8:16], magic) { - return nil, errBadMagic + // The bloom section, when present, sits between the last data block and the + // index, so it bounds the last block instead of the index doing so. + maxOffset := indexOffset + var bloom *bloomFilter + if bloomOffset > 0 { + maxOffset = bloomOffset + if bloom, err = readBloomFilter(r, bloomOffset); err != nil { + return nil, err + } } - indexOffset := int64(binary.LittleEndian.Uint64(tmp[:8])) // read index var index []blockInfo var info blockInfo + tmp := make([]byte, 2*binary.MaxVarintLen64) for pos := indexOffset; pos < footerOffset; { tmp = tmp[:2*binary.MaxVarintLen64] @@ -44,8 +48,7 @@ func NewReader(r io.ReaderAt, size int64) (*Reader, error) { tmp = tmp[:int(x)] } - _, err := r.ReadAt(tmp, pos) - if err != nil { + if _, err := r.ReadAt(tmp, pos); err != nil { return nil, err } @@ -64,10 +67,36 @@ func NewReader(r io.ReaderAt, size int64) (*Reader, error) { r: r, index: index, // block offsets - maxOffset: indexOffset, + maxOffset: maxOffset, + bloom: bloom, }, nil } +// readFooter reads the trailing footer and returns the index offset, the bloom +// offset (0 if absent), and the offset at which the footer begins. It accepts +// both the 16-byte v1 footer and the 24-byte v2 (bloom) footer. +func readFooter(r io.ReaderAt, size int64) (indexOffset, bloomOffset, footerOffset int64, err error) { + // Read up to the larger footer; a v2 footer is 24 bytes, a v1 footer 16. + n := min(int64(24), size) + buf := make([]byte, n) + if _, err = r.ReadAt(buf, size-n); err != nil { + return 0, 0, 0, err + } + + // v2: 24-byte footer ending in magicBloom. + if n >= 24 && bytes.Equal(buf[n-8:], magicBloom) { + indexOffset = int64(binary.LittleEndian.Uint64(buf[n-24:])) + bloomOffset = int64(binary.LittleEndian.Uint64(buf[n-16:])) + return indexOffset, bloomOffset, size - 24, nil + } + // v1: 16-byte footer ending in magic. + if n >= 16 && bytes.Equal(buf[n-8:], magic) { + indexOffset = int64(binary.LittleEndian.Uint64(buf[n-16:])) + return indexOffset, 0, size - 16, nil + } + return 0, 0, 0, errBadMagic +} + // NumBlocks returns the number of stored blocks. func (r *Reader) NumBlocks() int { return len(r.index) @@ -77,6 +106,12 @@ func (r *Reader) NumBlocks() int { // appends it to dst instead of allocating a new byte slice. // It may return an ErrNotFound error. func (r *Reader) Append(dst []byte, key uint64) ([]byte, error) { + // A present bloom filter rejects most absent keys here, before any block is + // read or decompressed. This is the main payoff of the filter. + if r.bloom != nil && !r.bloom.has(key) { + return dst, ErrNotFound + } + iter, err := r.Seek(key) if err != nil { return dst, err diff --git a/reader_test.go b/reader_test.go index 36be7ff..65bc99a 100644 --- a/reader_test.go +++ b/reader_test.go @@ -1,6 +1,7 @@ package sntable_test import ( + "bytes" "fmt" "testing" @@ -143,7 +144,7 @@ func TestReader(t *testing.T) { equal(t, section.Key(), uint64(192)) hasSuffix(t, section.Value(), "0192") - for i := 0; i < 12; i++ { + for range 12 { isTrue(t, section.More()) isTrue(t, section.Next()) } @@ -176,7 +177,7 @@ func TestReader(t *testing.T) { equal(t, iter.Key(), uint64(4)) hasSuffix(t, iter.Value(), "0004") - for i := 0; i < 97; i++ { + for range 97 { isTrue(t, iter.More()) isTrue(t, iter.Next()) } @@ -226,3 +227,70 @@ func TestReader(t *testing.T) { }) }) } + +// buildBloomTable writes n keys (0,2,4,...) with the given options and returns +// a reader. +func buildBloomTable(t *testing.T, n int, o *sntable.WriterOptions) *sntable.Reader { + t.Helper() + + var buf bytes.Buffer + w := sntable.NewWriter(&buf, o) + for i := range n { + key := uint64(i * 2) // only even keys are present + if err := w.Append(key, []byte{byte(i), byte(i >> 8)}); err != nil { + t.Fatal(err) + } + } + ok(t, w.Close()) + + r, err := sntable.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + ok(t, err) + return r +} + +func TestReader_withBloom(t *testing.T) { + const n = 5000 + opts := &sntable.WriterOptions{ + Compression: sntable.NoCompression, + BloomBitsPerKey: 10, + BloomExpectedKeys: n, + } + r := buildBloomTable(t, n, opts) + + // every present (even) key resolves + for i := range n { + val, err := r.Get(uint64(i * 2)) + ok(t, err) + equal(t, val[0], byte(i)) + } + + // odd keys are absent + for i := range n { + _, err := r.Get(uint64(i*2 + 1)) + errIs(t, err, sntable.ErrNotFound) + } + + // a key past the end is absent + _, err := r.Get(uint64(n * 4)) + errIs(t, err, sntable.ErrNotFound) +} + +// TestReader_bloomMatchesNoBloom asserts the filter changes performance, not +// results: the same query set gives identical answers with and without it. +func TestReader_bloomMatchesNoBloom(t *testing.T) { + const n = 2000 + plain := buildBloomTable(t, n, &sntable.WriterOptions{Compression: sntable.NoCompression}) + withBloom := buildBloomTable(t, n, &sntable.WriterOptions{ + Compression: sntable.NoCompression, + BloomBitsPerKey: 10, + BloomExpectedKeys: n, + }) + + for key := range uint64(n*2 + 50) { + vp, ep := plain.Get(key) + vb, eb := withBloom.Get(key) + if (ep == nil) != (eb == nil) || !bytes.Equal(vp, vb) { + t.Fatalf("mismatch at key %d: plain=(%v,%v) bloom=(%v,%v)", key, vp, ep, vb, eb) + } + } +} diff --git a/sntable.go b/sntable.go index f8a0b0f..029e414 100644 --- a/sntable.go +++ b/sntable.go @@ -6,8 +6,15 @@ import ( "github.com/klauspost/compress/zstd" ) +// magic terminates a v1 footer (16 bytes: indexOffset uint64 + magic). var magic = []byte{71, 39, 134, 190, 31, 122, 101, 219} +// magicBloom terminates a v2 footer (24 bytes: indexOffset uint64 + +// bloomOffset uint64 + magicBloom). A v2 footer is only written when a bloom +// filter is present, so files without a filter stay byte-identical to v1 and +// older readers keep working. +var magicBloom = []byte{71, 39, 134, 190, 31, 122, 101, 220} + const ( blockNoCompression = 0 blockSnappyCompression = 1 diff --git a/sntable_test.go b/sntable_test.go index 02c4762..71d10c7 100644 --- a/sntable_test.go +++ b/sntable_test.go @@ -23,7 +23,7 @@ func seedTable(buf *bytes.Buffer, sz int) error { rnd := rand.New(rand.NewSource(1)) val := make([]byte, 128) - for i := 0; i < sz; i++ { + for i := range sz { key := uint64(i * 4) if _, err := rnd.Read(val); err != nil { return err diff --git a/writer.go b/writer.go index db133e6..8bc619b 100644 --- a/writer.go +++ b/writer.go @@ -23,6 +23,20 @@ type WriterOptions struct { // The compression codec to use. // Default: SnappyCompression. Compression Compression + + // BloomBitsPerKey enables an in-file bloom filter with this many bits per + // key. 10 gives a ~1% false-positive rate. Zero (the default) disables the + // filter and keeps the file format identical to older versions. + // + // A filter lets the reader answer "definitely absent" without reading a + // data block, which is a large win when most lookups miss. + BloomBitsPerKey int + + // BloomExpectedKeys sizes the bloom filter up front so the writer never has + // to buffer keys. It is required when BloomBitsPerKey > 0. A rough estimate + // is fine: too low only raises the false-positive rate, it never causes a + // false negative. + BloomExpectedKeys int } func (o *WriterOptions) norm() *WriterOptions { @@ -58,14 +72,23 @@ type Writer struct { tmp []byte // scratch buffer index []blockInfo + bloom *bloomFilter // nil unless enabled via options } // NewWriter wraps a writer and returns a Writer. func NewWriter(w io.Writer, o *WriterOptions) *Writer { + oo := o.norm() + + var bloom *bloomFilter + if oo.BloomBitsPerKey > 0 { + bloom = newBloomFilter(oo.BloomExpectedKeys, oo.BloomBitsPerKey) + } + return &Writer{ - w: w, - o: o.norm(), - tmp: make([]byte, 2*binary.MaxVarintLen64), + w: w, + o: oo, + tmp: make([]byte, 2*binary.MaxVarintLen64), + bloom: bloom, } } @@ -85,6 +108,10 @@ func (w *Writer) Append(key uint64, value []byte) error { } } + if w.bloom != nil { + w.bloom.add(key) + } + skey := key if w.blen%w.o.BlockRestartInterval == 0 { // new section? w.soffs = append(w.soffs, len(w.buf)) @@ -112,12 +139,26 @@ func (w *Writer) Close() error { return err } + // The bloom section sits between the data blocks and the index, so its + // start doubles as the exclusive upper bound of the last data block. + var bloomOffset int64 + if w.bloom != nil { + bloomOffset = w.block.Offset + n, err := w.bloom.writeTo(w.w) + if err != nil { + return err + } + // writeTo goes straight to the underlying writer, so account for the + // bytes it wrote to keep block.Offset (our running file position) exact. + w.block.Offset += int64(n) + } + indexOffset := w.block.Offset if err := w.writeIndex(); err != nil { return err } - if err := w.writeFooter(indexOffset); err != nil { + if err := w.writeFooter(indexOffset, bloomOffset); err != nil { return err } w.tmp = nil @@ -146,15 +187,23 @@ func (w *Writer) writeIndex() error { return nil } -func (w *Writer) writeFooter(indexOffset int64) error { +func (w *Writer) writeFooter(indexOffset, bloomOffset int64) error { binary.LittleEndian.PutUint64(w.tmp[0:], uint64(indexOffset)) - if err := w.writeRaw(w.tmp[:8]); err != nil { - return err + + // No filter: keep the 16-byte v1 footer so the file stays byte-identical + // to older writers and older readers can still open it. + if w.bloom == nil { + if err := w.writeRaw(w.tmp[:8]); err != nil { + return err + } + return w.writeRaw(magic) } - if err := w.writeRaw(magic); err != nil { + + binary.LittleEndian.PutUint64(w.tmp[8:], uint64(bloomOffset)) + if err := w.writeRaw(w.tmp[:16]); err != nil { return err } - return nil + return w.writeRaw(magicBloom) } func (w *Writer) writeRaw(p []byte) error { From dece70b3e5bd807062ecc7489deb015f3d27bb9d Mon Sep 17 00:00:00 2001 From: Dimitrij Denissenko Date: Wed, 5 Aug 2026 15:28:57 +0100 Subject: [PATCH 2/2] Bump deps --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index eb842ef..6e3bbdf 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,4 @@ module github.com/bsm/sntable go 1.26 -require github.com/klauspost/compress v1.19.0 +require github.com/klauspost/compress v1.19.1 diff --git a/go.sum b/go.sum index 8916777..946bdf0 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,2 @@ -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=