diff --git a/Cargo.lock b/Cargo.lock index 206ca92..0d052cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + [[package]] name = "arrayref" version = "0.3.9" @@ -32,11 +38,29 @@ dependencies = [ "critical-section", ] +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +dependencies = [ + "serde_core", +] [[package]] name = "blake3" @@ -58,6 +82,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteview" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" + [[package]] name = "cc" version = "1.2.63" @@ -98,6 +128,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "compare" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -113,12 +149,95 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core 0.9.12", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "double-ended-peekable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" + +[[package]] +name = "doxygen-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" +dependencies = [ + "phf", +] + [[package]] name = "embedded-io" version = "0.4.0" @@ -137,18 +256,103 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "869b0adbda23651a9c5c0c3d270aac9fcb52e8622a8f2b17e57802d7791962f2" +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "error-code" version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fjall" +version = "2.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc" +dependencies = [ + "byteorder", + "byteview", + "dashmap", + "log", + "lsm-tree", + "path-absolutize", + "std-semaphore", + "tempfile", + "xxhash-rust", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "geolog-lang" version = "0.1.0" @@ -183,6 +387,25 @@ dependencies = [ "serde_json", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "guardian" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f" + [[package]] name = "hash32" version = "0.2.1" @@ -192,6 +415,33 @@ dependencies = [ "byteorder", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heapless" version = "0.7.17" @@ -206,6 +456,50 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "heed" +version = "0.20.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d4f449bab7320c56003d37732a917e18798e2f1709d80263face2b4f9436ddb" +dependencies = [ + "bitflags 2.11.1", + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-master-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", +] + +[[package]] +name = "heed-traits" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" + +[[package]] +name = "heed-types" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114" +dependencies = [ + "bincode", + "byteorder", + "heed-traits", + "serde", + "serde_json", +] + [[package]] name = "hex" version = "0.4.3" @@ -230,6 +524,145 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "interval-heap" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6" +dependencies = [ + "compare", +] + [[package]] name = "itoa" version = "1.0.18" @@ -248,12 +681,41 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lmdb-master-sys" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaeb9bd22e73bd1babffff614994b341e9b2008de7bb73bf1f7e9154f1978f8b" +dependencies = [ + "cc", + "doxygen-rs", + "libc", +] + [[package]] name = "lock_api" version = "0.4.14" @@ -269,6 +731,36 @@ version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +[[package]] +name = "lsm-tree" +version = "2.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799399117a2bfb37660e08be33f470958babb98386b04185288d829df362ea15" +dependencies = [ + "byteorder", + "crossbeam-skiplist", + "double-ended-peekable", + "enum_dispatch", + "guardian", + "interval-heap", + "log", + "lz4_flex", + "path-absolutize", + "quick_cache", + "rustc-hash", + "self_cell", + "tempfile", + "value-log", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + [[package]] name = "matchers" version = "0.2.0" @@ -299,7 +791,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags", + "bitflags 2.11.1", "cfg-if", "cfg_aliases", "libc", @@ -320,6 +812,120 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -339,6 +945,25 @@ dependencies = [ "serde", ] +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -356,7 +981,23 @@ version = "0.1.0" name = "query-storage" version = "0.1.0" dependencies = [ + "fjall", + "geomerge", + "heed", "query-ops", + "redb", + "sled", + "tempfile", +] + +[[package]] +name = "quick_cache" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c821816e9b928e20e92ed59bb3ac4aab321d16ca2316871c9fe7ca739cd477" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", ] [[package]] @@ -368,6 +1009,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radix_trie" version = "0.3.0" @@ -378,6 +1025,48 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.11.1", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -395,6 +1084,12 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustc_version" version = "0.4.1" @@ -404,13 +1099,26 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustyline" version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a990b25f351b25139ddc7f21ee3f6f56f86d6846b74ac8fad3a719a287cd4a0" dependencies = [ - "bitflags", + "bitflags 2.11.1", "cfg-if", "clipboard-win", "home", @@ -431,6 +1139,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "self_cell" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" + [[package]] name = "semver" version = "1.0.28" @@ -501,6 +1215,28 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot", +] + [[package]] name = "smallvec" version = "1.15.1" @@ -522,6 +1258,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "std-semaphore" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" + [[package]] name = "syn" version = "2.0.117" @@ -533,6 +1275,39 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synchronoise" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" +dependencies = [ + "crossbeam-queue", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -562,6 +1337,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tracing" version = "0.1.44" @@ -641,6 +1426,30 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -653,6 +1462,103 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "value-log" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" +dependencies = [ + "byteorder", + "byteview", + "interval-heap", + "log", + "path-absolutize", + "rustc-hash", + "tempfile", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "varint-rs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.1", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" @@ -668,6 +1574,189 @@ dependencies = [ "windows-link", ] +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.1", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/crates/query-storage/Cargo.toml b/crates/query-storage/Cargo.toml new file mode 100644 index 0000000..8b837b6 --- /dev/null +++ b/crates/query-storage/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "query-storage" +version = "0.1.0" +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints.rust] +unsafe_code = "deny" + +[lints.clippy] +pedantic = "warn" + +[features] +default = [] +lmdb = ["dep:heed"] +redb = ["dep:redb"] +fjall = ["dep:fjall"] +sled = ["dep:sled"] +geomerge = ["dep:geomerge"] + +[dependencies] +query-ops = { path = "../query-ops" } +heed = { version = "0.20", optional = true } +redb = { version = "2", optional = true } +fjall = { version = "2", optional = true } +sled = { version = "0.34", optional = true } +geomerge = { path = "../../external/geomerge/crates/geomerge", optional = true } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/query-storage/src/codec.rs b/crates/query-storage/src/codec.rs new file mode 100644 index 0000000..cfd99cd --- /dev/null +++ b/crates/query-storage/src/codec.rs @@ -0,0 +1,262 @@ +//! Wire format shared by every byte-oriented backend in this crate. +//! +//! The encoding is hand-rolled (no `serde`, no `bincode`) so that the +//! generated bytes are stable and inspectable. It is **not** versioned: adding +//! a new [`Value`] variant invalidates previously-stored data. That is fine +//! for a playground; production code would prepend a format byte. +//! +//! ## Row Format +//! +//! `[count: u32 LE] [val × count]` +//! +//! ## Value Format +//! +//! `[tag: u8] [payload]` +//! +//! | Tag | Variant | Payload | +//! |--------|---------------|--------------------------------------| +//! | `0x00` | `Value::Int` | `i64 LE` (8 bytes) | +//! | `0x01` | `Value::Str` | `[len: u32 LE] [bytes]` | +//! +//! ## Row Key Format +//! +//! Synthetic row IDs are `u64` encoded big-endian so lexicographic key order +//! matches insertion order. Backends with named sub-stores per relation can +//! use this directly as the key. +//! +//! ## Metadata Format +//! +//! Per-relation metadata is `[arity: u32 LE] [next_id: u64 LE]` = 12 bytes. + +use query_ops::value::Value; + +/// Errors raised by [`decode_row`] and [`decode_meta`]. +#[derive(Debug)] +pub enum CodecError { + /// The byte slice ended before the expected number of fields was read. + UnexpectedEof, + /// A value tag byte was unrecognized. + UnknownTag(u8), + /// A length field declared more bytes than the slice contains. + LengthOverrun { declared: usize, available: usize }, + /// A UTF-8 string payload could not be decoded. + InvalidUtf8, +} + +impl std::fmt::Display for CodecError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnexpectedEof => write!(f, "unexpected end of bytes"), + Self::UnknownTag(t) => write!(f, "unknown value tag: 0x{t:02x}"), + Self::LengthOverrun { + declared, + available, + } => write!( + f, + "declared length {declared} exceeds available {available} bytes" + ), + Self::InvalidUtf8 => write!(f, "invalid UTF-8 in string payload"), + } + } +} + +impl std::error::Error for CodecError {} + +/// Encode a row of [`Value`]s to bytes. +#[must_use] +pub fn encode_row(row: &[Value]) -> Vec { + let mut out = Vec::with_capacity(4 + row.len() * 9); + out.extend_from_slice(&u32::try_from(row.len()).unwrap_or(u32::MAX).to_le_bytes()); + for value in row { + match value { + Value::Int(i) => { + out.push(0x00); + out.extend_from_slice(&i.to_le_bytes()); + } + Value::Str(s) => { + out.push(0x01); + let bytes = s.as_bytes(); + out.extend_from_slice( + &u32::try_from(bytes.len()).unwrap_or(u32::MAX).to_le_bytes(), + ); + out.extend_from_slice(bytes); + } + } + } + out +} + +/// Decode a row of [`Value`]s from bytes. +/// +/// # Errors +/// Returns [`CodecError`] if the byte slice is malformed. +pub fn decode_row(mut bytes: &[u8]) -> Result, CodecError> { + let count = read_u32(&mut bytes)? as usize; + let mut row = Vec::with_capacity(count); + for _ in 0..count { + row.push(read_value(&mut bytes)?); + } + Ok(row) +} + +fn read_value(bytes: &mut &[u8]) -> Result { + let tag = read_u8(bytes)?; + match tag { + 0x00 => { + let i = read_i64(bytes)?; + Ok(Value::Int(i)) + } + 0x01 => { + let len = read_u32(bytes)? as usize; + if bytes.len() < len { + return Err(CodecError::LengthOverrun { + declared: len, + available: bytes.len(), + }); + } + let (head, tail) = bytes.split_at(len); + *bytes = tail; + let s = std::str::from_utf8(head) + .map_err(|_| CodecError::InvalidUtf8)? + .to_string(); + Ok(Value::Str(s)) + } + other => Err(CodecError::UnknownTag(other)), + } +} + +fn read_u8(bytes: &mut &[u8]) -> Result { + let (head, tail) = bytes.split_first().ok_or(CodecError::UnexpectedEof)?; + *bytes = tail; + Ok(*head) +} + +fn read_u32(bytes: &mut &[u8]) -> Result { + if bytes.len() < 4 { + return Err(CodecError::UnexpectedEof); + } + let (head, tail) = bytes.split_at(4); + *bytes = tail; + let mut buf = [0u8; 4]; + buf.copy_from_slice(head); + Ok(u32::from_le_bytes(buf)) +} + +fn read_u64(bytes: &mut &[u8]) -> Result { + if bytes.len() < 8 { + return Err(CodecError::UnexpectedEof); + } + let (head, tail) = bytes.split_at(8); + *bytes = tail; + let mut buf = [0u8; 8]; + buf.copy_from_slice(head); + Ok(u64::from_le_bytes(buf)) +} + +fn read_i64(bytes: &mut &[u8]) -> Result { + if bytes.len() < 8 { + return Err(CodecError::UnexpectedEof); + } + let (head, tail) = bytes.split_at(8); + *bytes = tail; + let mut buf = [0u8; 8]; + buf.copy_from_slice(head); + Ok(i64::from_le_bytes(buf)) +} + +/// Encode a row key from a synthetic u64 ID. +/// +/// Big-endian so lexicographic key order matches insertion order. +#[must_use] +pub fn row_key(id: u64) -> [u8; 8] { + id.to_be_bytes() +} + +/// Encode per-relation metadata: arity and next row ID. +#[must_use] +pub fn encode_meta(arity: u32, next_id: u64) -> [u8; 12] { + let mut out = [0u8; 12]; + out[0..4].copy_from_slice(&arity.to_le_bytes()); + out[4..12].copy_from_slice(&next_id.to_le_bytes()); + out +} + +/// Decode per-relation metadata. +/// +/// # Errors +/// Returns [`CodecError::UnexpectedEof`] if the slice is shorter than 12 bytes. +pub fn decode_meta(mut bytes: &[u8]) -> Result<(u32, u64), CodecError> { + let arity = read_u32(&mut bytes)?; + let next_id = read_u64(&mut bytes)?; + Ok((arity, next_id)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn i(x: i64) -> Value { + Value::Int(x) + } + + fn s(x: &str) -> Value { + Value::Str(x.to_string()) + } + + #[test] + fn encode_decode_int_only_row() -> Result<(), CodecError> { + let row = vec![i(1), i(-2), i(i64::MAX)]; + let bytes = encode_row(&row); + let decoded = decode_row(&bytes)?; + assert_eq!(decoded, row); + Ok(()) + } + + #[test] + fn encode_decode_mixed_row() -> Result<(), CodecError> { + let row = vec![s("Alice"), i(42), s("a longer string with spaces")]; + let bytes = encode_row(&row); + let decoded = decode_row(&bytes)?; + assert_eq!(decoded, row); + Ok(()) + } + + #[test] + fn encode_decode_empty_row() -> Result<(), CodecError> { + let bytes = encode_row(&[]); + let decoded = decode_row(&bytes)?; + assert!(decoded.is_empty()); + Ok(()) + } + + #[test] + fn decode_unknown_tag_fails() { + let bytes = vec![1, 0, 0, 0, 0xFF]; + assert!(matches!( + decode_row(&bytes), + Err(CodecError::UnknownTag(0xFF)) + )); + } + + #[test] + fn decode_truncated_fails() { + let bytes = vec![1, 0, 0, 0, 0x00, 0x01]; + assert!(matches!(decode_row(&bytes), Err(CodecError::UnexpectedEof))); + } + + #[test] + fn row_key_preserves_order() { + assert!(row_key(1) < row_key(2)); + assert!(row_key(255) < row_key(256)); + assert!(row_key(u64::MAX - 1) < row_key(u64::MAX)); + } + + #[test] + fn meta_roundtrip() -> Result<(), CodecError> { + let encoded = encode_meta(3, 12345); + let (arity, next_id) = decode_meta(&encoded)?; + assert_eq!(arity, 3); + assert_eq!(next_id, 12345); + Ok(()) + } +} diff --git a/crates/query-storage/src/fjall.rs b/crates/query-storage/src/fjall.rs new file mode 100644 index 0000000..6f97a31 --- /dev/null +++ b/crates/query-storage/src/fjall.rs @@ -0,0 +1,149 @@ +//! fjall adapter. +//! +//! Each relation gets a fjall [`PartitionHandle`](fjall::PartitionHandle) of +//! the same name. A reserved partition named `__meta` carries per-relation +//! metadata (arity and next synthetic row ID). + +use fjall::{Keyspace, PartitionCreateOptions, PartitionHandle}; + +use query_ops::value::Value; + +use crate::codec::{decode_meta, decode_row, encode_meta, encode_row, row_key}; +use crate::{Storage, StorageError}; + +const META_PARTITION: &str = "__meta"; + +fn backend(err: E) -> StorageError { + StorageError::Backend(Box::new(err)) +} + +/// fjall-backed [`Storage`] implementation. +pub struct FjallStorage { + keyspace: Keyspace, + meta: PartitionHandle, +} + +impl FjallStorage { + /// Open or create a fjall keyspace at `path`. + /// + /// # Errors + /// Returns [`StorageError::Backend`] if fjall fails to open the path. + pub fn open(path: impl AsRef) -> Result { + let keyspace = fjall::Config::new(path).open().map_err(backend)?; + let meta = keyspace + .open_partition(META_PARTITION, PartitionCreateOptions::default()) + .map_err(backend)?; + Ok(Self { keyspace, meta }) + } + + fn relation_partition(&self, name: &str) -> Result { + self.keyspace + .open_partition(name, PartitionCreateOptions::default()) + .map_err(backend) + } + + fn load_meta(&self, name: &str) -> Result<(u32, u64), StorageError> { + let raw = self + .meta + .get(name.as_bytes()) + .map_err(backend)? + .ok_or_else(|| StorageError::RelationNotFound(name.to_string()))?; + Ok(decode_meta(raw.as_ref())?) + } + + fn store_meta(&self, name: &str, arity: u32, next_id: u64) -> Result<(), StorageError> { + self.meta + .insert(name.as_bytes(), encode_meta(arity, next_id)) + .map_err(backend)?; + Ok(()) + } +} + +impl Storage for FjallStorage { + fn create_relation(&mut self, name: &str, arity: usize) -> Result<(), StorageError> { + if name == META_PARTITION { + return Err(StorageError::Validation(format!( + "relation name '{name}' is reserved" + ))); + } + if self.meta.contains_key(name.as_bytes()).map_err(backend)? { + return Err(StorageError::RelationExists(name.to_string())); + } + let arity_u32 = u32::try_from(arity) + .map_err(|_| StorageError::Validation(format!("arity {arity} exceeds u32 range")))?; + self.store_meta(name, arity_u32, 0)?; + let _ = self.relation_partition(name)?; + Ok(()) + } + + fn arity(&self, name: &str) -> Result { + let (arity, _) = self.load_meta(name)?; + Ok(arity as usize) + } + + fn scan(&self, name: &str) -> Result>, StorageError> { + let _ = self.load_meta(name)?; + let partition = self.relation_partition(name)?; + let mut rows = Vec::new(); + for entry in partition.iter() { + let (_, value) = entry.map_err(backend)?; + rows.push(decode_row(value.as_ref())?); + } + Ok(rows) + } + + fn insert(&mut self, name: &str, row: Vec) -> Result<(), StorageError> { + let (arity, next_id) = self.load_meta(name)?; + if row.len() != arity as usize { + return Err(StorageError::ArityMismatch { + expected: arity as usize, + got: row.len(), + }); + } + let partition = self.relation_partition(name)?; + partition + .insert(row_key(next_id), encode_row(&row)) + .map_err(backend)?; + self.store_meta(name, arity, next_id + 1)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn i(x: i64) -> Value { + Value::Int(x) + } + + fn open_temp() -> Result { + let dir = tempfile::tempdir().map_err(backend)?; + let storage = FjallStorage::open(dir.path())?; + std::mem::forget(dir); + Ok(storage) + } + + #[test] + fn create_insert_scan_roundtrip() -> Result<(), StorageError> { + let mut storage = open_temp()?; + storage.create_relation("edge", 2)?; + storage.insert("edge", vec![i(1), i(2)])?; + storage.insert("edge", vec![i(2), i(3)])?; + let rows = storage.scan("edge")?; + assert_eq!(rows, vec![vec![i(1), i(2)], vec![i(2), i(3)]]); + assert_eq!(storage.arity("edge")?, 2); + Ok(()) + } + + #[test] + fn duplicate_create_returns_err() -> Result<(), StorageError> { + let mut storage = open_temp()?; + storage.create_relation("edge", 2)?; + assert!(matches!( + storage.create_relation("edge", 2), + Err(StorageError::RelationExists(_)) + )); + Ok(()) + } +} diff --git a/crates/query-storage/src/geomerge.rs b/crates/query-storage/src/geomerge.rs new file mode 100644 index 0000000..77f638d --- /dev/null +++ b/crates/query-storage/src/geomerge.rs @@ -0,0 +1,252 @@ +//! Geomerge adapter. +//! +//! Unlike the other backends, geomerge schemas are **immutable after store +//! construction**: there is no public API to register a new table on a live +//! `Store`. The adapter therefore expects all relations to be declared up +//! front via a `FlatTheory` passed to [`GeomergeStorage::from_theory`], and +//! [`Storage::create_relation`] becomes a verifier that the relation exists +//! in the loaded theory and that its arity matches. +//! +//! Additional v1 mismatches with the trait: +//! +//! - Column types are typed (`PrimInt` / `PrimString`) in geomerge but the +//! trait's `create_relation` only carries `arity`. The adapter cannot +//! declare a relation at runtime, so this issue surfaces only at insert +//! time when geomerge rejects a row with `StorageError::Validation`. +//! - Cells of type `CellValue::Id` cannot be represented in our `Value` enum. +//! Scanning a table that contains such cells returns `StorageError::Validation`. +//! - Every `insert` opens a fresh `Transaction` and commits. Law violations +//! surface at commit time, not at the `add` call. + +use std::collections::HashSet; + +use geomerge::ir::{self, Path}; +use geomerge::store::Store; +use geomerge::table::CellValue; +use geomerge::txn::ops::TxnCellValue; + +use query_ops::value::Value; + +use crate::{Storage, StorageError}; + +fn backend(err: E) -> StorageError { + StorageError::Backend(Box::new(err)) +} + +fn validation(msg: impl Into) -> StorageError { + StorageError::Validation(msg.into()) +} + +/// Geomerge-backed [`Storage`] implementation. +/// +/// Construct via [`GeomergeStorage::new`] (empty store, no relations) or +/// [`GeomergeStorage::from_theory`] (preloaded with a `FlatTheory`). +pub struct GeomergeStorage { + store: Store, + declared: HashSet, +} + +impl Default for GeomergeStorage { + fn default() -> Self { + Self::new() + } +} + +impl GeomergeStorage { + /// Build an empty store. No relations are available until the store is + /// rebuilt via a theory. + #[must_use] + pub fn new() -> Self { + Self { + store: Store::new(), + declared: HashSet::new(), + } + } + + /// Build a store from a pre-defined `FlatTheory`. All `create_relation` + /// calls must reference relations declared in the theory. + /// + /// # Errors + /// Returns [`StorageError::Backend`] if geomerge rejects the theory. + pub fn from_theory(theory: ir::FlatTheory) -> Result { + let store = Store::try_from_theory(theory).map_err(|e| backend(*e))?; + Ok(Self { + store, + declared: HashSet::new(), + }) + } +} + +impl Storage for GeomergeStorage { + fn create_relation(&mut self, name: &str, arity: usize) -> Result<(), StorageError> { + if self.declared.contains(name) { + return Err(StorageError::RelationExists(name.to_string())); + } + let path: Path = name.into(); + let table = self.store.table_at(&path).ok_or_else(|| { + validation(format!( + "relation '{name}' is not declared in the loaded geomerge theory; \ + geomerge does not support runtime relation creation" + )) + })?; + let declared_arity = table.schema().columns.len(); + if declared_arity != arity { + return Err(StorageError::ArityMismatch { + expected: declared_arity, + got: arity, + }); + } + self.declared.insert(name.to_string()); + Ok(()) + } + + fn arity(&self, name: &str) -> Result { + let path: Path = name.into(); + let table = self + .store + .table_at(&path) + .ok_or_else(|| StorageError::RelationNotFound(name.to_string()))?; + Ok(table.schema().columns.len()) + } + + fn scan(&self, name: &str) -> Result>, StorageError> { + let path: Path = name.into(); + let table = self + .store + .table_at(&path) + .ok_or_else(|| StorageError::RelationNotFound(name.to_string()))?; + let arity = table.schema().columns.len(); + let mut rows = Vec::with_capacity(table.row_count()); + for r in 0..table.row_count() { + let mut row = Vec::with_capacity(arity); + for c in 0..arity { + let cell = table + .cell_at(r, c) + .ok_or_else(|| validation(format!("missing cell at ({r}, {c}) in '{name}'")))?; + row.push(cell_to_value(cell)?); + } + rows.push(row); + } + Ok(rows) + } + + fn insert(&mut self, name: &str, row: Vec) -> Result<(), StorageError> { + let path: Path = name.into(); + let arity = self.arity(name)?; + if row.len() != arity { + return Err(StorageError::ArityMismatch { + expected: arity, + got: row.len(), + }); + } + let values: Vec = row.into_iter().map(value_to_txn_cell).collect(); + let mut txn = self.store.transaction(); + txn.add(&path, values) + .map_err(|e| validation(e.to_string()))?; + // Law violations surface here at commit time, not at add time. + txn.commit().map_err(|e| validation(e.to_string()))?; + Ok(()) + } +} + +fn cell_to_value(cell: &CellValue) -> Result { + match cell { + CellValue::Int(i) => Ok(Value::Int(*i)), + CellValue::Str(s) => Ok(Value::Str(s.clone())), + CellValue::Id(_) => Err(validation( + "geomerge CellValue::Id cannot be represented in the playground's Value enum", + )), + } +} + +fn value_to_txn_cell(value: Value) -> TxnCellValue { + match value { + Value::Int(i) => TxnCellValue::Int(i), + Value::Str(s) => TxnCellValue::Str(s), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use geomerge::ir::{ColType, FlatTheory, PrimType, Schema, TableEntry}; + + fn i(x: i64) -> Value { + Value::Int(x) + } + + fn int_schema(arity: usize) -> Schema { + Schema { + columns: (0..arity) + .map(|_| ColType::PrimType { + prim: PrimType::PrimInt, + }) + .collect(), + primary_key: None, + } + } + + fn theory_with_one_int_table(name: &str, arity: usize) -> FlatTheory { + FlatTheory { + tables: vec![TableEntry { + path: name.into(), + table: int_schema(arity), + }], + laws: Vec::new(), + } + } + + #[test] + fn empty_store_has_no_relations() { + let storage = GeomergeStorage::new(); + assert!(matches!( + storage.arity("edge"), + Err(StorageError::RelationNotFound(_)) + )); + } + + #[test] + fn create_relation_on_undeclared_returns_validation_error() { + let mut storage = GeomergeStorage::new(); + assert!(matches!( + storage.create_relation("edge", 2), + Err(StorageError::Validation(_)) + )); + } + + #[test] + fn theory_loaded_insert_scan_roundtrip() -> Result<(), StorageError> { + let theory = theory_with_one_int_table("edge", 2); + let mut storage = GeomergeStorage::from_theory(theory)?; + storage.create_relation("edge", 2)?; + storage.insert("edge", vec![i(1), i(2)])?; + storage.insert("edge", vec![i(3), i(4)])?; + let rows = storage.scan("edge")?; + assert_eq!(rows, vec![vec![i(1), i(2)], vec![i(3), i(4)]]); + assert_eq!(storage.arity("edge")?, 2); + Ok(()) + } + + #[test] + fn duplicate_create_returns_err() -> Result<(), StorageError> { + let theory = theory_with_one_int_table("edge", 2); + let mut storage = GeomergeStorage::from_theory(theory)?; + storage.create_relation("edge", 2)?; + assert!(matches!( + storage.create_relation("edge", 2), + Err(StorageError::RelationExists(_)) + )); + Ok(()) + } + + #[test] + fn insert_wrong_type_returns_validation_error() -> Result<(), StorageError> { + let theory = theory_with_one_int_table("edge", 2); + let mut storage = GeomergeStorage::from_theory(theory)?; + storage.create_relation("edge", 2)?; + // Insert a Str into an Int column: geomerge rejects it. + let result = storage.insert("edge", vec![Value::Str("not an int".to_string()), i(2)]); + assert!(matches!(result, Err(StorageError::Validation(_)))); + Ok(()) + } +} diff --git a/crates/query-storage/src/lib.rs b/crates/query-storage/src/lib.rs new file mode 100644 index 0000000..9b978c5 --- /dev/null +++ b/crates/query-storage/src/lib.rs @@ -0,0 +1,144 @@ +//! Storage backend abstraction for the query-plan playground. +//! +//! Operators in [`query_ops`] work over in-memory [`Table`](query_ops::table::Table) +//! and [`Relation`](query_ops::relation::Relation) values. This crate adds a +//! [`Storage`] trait so the input tables can come from a backend (an +//! in-memory [`MemoryStorage`], an LMDB environment, a `geomerge` store, and +//! so on) instead of being built by hand in every test. +//! +//! The v1 surface is deliberately narrow: create a relation, scan all rows, +//! insert a row, ask for arity. Transactions, range scans, deletes, and delta +//! streams are not modeled yet, and will be added when a specific experiment +//! demands them. +//! +//! ## Backends +//! +//! [`MemoryStorage`] is always available. Other backends are gated behind +//! Cargo features so users only pay for what they need: +//! +//! - `lmdb` — LMDB via the `heed` crate +//! - `redb` — pure-Rust embedded KV +//! - `fjall` — pure-Rust LSM-tree +//! - `sled` — pure-Rust LSM-tree +//! - `geomerge` — the workspace's `geomerge` crate + +use query_ops::table::Table; +use query_ops::value::Value; + +pub mod codec; +pub mod memory; + +#[cfg(feature = "sled")] +pub mod sled; + +#[cfg(feature = "redb")] +pub mod redb; + +#[cfg(feature = "fjall")] +pub mod fjall; + +#[cfg(feature = "lmdb")] +pub mod lmdb; + +#[cfg(feature = "geomerge")] +pub mod geomerge; + +pub use memory::MemoryStorage; + +/// Errors returned by a [`Storage`] backend. +/// +/// Backend-specific failures (LMDB transaction aborts, sled I/O errors, etc.) +/// are wrapped in [`StorageError::Backend`]. +#[derive(Debug)] +pub enum StorageError { + /// No relation with the given name exists in this backend. + RelationNotFound(String), + /// A relation with the given name already exists. + RelationExists(String), + /// A row was offered with the wrong number of columns. + ArityMismatch { expected: usize, got: usize }, + /// A backend-defined validation rule rejected the operation, for example + /// a `geomerge` law violation. + Validation(String), + /// A row decoded from storage was malformed. + Decode(codec::CodecError), + /// A backend-specific error wrapped for transport across the trait. + Backend(Box), +} + +impl std::fmt::Display for StorageError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::RelationNotFound(name) => write!(f, "relation not found: {name}"), + Self::RelationExists(name) => write!(f, "relation already exists: {name}"), + Self::ArityMismatch { expected, got } => { + write!(f, "arity mismatch: expected {expected}, got {got}") + } + Self::Validation(msg) => write!(f, "validation failed: {msg}"), + Self::Decode(err) => write!(f, "decode error: {err}"), + Self::Backend(err) => write!(f, "backend error: {err}"), + } + } +} + +impl std::error::Error for StorageError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Backend(err) => Some(err.as_ref()), + Self::Decode(err) => Some(err), + _ => None, + } + } +} + +impl From for StorageError { + fn from(err: codec::CodecError) -> Self { + Self::Decode(err) + } +} + +/// Backend-agnostic interface for storing and retrieving rows. +/// +/// Each relation has a fixed name, a fixed arity (row width), and an ordered +/// collection of rows whose cells are [`Value`]s. Concrete implementations +/// include [`MemoryStorage`] in this crate plus the feature-gated backends. +pub trait Storage { + /// Create a new relation with the given name and arity. + /// + /// # Errors + /// Returns [`StorageError::RelationExists`] if a relation with the given + /// name already exists. + fn create_relation(&mut self, name: &str, arity: usize) -> Result<(), StorageError>; + + /// Return the arity of the given relation. + /// + /// # Errors + /// Returns [`StorageError::RelationNotFound`] if no such relation exists. + fn arity(&self, name: &str) -> Result; + + /// Scan all rows of the given relation in storage order. + /// + /// # Errors + /// Returns [`StorageError::RelationNotFound`] if no such relation exists. + fn scan(&self, name: &str) -> Result>, StorageError>; + + /// Append a row to the given relation. + /// + /// # Errors + /// Returns [`StorageError::RelationNotFound`] if no such relation exists, + /// [`StorageError::ArityMismatch`] if the row's length differs from the + /// declared arity, or [`StorageError::Validation`] / [`StorageError::Backend`] + /// if a backend-specific rule rejects the row. + fn insert(&mut self, name: &str, row: Vec) -> Result<(), StorageError>; +} + +/// Materialize a relation from a [`Storage`] backend as a [`Table`] that +/// [`scan_atom`](query_ops::atom::scan_atom) can consume. +/// +/// # Errors +/// Returns any error produced by [`Storage::arity`] or [`Storage::scan`]. +pub fn scan_as_table(storage: &dyn Storage, name: &str) -> Result { + let arity = storage.arity(name)?; + let rows = storage.scan(name)?; + Ok(Table::from_rows(arity, rows)) +} diff --git a/crates/query-storage/src/lmdb.rs b/crates/query-storage/src/lmdb.rs new file mode 100644 index 0000000..2405a02 --- /dev/null +++ b/crates/query-storage/src/lmdb.rs @@ -0,0 +1,201 @@ +//! LMDB adapter via the `heed` crate. +//! +//! Maps each relation onto a named LMDB sub-database of the same name. A +//! reserved sub-database named `__meta` carries per-relation metadata (arity +//! and next synthetic row ID). +//! +//! Note: every [`Storage::insert`] opens its own write transaction. LMDB +//! serializes writers across the env, so per-row inserts will be slow on real +//! workloads. The v1 trait does not yet expose batch inserts. + +use heed::types::Bytes; +use heed::{Database, Env, EnvOpenOptions}; + +use query_ops::value::Value; + +use crate::codec::{decode_meta, decode_row, encode_meta, encode_row, row_key}; +use crate::{Storage, StorageError}; + +const META_DB: &str = "__meta"; +const DEFAULT_MAX_DBS: u32 = 128; +const DEFAULT_MAP_SIZE: usize = 100 * 1024 * 1024; + +fn backend(err: E) -> StorageError { + StorageError::Backend(Box::new(err)) +} + +/// LMDB-backed [`Storage`] implementation. +pub struct LmdbStorage { + env: Env, + meta: Database, +} + +impl LmdbStorage { + /// Open or create an LMDB environment at `path`. + /// + /// The path must already exist as a directory; LMDB will create its data + /// files inside it. + /// + /// # Errors + /// Returns [`StorageError::Backend`] if LMDB fails to open. + /// + /// # Safety + /// Internally uses `EnvOpenOptions::open`, which heed marks `unsafe` + /// because the memory-mapped file's contents can be modified by other + /// processes. The adapter assumes single-process exclusive access. + #[allow(unsafe_code)] + pub fn open(path: impl AsRef) -> Result { + // SAFETY: heed marks `open` unsafe because the mmap'd file's contents + // can be modified by other processes, violating Rust's aliasing rules. + // This adapter assumes single-process exclusive access to the path, + // which holds for tests and typical playground use. + let env = unsafe { + EnvOpenOptions::new() + .max_dbs(DEFAULT_MAX_DBS) + .map_size(DEFAULT_MAP_SIZE) + .open(path) + .map_err(backend)? + }; + let mut wtxn = env.write_txn().map_err(backend)?; + let meta: Database = env + .create_database(&mut wtxn, Some(META_DB)) + .map_err(backend)?; + wtxn.commit().map_err(backend)?; + Ok(Self { env, meta }) + } + + fn open_relation_db( + &self, + wtxn: &mut heed::RwTxn, + name: &str, + ) -> Result, StorageError> { + self.env.create_database(wtxn, Some(name)).map_err(backend) + } +} + +impl Storage for LmdbStorage { + fn create_relation(&mut self, name: &str, arity: usize) -> Result<(), StorageError> { + if name == META_DB { + return Err(StorageError::Validation(format!( + "relation name '{name}' is reserved" + ))); + } + let arity_u32 = u32::try_from(arity) + .map_err(|_| StorageError::Validation(format!("arity {arity} exceeds u32 range")))?; + let mut wtxn = self.env.write_txn().map_err(backend)?; + if self + .meta + .get(&wtxn, name.as_bytes()) + .map_err(backend)? + .is_some() + { + return Err(StorageError::RelationExists(name.to_string())); + } + let encoded = encode_meta(arity_u32, 0); + self.meta + .put(&mut wtxn, name.as_bytes(), &encoded[..]) + .map_err(backend)?; + let _ = self.open_relation_db(&mut wtxn, name)?; + wtxn.commit().map_err(backend)?; + Ok(()) + } + + fn arity(&self, name: &str) -> Result { + let rtxn = self.env.read_txn().map_err(backend)?; + let raw = self + .meta + .get(&rtxn, name.as_bytes()) + .map_err(backend)? + .ok_or_else(|| StorageError::RelationNotFound(name.to_string()))?; + let (arity, _) = decode_meta(raw)?; + Ok(arity as usize) + } + + fn scan(&self, name: &str) -> Result>, StorageError> { + let rtxn = self.env.read_txn().map_err(backend)?; + if self + .meta + .get(&rtxn, name.as_bytes()) + .map_err(backend)? + .is_none() + { + return Err(StorageError::RelationNotFound(name.to_string())); + } + let db: Database = self + .env + .open_database(&rtxn, Some(name)) + .map_err(backend)? + .ok_or_else(|| StorageError::RelationNotFound(name.to_string()))?; + let mut rows = Vec::new(); + for entry in db.iter(&rtxn).map_err(backend)? { + let (_, value) = entry.map_err(backend)?; + rows.push(decode_row(value)?); + } + Ok(rows) + } + + fn insert(&mut self, name: &str, row: Vec) -> Result<(), StorageError> { + let mut wtxn = self.env.write_txn().map_err(backend)?; + let meta_bytes = self + .meta + .get(&wtxn, name.as_bytes()) + .map_err(backend)? + .ok_or_else(|| StorageError::RelationNotFound(name.to_string()))?; + let (arity, next_id) = decode_meta(meta_bytes)?; + if row.len() != arity as usize { + return Err(StorageError::ArityMismatch { + expected: arity as usize, + got: row.len(), + }); + } + let db = self.open_relation_db(&mut wtxn, name)?; + let key = row_key(next_id); + let value = encode_row(&row); + db.put(&mut wtxn, &key[..], &value[..]).map_err(backend)?; + let new_meta = encode_meta(arity, next_id + 1); + self.meta + .put(&mut wtxn, name.as_bytes(), &new_meta[..]) + .map_err(backend)?; + wtxn.commit().map_err(backend)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn i(x: i64) -> Value { + Value::Int(x) + } + + fn open_temp() -> Result { + let dir = tempfile::tempdir().map_err(backend)?; + let storage = LmdbStorage::open(dir.path())?; + std::mem::forget(dir); + Ok(storage) + } + + #[test] + fn create_insert_scan_roundtrip() -> Result<(), StorageError> { + let mut storage = open_temp()?; + storage.create_relation("edge", 2)?; + storage.insert("edge", vec![i(1), i(2)])?; + storage.insert("edge", vec![i(2), i(3)])?; + let rows = storage.scan("edge")?; + assert_eq!(rows, vec![vec![i(1), i(2)], vec![i(2), i(3)]]); + assert_eq!(storage.arity("edge")?, 2); + Ok(()) + } + + #[test] + fn duplicate_create_returns_err() -> Result<(), StorageError> { + let mut storage = open_temp()?; + storage.create_relation("edge", 2)?; + assert!(matches!( + storage.create_relation("edge", 2), + Err(StorageError::RelationExists(_)) + )); + Ok(()) + } +} diff --git a/crates/query-storage/src/memory.rs b/crates/query-storage/src/memory.rs new file mode 100644 index 0000000..a3294cf --- /dev/null +++ b/crates/query-storage/src/memory.rs @@ -0,0 +1,147 @@ +//! In-memory backend, keyed by relation name. Always available. + +use std::collections::HashMap; + +use query_ops::value::Value; + +use crate::{Storage, StorageError}; + +/// In-memory backend, useful as the default in tests and as a correctness +/// oracle for other backends. +#[derive(Debug, Default)] +pub struct MemoryStorage { + relations: HashMap, +} + +#[derive(Debug)] +struct MemoryRelation { + arity: usize, + rows: Vec>, +} + +impl MemoryStorage { + #[must_use] + pub fn new() -> Self { + Self::default() + } +} + +impl Storage for MemoryStorage { + fn create_relation(&mut self, name: &str, arity: usize) -> Result<(), StorageError> { + if self.relations.contains_key(name) { + return Err(StorageError::RelationExists(name.to_string())); + } + self.relations.insert( + name.to_string(), + MemoryRelation { + arity, + rows: Vec::new(), + }, + ); + Ok(()) + } + + fn arity(&self, name: &str) -> Result { + self.relations + .get(name) + .map(|r| r.arity) + .ok_or_else(|| StorageError::RelationNotFound(name.to_string())) + } + + fn scan(&self, name: &str) -> Result>, StorageError> { + self.relations + .get(name) + .map(|r| r.rows.clone()) + .ok_or_else(|| StorageError::RelationNotFound(name.to_string())) + } + + fn insert(&mut self, name: &str, row: Vec) -> Result<(), StorageError> { + let relation = self + .relations + .get_mut(name) + .ok_or_else(|| StorageError::RelationNotFound(name.to_string()))?; + if row.len() != relation.arity { + return Err(StorageError::ArityMismatch { + expected: relation.arity, + got: row.len(), + }); + } + relation.rows.push(row); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::scan_as_table; + + fn i(x: i64) -> Value { + Value::Int(x) + } + + #[test] + fn create_insert_scan_roundtrip() -> Result<(), StorageError> { + let mut storage = MemoryStorage::new(); + storage.create_relation("edge", 2)?; + storage.insert("edge", vec![i(1), i(2)])?; + storage.insert("edge", vec![i(2), i(3)])?; + let rows = storage.scan("edge")?; + assert_eq!(rows, vec![vec![i(1), i(2)], vec![i(2), i(3)]]); + Ok(()) + } + + #[test] + fn duplicate_create_returns_err() -> Result<(), StorageError> { + let mut storage = MemoryStorage::new(); + storage.create_relation("edge", 2)?; + assert!(matches!( + storage.create_relation("edge", 2), + Err(StorageError::RelationExists(_)) + )); + Ok(()) + } + + #[test] + fn scan_unknown_relation_returns_err() { + let storage = MemoryStorage::new(); + assert!(matches!( + storage.scan("missing"), + Err(StorageError::RelationNotFound(_)) + )); + } + + #[test] + fn arity_unknown_relation_returns_err() { + let storage = MemoryStorage::new(); + assert!(matches!( + storage.arity("missing"), + Err(StorageError::RelationNotFound(_)) + )); + } + + #[test] + fn insert_wrong_arity_returns_err() -> Result<(), StorageError> { + let mut storage = MemoryStorage::new(); + storage.create_relation("edge", 2)?; + assert!(matches!( + storage.insert("edge", vec![i(1)]), + Err(StorageError::ArityMismatch { + expected: 2, + got: 1 + }) + )); + Ok(()) + } + + #[test] + fn scan_as_table_materializes_table() -> Result<(), StorageError> { + let mut storage = MemoryStorage::new(); + storage.create_relation("edge", 2)?; + storage.insert("edge", vec![i(1), i(2)])?; + let table = scan_as_table(&storage, "edge")?; + assert_eq!(table.arity, 2); + assert_eq!(table.rows, vec![vec![i(1), i(2)]]); + Ok(()) + } +} diff --git a/crates/query-storage/src/redb.rs b/crates/query-storage/src/redb.rs new file mode 100644 index 0000000..cc1784a --- /dev/null +++ b/crates/query-storage/src/redb.rs @@ -0,0 +1,183 @@ +//! redb adapter. +//! +//! Each relation gets a redb table named after it, keyed by `u64` row IDs. +//! A reserved table named `__meta`, keyed by relation name, carries per-relation +//! metadata (arity and next synthetic row ID). + +use redb::{Database, ReadableTable, TableDefinition}; + +use query_ops::value::Value; + +use crate::codec::{decode_meta, decode_row, encode_meta, encode_row}; +use crate::{Storage, StorageError}; + +const META_TABLE: &str = "__meta"; + +fn backend(err: E) -> StorageError { + StorageError::Backend(Box::new(err)) +} + +fn meta_def() -> TableDefinition<'static, &'static str, &'static [u8]> { + TableDefinition::new(META_TABLE) +} + +fn rows_def(name: &str) -> TableDefinition<'_, u64, &'static [u8]> { + TableDefinition::new(name) +} + +/// redb-backed [`Storage`] implementation. +pub struct RedbStorage { + db: Database, +} + +impl RedbStorage { + /// Open or create a redb database at `path`. + /// + /// # Errors + /// Returns [`StorageError::Backend`] if redb fails to open the file. + pub fn open(path: impl AsRef) -> Result { + let db = Database::create(path).map_err(backend)?; + Ok(Self { db }) + } +} + +impl Storage for RedbStorage { + fn create_relation(&mut self, name: &str, arity: usize) -> Result<(), StorageError> { + if name == META_TABLE { + return Err(StorageError::Validation(format!( + "relation name '{name}' is reserved" + ))); + } + let arity_u32 = u32::try_from(arity) + .map_err(|_| StorageError::Validation(format!("arity {arity} exceeds u32 range")))?; + let txn = self.db.begin_write().map_err(backend)?; + { + let mut meta = txn.open_table(meta_def()).map_err(backend)?; + if meta.get(name).map_err(backend)?.is_some() { + return Err(StorageError::RelationExists(name.to_string())); + } + let encoded = encode_meta(arity_u32, 0); + meta.insert(name, &encoded[..]).map_err(backend)?; + // open_table creates the rows table if it does not exist + let _ = txn.open_table(rows_def(name)).map_err(backend)?; + } + txn.commit().map_err(backend)?; + Ok(()) + } + + fn arity(&self, name: &str) -> Result { + let txn = self.db.begin_read().map_err(backend)?; + let meta = txn.open_table(meta_def()).map_err(backend)?; + let raw = meta + .get(name) + .map_err(backend)? + .ok_or_else(|| StorageError::RelationNotFound(name.to_string()))?; + let (arity, _) = decode_meta(raw.value())?; + Ok(arity as usize) + } + + fn scan(&self, name: &str) -> Result>, StorageError> { + let txn = self.db.begin_read().map_err(backend)?; + let meta = txn.open_table(meta_def()).map_err(backend)?; + if meta.get(name).map_err(backend)?.is_none() { + return Err(StorageError::RelationNotFound(name.to_string())); + } + let table = txn.open_table(rows_def(name)).map_err(backend)?; + let mut rows = Vec::new(); + for entry in table.iter().map_err(backend)? { + let (_, value) = entry.map_err(backend)?; + rows.push(decode_row(value.value())?); + } + Ok(rows) + } + + fn insert(&mut self, name: &str, row: Vec) -> Result<(), StorageError> { + let txn = self.db.begin_write().map_err(backend)?; + let (arity, next_id) = { + let meta = txn.open_table(meta_def()).map_err(backend)?; + let entry = meta + .get(name) + .map_err(backend)? + .ok_or_else(|| StorageError::RelationNotFound(name.to_string()))?; + decode_meta(entry.value())? + }; + if row.len() != arity as usize { + return Err(StorageError::ArityMismatch { + expected: arity as usize, + got: row.len(), + }); + } + { + let mut rows = txn.open_table(rows_def(name)).map_err(backend)?; + let encoded = encode_row(&row); + rows.insert(next_id, &encoded[..]).map_err(backend)?; + } + { + let mut meta = txn.open_table(meta_def()).map_err(backend)?; + let new_meta = encode_meta(arity, next_id + 1); + meta.insert(name, new_meta.as_ref()).map_err(backend)?; + } + txn.commit().map_err(backend)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn i(x: i64) -> Value { + Value::Int(x) + } + + fn s(x: &str) -> Value { + Value::Str(x.to_string()) + } + + fn open_temp() -> Result { + let dir = tempfile::tempdir().map_err(backend)?; + // The file does not have to exist for redb::create. + let path = dir.path().join("test.redb"); + let storage = RedbStorage::open(&path)?; + // Keep the tempdir alive by leaking it (test-only). + std::mem::forget(dir); + Ok(storage) + } + + #[test] + fn create_insert_scan_roundtrip() -> Result<(), StorageError> { + let mut storage = open_temp()?; + storage.create_relation("edge", 2)?; + storage.insert("edge", vec![i(1), i(2)])?; + storage.insert("edge", vec![s("hello"), i(7)])?; + let rows = storage.scan("edge")?; + assert_eq!(rows, vec![vec![i(1), i(2)], vec![s("hello"), i(7)]]); + assert_eq!(storage.arity("edge")?, 2); + Ok(()) + } + + #[test] + fn duplicate_create_returns_err() -> Result<(), StorageError> { + let mut storage = open_temp()?; + storage.create_relation("edge", 2)?; + assert!(matches!( + storage.create_relation("edge", 2), + Err(StorageError::RelationExists(_)) + )); + Ok(()) + } + + #[test] + fn insert_wrong_arity_returns_err() -> Result<(), StorageError> { + let mut storage = open_temp()?; + storage.create_relation("edge", 2)?; + assert!(matches!( + storage.insert("edge", vec![i(1)]), + Err(StorageError::ArityMismatch { + expected: 2, + got: 1, + }) + )); + Ok(()) + } +} diff --git a/crates/query-storage/src/sled.rs b/crates/query-storage/src/sled.rs new file mode 100644 index 0000000..421121a --- /dev/null +++ b/crates/query-storage/src/sled.rs @@ -0,0 +1,161 @@ +//! Sled adapter. +//! +//! Maps each relation onto a sled [`Tree`](sled::Tree) of the same name. A +//! reserved tree named `__meta` carries per-relation metadata (arity and the +//! next synthetic row ID). + +use query_ops::value::Value; + +use crate::codec::{decode_meta, decode_row, encode_meta, encode_row, row_key}; +use crate::{Storage, StorageError}; + +const META_TREE: &str = "__meta"; + +fn backend(err: E) -> StorageError { + StorageError::Backend(Box::new(err)) +} + +/// Sled-backed [`Storage`] implementation. +pub struct SledStorage { + db: sled::Db, +} + +impl SledStorage { + /// Open or create a sled database at `path`. + /// + /// # Errors + /// Returns [`StorageError::Backend`] if sled fails to open the path. + pub fn open(path: impl AsRef) -> Result { + let db = sled::open(path).map_err(backend)?; + Ok(Self { db }) + } + + fn meta_tree(&self) -> Result { + self.db.open_tree(META_TREE).map_err(backend) + } + + fn relation_tree(&self, name: &str) -> Result { + self.db.open_tree(name).map_err(backend) + } + + fn load_meta(&self, name: &str) -> Result<(u32, u64), StorageError> { + let meta = self.meta_tree()?; + let raw = meta + .get(name.as_bytes()) + .map_err(backend)? + .ok_or_else(|| StorageError::RelationNotFound(name.to_string()))?; + Ok(decode_meta(raw.as_ref())?) + } + + fn store_meta(&self, name: &str, arity: u32, next_id: u64) -> Result<(), StorageError> { + let meta = self.meta_tree()?; + let encoded = encode_meta(arity, next_id); + meta.insert(name.as_bytes(), encoded.as_ref()) + .map_err(backend)?; + Ok(()) + } +} + +impl Storage for SledStorage { + fn create_relation(&mut self, name: &str, arity: usize) -> Result<(), StorageError> { + if name == META_TREE { + return Err(StorageError::Validation(format!( + "relation name '{name}' is reserved" + ))); + } + let meta = self.meta_tree()?; + if meta.contains_key(name.as_bytes()).map_err(backend)? { + return Err(StorageError::RelationExists(name.to_string())); + } + let arity_u32 = u32::try_from(arity) + .map_err(|_| StorageError::Validation(format!("arity {arity} exceeds u32 range")))?; + self.store_meta(name, arity_u32, 0)?; + // open_tree creates the tree if it doesn't exist + let _ = self.relation_tree(name)?; + Ok(()) + } + + fn arity(&self, name: &str) -> Result { + let (arity, _) = self.load_meta(name)?; + Ok(arity as usize) + } + + fn scan(&self, name: &str) -> Result>, StorageError> { + let _ = self.load_meta(name)?; + let tree = self.relation_tree(name)?; + let mut rows = Vec::new(); + for entry in &tree { + let (_, value) = entry.map_err(backend)?; + rows.push(decode_row(value.as_ref())?); + } + Ok(rows) + } + + fn insert(&mut self, name: &str, row: Vec) -> Result<(), StorageError> { + let (arity, next_id) = self.load_meta(name)?; + if row.len() != arity as usize { + return Err(StorageError::ArityMismatch { + expected: arity as usize, + got: row.len(), + }); + } + let tree = self.relation_tree(name)?; + tree.insert(row_key(next_id), encode_row(&row)) + .map_err(backend)?; + self.store_meta(name, arity, next_id + 1)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn i(x: i64) -> Value { + Value::Int(x) + } + + #[test] + fn create_insert_scan_roundtrip() -> Result<(), StorageError> { + let dir = tempfile::tempdir().map_err(backend)?; + let mut storage = SledStorage::open(dir.path())?; + storage.create_relation("edge", 2)?; + storage.insert("edge", vec![i(1), i(2)])?; + storage.insert("edge", vec![i(2), i(3)])?; + storage.insert("edge", vec![i(3), i(3)])?; + let rows = storage.scan("edge")?; + assert_eq!( + rows, + vec![vec![i(1), i(2)], vec![i(2), i(3)], vec![i(3), i(3)],], + ); + assert_eq!(storage.arity("edge")?, 2); + Ok(()) + } + + #[test] + fn duplicate_create_returns_err() -> Result<(), StorageError> { + let dir = tempfile::tempdir().map_err(backend)?; + let mut storage = SledStorage::open(dir.path())?; + storage.create_relation("edge", 2)?; + assert!(matches!( + storage.create_relation("edge", 2), + Err(StorageError::RelationExists(_)) + )); + Ok(()) + } + + #[test] + fn insert_wrong_arity_returns_err() -> Result<(), StorageError> { + let dir = tempfile::tempdir().map_err(backend)?; + let mut storage = SledStorage::open(dir.path())?; + storage.create_relation("edge", 2)?; + assert!(matches!( + storage.insert("edge", vec![i(1)]), + Err(StorageError::ArityMismatch { + expected: 2, + got: 1, + }) + )); + Ok(()) + } +} diff --git a/crates/query-storage/tests/integration.rs b/crates/query-storage/tests/integration.rs new file mode 100644 index 0000000..dca2878 --- /dev/null +++ b/crates/query-storage/tests/integration.rs @@ -0,0 +1,40 @@ +//! End-to-end: load rows into [`MemoryStorage`], scan as a [`Table`], +//! run [`scan_atom`] against it. +//! +//! Demonstrates that `query-ops` operators can consume from a storage backend +//! through the [`scan_as_table`] bridge, with no changes to `query-ops` itself. + +use query_ops::atom::{AtomPattern, Term, scan_atom}; +use query_ops::table::Table; +use query_ops::value::Value; +use query_storage::{MemoryStorage, Storage, StorageError, scan_as_table}; + +fn i(x: i64) -> Value { + Value::Int(x) +} + +fn var(name: &str) -> Term { + Term::Var(name.to_string()) +} + +#[test] +fn scan_atom_consumes_from_memory_backend() -> Result<(), StorageError> { + let mut storage = MemoryStorage::new(); + storage.create_relation("edge", 2)?; + storage.insert("edge", vec![i(1), i(2)])?; + storage.insert("edge", vec![i(2), i(2)])?; + storage.insert("edge", vec![i(3), i(3)])?; + storage.insert("edge", vec![i(4), i(5)])?; + + let edge_table: Table = scan_as_table(&storage, "edge")?; + let self_loops = scan_atom( + &edge_table, + &AtomPattern { + columns: vec![var("X"), var("X")], + }, + ); + + assert_eq!(self_loops.columns, vec!["X".to_string()]); + assert_eq!(self_loops.rows, vec![vec![i(2)], vec![i(3)]]); + Ok(()) +}