From ca82b2940d9241b99e694f3c8d597feb5bf1bbc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 9 Oct 2022 21:56:56 +0200 Subject: [PATCH] fix: sending does not work We were inserting one too many 0xff bytes --- Cargo.lock | 47 ++++------------------------- Cargo.toml | 4 +-- src/database/abstraction/rocksdb.rs | 3 ++ src/database/key_value/sending.rs | 4 ++- src/database/mod.rs | 4 +-- src/service/sending/mod.rs | 3 +- 6 files changed, 18 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35bcdeaf..941634e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -198,9 +198,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.60.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6" +checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" dependencies = [ "bitflags", "cexpr", @@ -295,17 +295,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "cc" version = "1.0.73" @@ -1283,17 +1272,14 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "0.8.0+7.4.4" +version = "6.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d" +checksum = "c309a9d2470844aceb9a4a098cf5286154d20596868b75a6b36357d2bb9ca25d" dependencies = [ "bindgen", - "bzip2-sys", "cc", "glob", "libc", - "libz-sys", - "zstd-sys", ] [[package]] @@ -1307,17 +1293,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "libz-sys" -version = "1.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2053,9 +2028,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.19.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc" +checksum = "7a62eca5cacf2c8261128631bed9f045598d40bfbe4b29f5163f0f802f8f44a7" dependencies = [ "libc", "librocksdb-sys", @@ -3286,13 +3261,3 @@ checksum = "70b40401a28d86ce16a330b863b86fd7dbee4d7c940587ab09ab8c019f9e3fdf" dependencies = [ "num-traits", ] - -[[package]] -name = "zstd-sys" -version = "2.0.1+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" -dependencies = [ - "cc", - "libc", -] diff --git a/Cargo.toml b/Cargo.toml index f1ef0bb1..031f2798 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ crossbeam = { version = "0.8.1", optional = true } num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } -rocksdb = { version = "0.19.0", default-features = true, features = ["multi-threaded-cf", "zstd"], optional = true } +rocksdb = { version = "0.17.0", default-features = true, features = ["multi-threaded-cf", "zstd"], optional = true } thread_local = "1.1.3" # used for TURN server authentication @@ -94,7 +94,7 @@ lazy_static = "1.4.0" async-trait = "0.1.57" [features] -default = ["conduit_bin", "backend_sqlite", "jemalloc"] +default = ["conduit_bin", "backend_sqlite", "backend_rocksdb", "jemalloc"] #backend_sled = ["sled"] backend_persy = ["persy", "parking_lot"] backend_sqlite = ["sqlite"] diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index 07277287..96027f6a 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -161,6 +161,7 @@ impl KvTree for RocksDbEngineTree<'_> { self.db .rocks .iterator_cf(&self.cf(), rocksdb::IteratorMode::Start) + //.map(|r| r.unwrap()) .map(|(k, v)| (Vec::from(k), Vec::from(v))), ) } @@ -184,6 +185,7 @@ impl KvTree for RocksDbEngineTree<'_> { }, ), ) + //.map(|r| r.unwrap()) .map(|(k, v)| (Vec::from(k), Vec::from(v))), ) } @@ -224,6 +226,7 @@ impl KvTree for RocksDbEngineTree<'_> { &self.cf(), rocksdb::IteratorMode::From(&prefix, rocksdb::Direction::Forward), ) + //.map(|r| r.unwrap()) .map(|(k, v)| (Vec::from(k), Vec::from(v))) .take_while(move |(k, _)| k.starts_with(&prefix)), ) diff --git a/src/database/key_value/sending.rs b/src/database/key_value/sending.rs index d84bd494..fddbd67d 100644 --- a/src/database/key_value/sending.rs +++ b/src/database/key_value/sending.rs @@ -66,7 +66,6 @@ impl service::sending::Data for KeyValueDatabase { let mut keys = Vec::new(); for (outgoing_kind, event) in requests { let mut key = outgoing_kind.get_prefix(); - key.push(0xff); key.extend_from_slice(if let SendingEventType::Pdu(value) = &event { &**value } else { @@ -139,6 +138,7 @@ fn parse_servercurrentevent( let event = parts .next() .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?; + let server = utils::string_from_bytes(server).map_err(|_| { Error::bad_database("Invalid server bytes in server_currenttransaction") })?; @@ -169,6 +169,7 @@ fn parse_servercurrentevent( let event = parts .next() .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?; + ( OutgoingKind::Push(user_id, pushkey_string), if value.is_empty() { @@ -185,6 +186,7 @@ fn parse_servercurrentevent( let event = parts .next() .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?; + let server = utils::string_from_bytes(server).map_err(|_| { Error::bad_database("Invalid server bytes in server_currenttransaction") })?; diff --git a/src/database/mod.rs b/src/database/mod.rs index 689ab57e..9f893d6f 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -556,7 +556,7 @@ impl KeyValueDatabase { services() .rooms .state_compressor - .load_shortstatehash_info(dbg!(last_roomsstatehash)) + .load_shortstatehash_info(last_roomsstatehash) }, )?; @@ -579,7 +579,7 @@ impl KeyValueDatabase { }; services().rooms.state_compressor.save_state_from_diff( - dbg!(current_sstatehash), + current_sstatehash, statediffnew, statediffremoved, 2, // every state change is 2 event changes on average diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 697ca85c..20c652f5 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -587,7 +587,7 @@ impl Service { .notification_count(&userid, &pdu.room_id) .map_err(|e| (kind.clone(), e))? .try_into() - .expect("notifiation count can't go that high"); + .expect("notification count can't go that high"); let permit = services().sending.maximum_requests.acquire().await; @@ -616,6 +616,7 @@ impl Service { .get_pdu_json_from_id(pdu_id) .map_err(|e| (OutgoingKind::Normal(server.clone()), e))? .ok_or_else(|| { + error!("event not found: {server} {pdu_id:?}"); ( OutgoingKind::Normal(server.clone()), Error::bad_database(