From c4f5a0a6316200f797bf81d77c3cd2815a73706d Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Thu, 6 Aug 2020 08:29:59 -0400 Subject: [PATCH 01/51] Keep track of State at event for state resolution feat: first steps towards joining rooms over federation Add state-res as a dependency of conduit Add reverse_topological_power_sort before append_pdu Implement statehashstatid_pduid tree for keeping track of state Clean up implementation of state_hash as key for tracking state --- Cargo.lock | 206 +++++++++++- Cargo.toml | 7 +- src/client_server/account.rs | 6 +- src/client_server/alias.rs | 16 +- src/client_server/context.rs | 30 +- src/client_server/device.rs | 8 +- src/client_server/directory.rs | 13 +- src/client_server/filter.rs | 21 +- src/client_server/keys.rs | 2 +- src/client_server/media.rs | 4 +- src/client_server/membership.rs | 125 ++++++-- src/client_server/message.rs | 43 +-- src/client_server/room.rs | 2 +- src/client_server/session.rs | 20 +- src/client_server/state.rs | 48 ++- src/client_server/sync.rs | 2 +- src/client_server/unversioned.rs | 13 +- src/database.rs | 5 +- src/database/rooms.rs | 525 +++++++++++++++++++++---------- src/database/uiaa.rs | 6 +- src/pdu.rs | 29 ++ src/ruma_wrapper.rs | 10 +- src/server_server.rs | 27 +- src/utils.rs | 6 + 24 files changed, 818 insertions(+), 356 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a7334ca..ffee8ea5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,6 +75,15 @@ dependencies = [ "opaque-debug 0.2.3", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "arc-swap" version = "0.4.7" @@ -248,6 +257,17 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "chrono" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942f72db697d8767c22d46a598e01f2d3b475501ea43d0db4f16d90259182d0b" +dependencies = [ + "num-integer", + "num-traits", + "time 0.1.43", +] + [[package]] name = "cloudabi" version = "0.1.0" @@ -281,6 +301,7 @@ dependencies = [ "serde", "serde_json", "sled", + "state-res", "thiserror", "tokio", ] @@ -456,6 +477,12 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" +[[package]] +name = "either" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" + [[package]] name = "encoding_rs" version = "0.8.23" @@ -872,6 +899,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.6" @@ -951,6 +987,21 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.8" @@ -1439,6 +1490,31 @@ dependencies = [ "syn", ] +[[package]] +name = "regex" +version = "1.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -1560,21 +1636,23 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-api", + "ruma-appservice-api", "ruma-client-api", "ruma-common", "ruma-events", "ruma-federation-api", "ruma-identifiers", + "ruma-serde", "ruma-signatures", ] [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "http", "percent-encoding", @@ -1589,7 +1667,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1597,14 +1675,28 @@ dependencies = [ "syn", ] +[[package]] +name = "ruma-appservice-api" +version = "0.2.0-alpha.1" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" +dependencies = [ + "ruma-api", + "ruma-common", + "ruma-events", + "ruma-identifiers", + "serde", + "serde_json", +] + [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "assign", "http", "js_int", + "percent-encoding", "ruma-api", "ruma-common", "ruma-events", @@ -1618,7 +1710,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-identifiers", @@ -1631,7 +1723,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-common", @@ -1646,7 +1738,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1657,7 +1749,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-api", @@ -1672,7 +1764,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1684,7 +1776,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro2", "quote", @@ -1695,7 +1787,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-serde", "serde", @@ -1706,7 +1798,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "form_urlencoded", "itoa", @@ -1718,7 +1810,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "base64 0.12.3", "ring", @@ -1910,6 +2002,15 @@ dependencies = [ "opaque-debug 0.3.0", ] +[[package]] +name = "sharded-slab" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06d5a3f5166fb5b42a5439f2eee8b9de149e235961e3eb21c5808fc3ea17ff3e" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.2.1" @@ -1983,6 +2084,22 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" +[[package]] +name = "state-res" +version = "0.1.0" +source = "git+https://github.com/ruma/state-res#789c8140890e076d38b23fa1147c4ff0500c0d38" +dependencies = [ + "itertools", + "js_int", + "maplit", + "ruma", + "serde", + "serde_json", + "thiserror", + "tracing", + "tracing-subscriber", +] + [[package]] name = "stdweb" version = "0.4.20" @@ -2104,6 +2221,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + [[package]] name = "time" version = "0.1.43" @@ -2251,9 +2377,21 @@ checksum = "6d79ca061b032d6ce30c660fded31189ca0b9922bf483cd70759f13a2d86786c" dependencies = [ "cfg-if", "log", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe233f4227389ab7df5b32649239da7ebe0b281824b4e84b342d04d3fd8c25e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.14" @@ -2263,6 +2401,48 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-log" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0f8c7178e13481ff6765bd169b33e8d554c5d2bbede5e32c356194be02b9b9" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ccba2f8f16e0ed268fc765d9b7ff22e965e7185d32f8f1ec8294fe17d86e79" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd165311cc4d7a555ad11cc77a37756df836182db0d81aac908c8184c584f40" +dependencies = [ + "ansi_term", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", + "tracing-serde", +] + [[package]] name = "try-lock" version = "0.2.3" diff --git a/Cargo.toml b/Cargo.toml index 4945e3c8..4c14d712 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,9 +16,7 @@ edition = "2018" #rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } -#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "987d48666cf166cf12100b5dbc61b5e3385c4014" } # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fixes" } # Used for matrix spec type definitions and helpers -#ruma = { path = "../ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } +ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers tokio = "0.2.22" # Used for long polling sled = "0.32.0" # Used for storing data permanently log = "0.4.8" # Used for emitting log entries @@ -33,6 +31,9 @@ reqwest = "0.10.6" # Used to send requests thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key +# state-res = { path = "../../state-res" } +state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0" } + [features] default = ["conduit_bin"] diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 9837d1b9..9fa1a9c7 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -75,7 +75,7 @@ pub fn get_register_available_route( )] pub fn register_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { if db.globals.registration_disabled() { return Err(Error::BadRequest( @@ -223,7 +223,7 @@ pub fn register_route( )] pub fn change_password_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -305,7 +305,7 @@ pub fn whoami_route(body: Ruma) -> ConduitResult, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index a0293881..7dc90783 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -26,7 +26,7 @@ pub fn create_alias_route( db.rooms .set_alias(&body.room_alias, Some(&body.room_id), &db.globals)?; - Ok(create_alias::Response.into()) + Ok(create_alias::Response::new().into()) } #[cfg_attr( @@ -39,7 +39,7 @@ pub fn delete_alias_route( ) -> ConduitResult { db.rooms.set_alias(&body.room_alias, None, &db.globals)?; - Ok(delete_alias::Response.into()) + Ok(delete_alias::Response::new().into()) } #[cfg_attr( @@ -60,11 +60,7 @@ pub async fn get_alias_route( ) .await?; - return Ok(get_alias::Response { - room_id: response.room_id, - servers: response.servers, - } - .into()); + return Ok(get_alias::Response::new(response.room_id, response.servers).into()); } let room_id = db @@ -75,9 +71,5 @@ pub async fn get_alias_route( "Room with alias not found.", ))?; - Ok(get_alias::Response { - room_id, - servers: vec![db.globals.server_name().to_string()], - } - .into()) + Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_string()]).into()) } diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 7a6cbce0..7b1fad91 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -12,7 +12,7 @@ use rocket::get; )] pub fn get_context_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -75,18 +75,18 @@ pub fn get_context_route( .map(|(_, pdu)| pdu.to_room_event()) .collect::>(); - Ok(get_context::Response { - start: start_token, - end: end_token, - events_before, - event: Some(base_event), - events_after, - state: db // TODO: State at event - .rooms - .room_state_full(&body.room_id)? - .values() - .map(|pdu| pdu.to_state_event()) - .collect(), - } - .into()) + let mut resp = get_context::Response::new(); + resp.start = start_token; + resp.end = end_token; + resp.events_before = events_before; + resp.event = Some(base_event); + resp.events_after = events_after; + resp.state = db // TODO: State at event + .rooms + .room_state_full(&body.room_id)? + .values() + .map(|pdu| pdu.to_state_event()) + .collect(); + + Ok(resp.into()) } diff --git a/src/client_server/device.rs b/src/client_server/device.rs index 379f827e..89033f06 100644 --- a/src/client_server/device.rs +++ b/src/client_server/device.rs @@ -37,7 +37,7 @@ pub fn get_devices_route( )] pub fn get_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,7 @@ pub fn get_device_route( )] pub fn update_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -80,7 +80,7 @@ pub fn update_device_route( )] pub fn delete_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -127,7 +127,7 @@ pub fn delete_device_route( )] pub fn delete_devices_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 26188f73..0aace15d 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -6,7 +6,7 @@ use ruma::{ error::ErrorKind, r0::{ directory::{ - self, get_public_rooms, get_public_rooms_filtered, get_room_visibility, + get_public_rooms, get_public_rooms_filtered, get_room_visibility, set_room_visibility, }, room, @@ -14,6 +14,7 @@ use ruma::{ }, federation, }, + directory::PublicRoomsChunk, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, @@ -35,15 +36,15 @@ pub async fn get_public_rooms_filtered_route( if let Some(other_server) = body .server .clone() - .filter(|server| server != &db.globals.server_name().as_str()) + .filter(|server| server != db.globals.server_name().as_str()) { let response = server_server::send_request( &db, other_server, federation::directory::get_public_rooms::v1::Request { limit: body.limit, - since: body.since.clone(), - room_network: federation::directory::get_public_rooms::v1::RoomNetwork::Matrix, + since: body.since.as_deref(), + room_network: ruma::directory::RoomNetwork::Matrix, }, ) .await?; @@ -107,7 +108,7 @@ pub async fn get_public_rooms_filtered_route( // TODO: Do not load full state? let state = db.rooms.room_state_full(&room_id)?; - let chunk = directory::PublicRoomsChunk { + let chunk = PublicRoomsChunk { aliases: Vec::new(), canonical_alias: state .get(&(EventType::RoomCanonicalAlias, "".to_owned())) @@ -272,7 +273,7 @@ pub async fn get_public_rooms_route( body: get_public_rooms_filtered::IncomingRequest { filter: None, limit, - room_network: get_public_rooms_filtered::RoomNetwork::Matrix, + room_network: ruma::directory::RoomNetwork::Matrix, server, since, }, diff --git a/src/client_server/filter.rs b/src/client_server/filter.rs index 165419a8..4322de3e 100644 --- a/src/client_server/filter.rs +++ b/src/client_server/filter.rs @@ -7,23 +7,18 @@ use rocket::{get, post}; #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/user/<_>/filter/<_>"))] pub fn get_filter_route() -> ConduitResult { // TODO - Ok(get_filter::Response { - filter: filter::FilterDefinition { - event_fields: None, - event_format: None, - account_data: None, - room: None, - presence: None, - }, - } + Ok(get_filter::Response::new(filter::FilterDefinition { + event_fields: None, + event_format: None, + account_data: None, + room: None, + presence: None, + }) .into()) } #[cfg_attr(feature = "conduit_bin", post("/_matrix/client/r0/user/<_>/filter"))] pub fn create_filter_route() -> ConduitResult { // TODO - Ok(create_filter::Response { - filter_id: utils::random_string(10), - } - .into()) + Ok(create_filter::Response::new(utils::random_string(10)).into()) } diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index f88878ce..33115296 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -167,7 +167,7 @@ pub fn claim_keys_route( )] pub fn upload_signing_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/media.rs b/src/client_server/media.rs index efcb3a6d..79c1f080 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -53,7 +53,7 @@ pub fn create_content_route( )] pub fn get_content_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _server_name: String, _media_id: String, ) -> ConduitResult { @@ -85,7 +85,7 @@ pub fn get_content_route( )] pub fn get_content_thumbnail_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _server_name: String, _media_id: String, ) -> ConduitResult { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 84c0ebd3..c04cf7f6 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -20,6 +20,8 @@ use ruma::{ events::{room::member, EventType}, EventId, Raw, RoomId, RoomVersionId, }; +use state_res::StateEvent; + use std::{collections::BTreeMap, convert::TryFrom}; #[cfg(feature = "conduit_bin")] @@ -92,17 +94,73 @@ pub async fn join_room_by_id_route( let send_join_response = server_server::send_request( &db, body.room_id.server_name().to_string(), - federation::membership::create_join_event::v2::Request { + federation::membership::create_join_event::v1::Request { room_id: body.room_id.clone(), event_id, - pdu_stub: serde_json::from_value::>(join_event_stub_value) + pdu_stub: serde_json::from_value(join_event_stub_value) .expect("Raw::from_value always works"), }, ) .await?; - dbg!(send_join_response); - todo!("Take send_join_response and 'create' the room using that data"); + dbg!(&send_join_response); + // todo!("Take send_join_response and 'create' the room using that data"); + + let mut event_map = send_join_response + .room_state + .state + .iter() + .map(|pdu| pdu.deserialize().map(StateEvent::Full)) + .map(|ev| { + let ev = ev?; + Ok::<_, serde_json::Error>((ev.event_id(), ev)) + }) + .collect::, _>>() + .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; + + let _auth_chain = send_join_response + .room_state + .auth_chain + .iter() + .flat_map(|pdu| pdu.deserialize().ok()) + .map(StateEvent::Full) + .collect::>(); + + // TODO make StateResolution's methods free functions ? or no self param ? + let sorted_events_ids = state_res::StateResolution::default() + .reverse_topological_power_sort( + &body.room_id, + &event_map.keys().cloned().collect::>(), + &mut event_map, + &db.rooms, + &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort + ); + + for ev_id in &sorted_events_ids { + // this is a `state_res::StateEvent` that holds a `ruma::Pdu` + let pdu = event_map.get(ev_id).ok_or_else(|| { + Error::Conflict("Found event_id in sorted events that is not in resolved state") + })?; + + db.rooms.append_pdu( + PduBuilder { + room_id: pdu.room_id().unwrap_or(&body.room_id).clone(), + sender: pdu.sender().clone(), + event_type: pdu.kind(), + content: pdu.content().clone(), + unsigned: Some( + pdu.unsigned() + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + ), + state_key: pdu.state_key(), + redacts: pdu.redacts().cloned(), + }, + &db.globals, + &db.account_data, + )?; + } } let event = member::MemberEventContent { @@ -127,10 +185,7 @@ pub async fn join_room_by_id_route( &db.account_data, )?; - Ok(join_room_by_id::Response { - room_id: body.room_id.clone(), - } - .into()) + Ok(join_room_by_id::Response::new(body.room_id.clone()).into()) } #[cfg_attr( @@ -140,7 +195,7 @@ pub async fn join_room_by_id_route( pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, db2: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, @@ -148,7 +203,13 @@ pub async fn join_room_by_id_or_alias_route( client_server::get_alias_route( db, Ruma { - body: alias::get_alias::IncomingRequest { room_alias }, + body: alias::get_alias::IncomingRequest::try_from(http::Request::new( + serde_json::json!({ "room_alias": room_alias }) + .to_string() + .as_bytes() + .to_vec(), + )) + .unwrap(), sender_id: body.sender_id.clone(), device_id: body.device_id.clone(), json_body: None, @@ -160,14 +221,32 @@ pub async fn join_room_by_id_or_alias_route( } }; + // TODO ruma needs to implement the same constructors for the Incoming variants + let tps = if let Some(in_tps) = &body.third_party_signed { + Some(ruma::api::client::r0::membership::ThirdPartySigned { + token: &in_tps.token, + sender: &in_tps.sender, + signatures: in_tps.signatures.clone(), + mxid: &in_tps.mxid, + }) + } else { + None + }; + let body = Ruma { sender_id: body.sender_id.clone(), device_id: body.device_id.clone(), json_body: None, - body: join_room_by_id::IncomingRequest { - room_id, - third_party_signed: body.third_party_signed.clone(), - }, + body: join_room_by_id::IncomingRequest::try_from(http::Request::new( + serde_json::json!({ + "room_id": room_id, + "third_party_signed": tps, + }) + .to_string() + .as_bytes() + .to_vec(), + )) + .unwrap(), }; Ok(join_room_by_id_or_alias::Response { @@ -219,7 +298,7 @@ pub fn leave_room_route( &db.account_data, )?; - Ok(leave_room::Response.into()) + Ok(leave_room::Response::new().into()) } #[cfg_attr( @@ -266,7 +345,7 @@ pub fn invite_user_route( )] pub fn kick_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -304,7 +383,7 @@ pub fn kick_user_route( &db.account_data, )?; - Ok(kick_user::Response.into()) + Ok(kick_user::Response::new().into()) } #[cfg_attr( @@ -313,7 +392,7 @@ pub fn kick_user_route( )] pub fn ban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -359,7 +438,7 @@ pub fn ban_user_route( &db.account_data, )?; - Ok(ban_user::Response.into()) + Ok(ban_user::Response::new().into()) } #[cfg_attr( @@ -368,7 +447,7 @@ pub fn ban_user_route( )] pub fn unban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -405,7 +484,7 @@ pub fn unban_user_route( &db.account_data, )?; - Ok(unban_user::Response.into()) + Ok(unban_user::Response::new().into()) } #[cfg_attr( @@ -414,13 +493,13 @@ pub fn unban_user_route( )] pub fn forget_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); db.rooms.forget(&body.room_id, &sender_id)?; - Ok(forget_room::Response.into()) + Ok(forget_room::Response::new().into()) } #[cfg_attr( diff --git a/src/client_server/message.rs b/src/client_server/message.rs index d8512148..1b461d24 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -1,8 +1,11 @@ use super::State; use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Ruma}; -use ruma::api::client::{ - error::ErrorKind, - r0::message::{get_message_events, send_message_event}, +use ruma::{ + api::client::{ + error::ErrorKind, + r0::message::{get_message_events, send_message_event}, + }, + events::EventContent, }; use std::convert::TryInto; @@ -26,7 +29,7 @@ pub fn send_message_event_route( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), - event_type: body.event_type.clone(), + event_type: body.content.event_type().into(), content: serde_json::from_str( body.json_body .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? @@ -41,7 +44,7 @@ pub fn send_message_event_route( &db.account_data, )?; - Ok(send_message_event::Response { event_id }.into()) + Ok(send_message_event::Response::new(event_id).into()) } #[cfg_attr( @@ -50,7 +53,7 @@ pub fn send_message_event_route( )] pub fn get_message_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -92,13 +95,13 @@ pub fn get_message_events_route( .map(|(_, pdu)| pdu.to_room_event()) .collect::>(); - Ok(get_message_events::Response { - start: Some(body.from.clone()), - end: end_token, - chunk: events_after, - state: Vec::new(), - } - .into()) + let mut resp = get_message_events::Response::new(); + resp.start = Some(body.from.clone()); + resp.end = end_token; + resp.chunk = events_after; + resp.state = Vec::new(); + + Ok(resp.into()) } get_message_events::Direction::Backward => { let events_before = db @@ -116,13 +119,13 @@ pub fn get_message_events_route( .map(|(_, pdu)| pdu.to_room_event()) .collect::>(); - Ok(get_message_events::Response { - start: Some(body.from.clone()), - end: start_token, - chunk: events_before, - state: Vec::new(), - } - .into()) + let mut resp = get_message_events::Response::new(); + resp.start = Some(body.from.clone()); + resp.end = start_token; + resp.chunk = events_before; + resp.state = Vec::new(); + + Ok(resp.into()) } } } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index b5f1529e..589a2dcd 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -315,7 +315,7 @@ pub fn create_room_route( db.rooms.set_public(&room_id, true)?; } - Ok(create_room::Response { room_id }.into()) + Ok(create_room::Response::new(room_id).into()) } #[cfg_attr( diff --git a/src/client_server/session.rs b/src/client_server/session.rs index 40110588..948b455b 100644 --- a/src/client_server/session.rs +++ b/src/client_server/session.rs @@ -1,5 +1,4 @@ -use super::State; -use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH}; +use super::{State, DEVICE_ID_LENGTH, TOKEN_LENGTH}; use crate::{utils, ConduitResult, Database, Error, Ruma}; use ruma::{ api::client::{ @@ -18,10 +17,7 @@ use rocket::{get, post}; /// when logging in. #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/login"))] pub fn get_login_types_route() -> ConduitResult { - Ok(get_login_types::Response { - flows: vec![get_login_types::LoginType::Password], - } - .into()) + Ok(get_login_types::Response::new(vec![get_login_types::LoginType::Password]).into()) } /// # `POST /_matrix/client/r0/login` @@ -40,15 +36,15 @@ pub fn get_login_types_route() -> ConduitResult { )] pub fn login_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { // Validate login method let user_id = // TODO: Other login methods - if let (login::UserInfo::MatrixId(username), login::LoginInfo::Password { password }) = - (body.user.clone(), body.login_info.clone()) + if let (login::IncomingUserInfo::MatrixId(username), login::IncomingLoginInfo::Password { password }) = + (&body.user, &body.login_info) { - let user_id = UserId::parse_with_server_name(username, db.globals.server_name()) + let user_id = UserId::parse_with_server_name(username.to_string(), db.globals.server_name()) .map_err(|_| Error::BadRequest( ErrorKind::InvalidUsername, "Username is invalid." @@ -126,7 +122,7 @@ pub fn logout_route( db.users.remove_device(&sender_id, device_id)?; - Ok(logout::Response.into()) + Ok(logout::Response::new().into()) } /// # `POST /_matrix/client/r0/logout/all` @@ -154,5 +150,5 @@ pub fn logout_all_route( } } - Ok(logout_all::Response.into()) + Ok(logout_all::Response::new().into()) } diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 60b3e9f1..14cc4972 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -8,9 +8,9 @@ use ruma::{ send_state_event_for_empty_key, send_state_event_for_key, }, }, - events::{room::canonical_alias, EventType}, - Raw, + events::{AnyStateEventContent, EventContent}, }; +use std::convert::TryFrom; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; @@ -33,17 +33,10 @@ pub fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - if body.event_type == EventType::RoomCanonicalAlias { - let canonical_alias = serde_json::from_value::< - Raw, - >(content.clone()) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid canonical alias."))?; + if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = &body.content { + let mut aliases = canonical_alias.alt_aliases.clone(); - let mut aliases = canonical_alias.alt_aliases; - - if let Some(alias) = canonical_alias.alias { + if let Some(alias) = canonical_alias.alias.clone() { aliases.push(alias); } @@ -68,7 +61,7 @@ pub fn send_state_event_for_key_route( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), - event_type: body.event_type.clone(), + event_type: body.content.event_type().into(), content, unsigned: None, state_key: Some(body.state_key.clone()), @@ -78,7 +71,7 @@ pub fn send_state_event_for_key_route( &db.account_data, )?; - Ok(send_state_event_for_key::Response { event_id }.into()) + Ok(send_state_event_for_key::Response::new(event_id).into()) } #[cfg_attr( @@ -93,25 +86,28 @@ pub fn send_state_event_for_empty_key_route( let Ruma { body: send_state_event_for_empty_key::IncomingRequest { - room_id, - event_type, - data, + room_id, content, .. }, sender_id, device_id, json_body, } = body; - Ok(send_state_event_for_empty_key::Response { - event_id: send_state_event_for_key_route( + Ok(send_state_event_for_empty_key::Response::new( + send_state_event_for_key_route( db, Ruma { - body: send_state_event_for_key::IncomingRequest { - room_id, - event_type, - data, - state_key: "".to_owned(), - }, + body: send_state_event_for_key::IncomingRequest::try_from(http::Request::new( + serde_json::json!({ + "room_id": room_id, + "state_key": "", + "content": content, + }) + .to_string() + .as_bytes() + .to_vec(), + )) + .unwrap(), sender_id, device_id, json_body, @@ -119,7 +115,7 @@ pub fn send_state_event_for_empty_key_route( )? .0 .event_id, - } + ) .into()) } diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 2307f028..ae4c2242 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -31,7 +31,7 @@ use std::{ )] pub async fn sync_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/unversioned.rs b/src/client_server/unversioned.rs index 3ff8bec2..ea7f6338 100644 --- a/src/client_server/unversioned.rs +++ b/src/client_server/unversioned.rs @@ -1,6 +1,5 @@ use crate::ConduitResult; use ruma::api::client::unversioned::get_supported_versions; -use std::collections::BTreeMap; #[cfg(feature = "conduit_bin")] use rocket::get; @@ -17,13 +16,11 @@ use rocket::get; /// unstable features in their stable releases #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/versions"))] pub fn get_supported_versions_route() -> ConduitResult { - let mut unstable_features = BTreeMap::new(); + let mut resp = + get_supported_versions::Response::new(vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()]); - unstable_features.insert("org.matrix.e2e_cross_signing".to_owned(), true); + resp.unstable_features + .insert("org.matrix.e2e_cross_signing".to_owned(), true); - Ok(get_supported_versions::Response { - versions: vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()], - unstable_features, - } - .into()) + Ok(resp.into()) } diff --git a/src/database.rs b/src/database.rs index 7bbb6dd7..6cd65c31 100644 --- a/src/database.rs +++ b/src/database.rs @@ -97,8 +97,8 @@ impl Database { }, pduid_pdu: db.open_tree("pduid_pdu")?, eventid_pduid: db.open_tree("eventid_pduid")?, + roomstateid_pduid: db.open_tree("roomstateid_pduid")?, roomid_pduleaves: db.open_tree("roomid_pduleaves")?, - roomstateid_pdu: db.open_tree("roomstateid_pdu")?, alias_roomid: db.open_tree("alias_roomid")?, aliasid_alias: db.open_tree("alias_roomid")?, @@ -111,6 +111,9 @@ impl Database { userroomid_invited: db.open_tree("userroomid_invited")?, roomuserid_invited: db.open_tree("roomuserid_invited")?, userroomid_left: db.open_tree("userroomid_left")?, + + stateid_pduid: db.open_tree("stateid_pduid")?, + pduid_statehash: db.open_tree("pduid_statehash")?, }, account_data: account_data::AccountData { roomuserdataid_accountdata: db.open_tree("roomuserdataid_accountdata")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index d2cd5e9f..0d363262 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -9,7 +9,7 @@ use ruma::{ events::{ ignored_user_list, room::{ - join_rules, member, + member, power_levels::{self, PowerLevelsEventContent}, redaction, }, @@ -18,19 +18,31 @@ use ruma::{ EventId, Raw, RoomAliasId, RoomId, UserId, }; use sled::IVec; +use state_res::{event_auth, Requester, StateEvent, StateMap, StateStore}; + use std::{ - collections::{BTreeMap, HashMap}, + collections::{hash_map::DefaultHasher, BTreeMap, HashMap}, convert::{TryFrom, TryInto}, + hash::{Hash, Hasher}, mem, + result::Result as StdResult, }; +/// The unique identifier of each state group. +/// +/// This is created when a state group is added to the database by +/// hashing the entire state. +pub type StateHashId = String; + +/// This identifier consists of roomId + count. It represents a +/// unique event, it will never be overwritten or removed. +pub type PduId = IVec; + pub struct Rooms { pub edus: edus::RoomEdus, pub(super) pduid_pdu: sled::Tree, // PduId = RoomId + Count pub(super) eventid_pduid: sled::Tree, pub(super) roomid_pduleaves: sled::Tree, - pub(super) roomstateid_pdu: sled::Tree, // RoomStateId = Room + StateType + StateKey - pub(super) alias_roomid: sled::Tree, pub(super) aliasid_alias: sled::Tree, // AliasId = RoomId + Count pub(super) publicroomids: sled::Tree, @@ -42,9 +54,263 @@ pub struct Rooms { pub(super) userroomid_invited: sled::Tree, pub(super) roomuserid_invited: sled::Tree, pub(super) userroomid_left: sled::Tree, + + // STATE TREES + /// This holds the full current state, including the latest event. + pub(super) roomstateid_pduid: sled::Tree, // RoomStateId = Room + StateType + StateKey + /// This holds the full room state minus the latest event. + pub(super) pduid_statehash: sled::Tree, // PDU id -> StateHash + /// Also holds the full room state minus the latest event. + pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + (EventType, StateKey) +} + +impl StateStore for Rooms { + fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> StdResult { + let pid = self + .eventid_pduid + .get(event_id.as_bytes()) + .map_err(|e| e.to_string())? + .ok_or_else(|| "PDU via room_id and event_id not found in the db.".to_owned())?; + + utils::deserialize( + &self + .pduid_pdu + .get(pid) + .map_err(|e| e.to_string())? + .ok_or_else(|| "PDU via pduid not found in db.".to_owned())?, + ) + .and_then(|pdu: StateEvent| { + // conduit's PDU's always contain a room_id but some + // of ruma's do not so this must be an Option + if pdu.room_id() == Some(room_id) { + Ok(pdu) + } else { + Err(Error::bad_database("Found PDU for incorrect room in db.")) + } + }) + .map_err(|e| e.to_string()) + } } +// These are the methods related to STATE resolution. impl Rooms { + /// Generates a new StateHash and associates it with the incoming event. + /// + /// This adds all current state events (not including the incoming event) + /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. + /// The incoming event is the `pdu_id` passed to this method. + pub fn append_state_pdu(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result { + let state_hash = self.new_state_hash_id(room_id)?; + let state = self.current_state_pduids(room_id)?; + + let mut key = state_hash.as_bytes().to_vec(); + key.push(0xff); + + // TODO eventually we could avoid writing to the DB so much on every event + // by keeping track of the delta and write that every so often + for ((ev_ty, state_key), pid) in state { + let mut state_id = key.to_vec(); + state_id.extend_from_slice(ev_ty.to_string().as_bytes()); + key.push(0xff); + state_id.extend_from_slice(state_key.expect("state event").as_bytes()); + key.push(0xff); + + self.stateid_pduid.insert(&state_id, &pid)?; + } + + // This event's state does not include the event itself. `current_state_pduids` + // uses `roomstateid_pduid` before the current event is inserted to the tree so the state + // will be everything up to but not including the incoming event. + self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + + Ok(state_hash) + } + + /// Builds a `StateMap` by iterating over all keys that start + /// with `state_hash`, this gives the full state at event "x". + pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { + self.stateid_pduid + .scan_prefix(state_hash.as_bytes()) + .values() + .map(|pduid| { + self.pduid_pdu.get(&pduid?)?.map_or_else( + || Err(Error::bad_database("Failed to find StateMap.")), + |b| { + serde_json::from_slice::(&b) + .map_err(|_| Error::bad_database("Invalid PDU in db.")) + }, + ) + }) + .map(|pdu| { + let pdu = pdu?; + Ok(((pdu.kind, pdu.state_key), pdu.event_id)) + }) + .collect::>>() + } + + // TODO make this return Result + /// Fetches the previous StateHash ID to `current`. + pub fn prev_state_hash(&self, current: StateHashId) -> Option { + let mut found = false; + for pair in self.pduid_statehash.iter().rev() { + let prev = utils::string_from_bytes(&pair.ok()?.1).ok()?; + if current == prev { + found = true; + } + if current != prev && found { + return Some(prev); + } + } + None + } + + /// Fetch the current State using the `roomstateid_pduid` tree. + pub fn current_state_pduids(&self, room_id: &RoomId) -> Result> { + // TODO this could also scan roomstateid_pduid if we passed in room_id ? + self.roomstateid_pduid + .scan_prefix(room_id.as_bytes()) + .values() + .map(|pduid| { + let pduid = &pduid?; + self.pduid_pdu.get(pduid)?.map_or_else( + || { + Err(Error::bad_database( + "Failed to find current state of pduid's.", + )) + }, + |b| { + Ok(( + serde_json::from_slice::(&b) + .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + pduid.clone(), + )) + }, + ) + }) + .map(|pair| { + let (pdu, id) = pair?; + Ok(((pdu.kind, pdu.state_key), id)) + }) + .collect::>>() + } + + /// Returns the last state hash key added to the db. + pub fn current_state_hash(&self, room_id: &RoomId) -> Result { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + + // We must check here because this method is called outside and before + // `append_state_pdu` so the DB can be empty + if self.pduid_statehash.scan_prefix(prefix).next().is_none() { + // TODO use ring crate to hash + return Ok(room_id.as_str().to_owned()); + } + + self.pduid_statehash + .iter() + .next_back() + .map(|pair| { + utils::string_from_bytes(&pair?.1) + .map_err(|_| Error::bad_database("Invalid state hash string in db.")) + }) + .ok_or_else(|| Error::bad_database("No PDU's found for this room."))? + } + + /// This fetches auth event_ids from the current state using the + /// full `roomstateid_pdu` tree. + pub fn get_auth_event_ids( + &self, + room_id: &RoomId, + kind: &EventType, + sender: &UserId, + state_key: Option<&str>, + content: serde_json::Value, + ) -> Result> { + let auth_events = state_res::auth_types_for_event( + kind.clone(), + sender, + state_key.map(|s| s.to_string()), + content, + ); + + let mut events = vec![]; + for (event_type, state_key) in auth_events { + if let Some(state_key) = state_key.as_ref() { + if let Some(id) = self.room_state_get(room_id, &event_type, state_key)? { + events.push(id.event_id); + } + } + } + Ok(events) + } + + // This fetches auth events from the current state using the + /// full `roomstateid_pdu` tree. + pub fn get_auth_events( + &self, + room_id: &RoomId, + kind: &EventType, + sender: &UserId, + state_key: Option<&str>, + content: serde_json::Value, + ) -> Result> { + let auth_events = state_res::auth_types_for_event( + kind.clone(), + sender, + state_key.map(|s| s.to_string()), + content, + ); + + let mut events = StateMap::new(); + for (event_type, state_key) in auth_events { + if let Some(s_key) = state_key.as_ref() { + if let Some(pdu) = self.room_state_get(room_id, &event_type, s_key)? { + events.insert((event_type, state_key), pdu); + } + } + } + Ok(events) + } + + /// Generate a new StateHash. + /// + /// A unique hash made from hashing the current states pduid's. + /// Because `append_state_pdu` handles the empty state db case it does not + /// have to be here. + fn new_state_hash_id(&self, room_id: &RoomId) -> Result { + // Use hashed roomId as the first StateHash key for first state event in room + if self + .pduid_statehash + .scan_prefix(room_id.as_bytes()) + .next() + .is_none() + { + // TODO use ring crate to hash + return Ok(room_id.as_str().to_owned()); + } + + let pdu_ids_to_hash = self + .pduid_statehash + .scan_prefix(room_id.as_bytes()) + .values() + .next_back() + .unwrap() // We just checked if the tree was empty + .map(|hash| { + self.stateid_pduid + .scan_prefix(hash) + .values() + // pduid is roomId + count so just hash the whole thing + .map(|pid| Ok(pid?.to_vec())) + .collect::>>>() + })??; + + let mut hasher = DefaultHasher::new(); + pdu_ids_to_hash.hash(&mut hasher); + let hash = hasher.finish().to_string(); + // TODO not sure how you want to hash this + Ok(hash) + } + /// Checks if a room exists. pub fn exists(&self, room_id: &RoomId) -> Result { let mut prefix = room_id.to_string().as_bytes().to_vec(); @@ -64,16 +330,20 @@ impl Rooms { room_id: &RoomId, ) -> Result> { let mut hashmap = HashMap::new(); - for pdu in self - .roomstateid_pdu - .scan_prefix(&room_id.to_string().as_bytes()) - .values() - .map(|value| { - Ok::<_, Error>( - serde_json::from_slice::(&value?) + for pdu in + self.roomstateid_pduid + .scan_prefix(&room_id.to_string().as_bytes()) + .values() + .map(|value| { + Ok::<_, Error>( + serde_json::from_slice::( + &self.pduid_pdu.get(value?)?.ok_or_else(|| { + Error::bad_database("PDU not found for ID in db.") + })?, + ) .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - ) - }) + ) + }) { let pdu = pdu?; let state_key = pdu.state_key.clone().ok_or_else(|| { @@ -95,16 +365,20 @@ impl Rooms { prefix.extend_from_slice(&event_type.to_string().as_bytes()); let mut hashmap = HashMap::new(); - for pdu in self - .roomstateid_pdu - .scan_prefix(&prefix) - .values() - .map(|value| { - Ok::<_, Error>( - serde_json::from_slice::(&value?) + for pdu in + self.roomstateid_pduid + .scan_prefix(&prefix) + .values() + .map(|value| { + Ok::<_, Error>( + serde_json::from_slice::( + &self.pduid_pdu.get(value?)?.ok_or_else(|| { + Error::bad_database("PDU not found for ID in db.") + })?, + ) .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - ) - }) + ) + }) { let pdu = pdu?; let state_key = pdu.state_key.clone().ok_or_else(|| { @@ -115,23 +389,28 @@ impl Rooms { Ok(hashmap) } - /// Returns the full room state. + /// Returns a single PDU in `room_id` with key (`event_type`, `state_key`). pub fn room_state_get( &self, room_id: &RoomId, event_type: &EventType, state_key: &str, ) -> Result> { - let mut key = room_id.to_string().as_bytes().to_vec(); + let mut key = room_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(&event_type.to_string().as_bytes()); key.push(0xff); key.extend_from_slice(&state_key.as_bytes()); - self.roomstateid_pdu.get(&key)?.map_or(Ok(None), |value| { + self.roomstateid_pduid.get(&key)?.map_or(Ok(None), |value| { Ok::<_, Error>(Some( - serde_json::from_slice::(&value) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + serde_json::from_slice::( + &self + .pduid_pdu + .get(value)? + .ok_or_else(|| Error::bad_database("PDU not found for ID in db."))?, + ) + .map_err(|_| Error::bad_database("Invalid PDU in db."))?, )) }) } @@ -139,7 +418,7 @@ impl Rooms { /// Returns the `count` of this pdu's id. pub fn get_pdu_count(&self, event_id: &EventId) -> Result> { self.eventid_pduid - .get(event_id.to_string().as_bytes())? + .get(event_id.as_bytes())? .map_or(Ok(None), |pdu_id| { Ok(Some( utils::u64_from_bytes( @@ -153,7 +432,7 @@ impl Rooms { /// Returns the json of a pdu. pub fn get_pdu_json(&self, event_id: &EventId) -> Result> { self.eventid_pduid - .get(event_id.to_string().as_bytes())? + .get(event_id.as_bytes())? .map_or(Ok(None), |pdu_id| { Ok(Some( serde_json::from_slice(&self.pduid_pdu.get(pdu_id)?.ok_or_else(|| { @@ -174,7 +453,7 @@ impl Rooms { /// Returns the pdu. pub fn get_pdu(&self, event_id: &EventId) -> Result> { self.eventid_pduid - .get(event_id.to_string().as_bytes())? + .get(event_id.as_bytes())? .map_or(Ok(None), |pdu_id| { Ok(Some( serde_json::from_slice(&self.pduid_pdu.get(pdu_id)?.ok_or_else(|| { @@ -238,16 +517,15 @@ impl Rooms { /// Replace the leaves of a room with a new event. pub fn replace_pdu_leaves(&self, room_id: &RoomId, event_id: &EventId) -> Result<()> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); for key in self.roomid_pduleaves.scan_prefix(&prefix).keys() { self.roomid_pduleaves.remove(key?)?; } - prefix.extend_from_slice(event_id.to_string().as_bytes()); - self.roomid_pduleaves - .insert(&prefix, &*event_id.to_string())?; + prefix.extend_from_slice(event_id.as_bytes()); + self.roomid_pduleaves.insert(&prefix, event_id.as_bytes())?; Ok(()) } @@ -272,6 +550,14 @@ impl Rooms { // TODO: Make sure this isn't called twice in parallel let prev_events = self.get_pdu_leaves(&room_id)?; + let auth_events = self.get_auth_events( + &room_id, + &event_type, + &sender, + state_key.as_deref(), + content.clone(), + )?; + // Is the event authorized? if let Some(state_key) = &state_key { let power_levels = self @@ -333,138 +619,24 @@ impl Rooms { // Don't allow encryption events when it's disabled !globals.encryption_disabled() } - EventType::RoomMember => { - let target_user_id = UserId::try_from(&**state_key).map_err(|_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "State key of member event does not contain user id.", - ) - })?; - - let current_membership = self - .room_state_get( - &room_id, - &EventType::RoomMember, - &target_user_id.to_string(), - )? - .map_or(Ok::<_, Error>(member::MembershipState::Leave), |pdu| { - Ok(serde_json::from_value::>( - pdu.content, - ) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| Error::bad_database("Invalid Member event in db."))? - .membership) - })?; - - let target_membership = - serde_json::from_value::>(content.clone()) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| Error::bad_database("Invalid Member event in db."))? - .membership; - - let target_power = power_levels.users.get(&target_user_id).map_or_else( - || { - if target_membership != member::MembershipState::Join { - None - } else { - Some(&power_levels.users_default) - } - }, - // If it's okay, wrap with Some(_) - Some, - ); - - let join_rules = - self.room_state_get(&room_id, &EventType::RoomJoinRules, "")? - .map_or(Ok::<_, Error>(join_rules::JoinRule::Public), |pdu| { - Ok(serde_json::from_value::< - Raw, - >(pdu.content) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| { - Error::bad_database("Database contains invalid JoinRules event") - })? - .join_rule) - })?; - - if target_membership == member::MembershipState::Join { - let mut prev_events = prev_events.iter(); - let prev_event = self - .get_pdu(prev_events.next().ok_or(Error::BadRequest( - ErrorKind::Unknown, - "Membership can't be the first event", - ))?)? - .ok_or_else(|| { - Error::bad_database("PDU leaf points to invalid event!") - })?; - if prev_event.kind == EventType::RoomCreate - && prev_event.prev_events.is_empty() - { - true - } else if sender != target_user_id { - false - } else if let member::MembershipState::Ban = current_membership { - false - } else { - join_rules == join_rules::JoinRule::Invite - && (current_membership == member::MembershipState::Join - || current_membership == member::MembershipState::Invite) - || join_rules == join_rules::JoinRule::Public - } - } else if target_membership == member::MembershipState::Invite { - if let Some(third_party_invite_json) = content.get("third_party_invite") { - if current_membership == member::MembershipState::Ban { - false - } else { - let _third_party_invite = - serde_json::from_value::( - third_party_invite_json.clone(), - ) - .map_err(|_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "ThirdPartyInvite is invalid", - ) - })?; - todo!("handle third party invites"); - } - } else if sender_membership != member::MembershipState::Join - || current_membership == member::MembershipState::Join - || current_membership == member::MembershipState::Ban - { - false - } else { - sender_power - .filter(|&p| p >= &power_levels.invite) - .is_some() - } - } else if target_membership == member::MembershipState::Leave { - if sender == target_user_id { - current_membership == member::MembershipState::Join - || current_membership == member::MembershipState::Invite - } else if sender_membership != member::MembershipState::Join - || current_membership == member::MembershipState::Ban - && sender_power.filter(|&p| p < &power_levels.ban).is_some() - { - false - } else { - sender_power.filter(|&p| p >= &power_levels.kick).is_some() - && target_power < sender_power - } - } else if target_membership == member::MembershipState::Ban { - if sender_membership != member::MembershipState::Join { - false - } else { - sender_power.filter(|&p| p >= &power_levels.ban).is_some() - && target_power < sender_power - } - } else { - false - } - } + EventType::RoomMember => event_auth::is_membership_change_allowed( + // TODO this is a bit of a hack but not sure how to have a type + // declared in `state_res` crate be + Requester { + prev_event_ids: prev_events.to_owned(), + room_id: &room_id, + content: &content, + state_key: Some(state_key.to_owned()), + sender: &sender, + }, + &auth_events + .iter() + .map(|((ty, key), pdu)| { + Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) + }) + .collect::>>()?, + ) + .ok_or(Error::Conflict("Found incoming PDU with invalid data."))?, EventType::RoomCreate => prev_events.is_empty(), // Not allow any of the following events if the sender is not joined. _ if sender_membership != member::MembershipState::Join => false, @@ -474,7 +646,7 @@ impl Rooms { >= &power_levels.state_default } } { - error!("Unauthorized"); + error!("Unauthorized {}", event_type); // Not authorized return Err(Error::BadRequest( ErrorKind::Forbidden, @@ -483,7 +655,7 @@ impl Rooms { } } else if !self.is_joined(&sender, &room_id)? { // TODO: auth rules apply to all events, not only those with a state key - error!("Unauthorized"); + error!("Unauthorized {}", event_type); return Err(Error::BadRequest( ErrorKind::Forbidden, "Event is not authorized", @@ -524,7 +696,10 @@ impl Rooms { depth: depth .try_into() .map_err(|_| Error::bad_database("Depth is invalid"))?, - auth_events: Vec::new(), + auth_events: auth_events + .into_iter() + .map(|(_, pdu)| pdu.event_id) + .collect(), redacts: redacts.clone(), unsigned, hashes: ruma::events::pdu::EventHash { @@ -564,15 +739,19 @@ impl Rooms { self.pduid_pdu.insert(&pdu_id, &*pdu_json.to_string())?; self.eventid_pduid - .insert(pdu.event_id.to_string(), pdu_id.clone())?; + .insert(pdu.event_id.to_string(), &*pdu_id)?; + + if let Some(state_key) = &pdu.state_key { + // We call this first because our StateHash relies on the + // state before the new event + self.append_state_pdu(&room_id, &pdu_id)?; - if let Some(state_key) = pdu.state_key { - let mut key = room_id.to_string().as_bytes().to_vec(); + let mut key = room_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(pdu.kind.to_string().as_bytes()); key.push(0xff); key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pdu.insert(key, &*pdu_json.to_string())?; + self.roomstateid_pduid.insert(key, pdu_id.as_slice())?; } match event_type { diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index cece8dbe..e318f436 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -2,7 +2,7 @@ use crate::{Error, Result}; use ruma::{ api::client::{ error::ErrorKind, - r0::uiaa::{AuthData, UiaaInfo}, + r0::uiaa::{IncomingAuthData, UiaaInfo}, }, DeviceId, UserId, }; @@ -26,12 +26,12 @@ impl Uiaa { &self, user_id: &UserId, device_id: &DeviceId, - auth: &AuthData, + auth: &IncomingAuthData, uiaainfo: &UiaaInfo, users: &super::users::Users, globals: &super::globals::Globals, ) -> Result<(bool, UiaaInfo)> { - if let AuthData::DirectRequest { + if let IncomingAuthData::DirectRequest { kind, session, auth_parameters, diff --git a/src/pdu.rs b/src/pdu.rs index 9936802e..5485f236 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -177,6 +177,35 @@ impl PduEvent { } } +impl PduEvent { + pub fn convert_for_state_res(&self) -> Result { + serde_json::from_value(json!({ + "event_id": self.event_id, + "room_id": self.room_id, + "sender": self.sender, + "origin": self.origin, + "origin_server_ts": self.origin_server_ts, + "type": self.kind, + "content": self.content, + "state_key": self.state_key, + "prev_events": self.prev_events + .iter() + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "depth": self.depth, + "auth_events": self.auth_events + .iter() + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "redacts": self.redacts, + "unsigned": self.unsigned, + "hashes": self.hashes, + "signatures": self.signatures, + })) + .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) + } +} + /// Build the start of a PDU in order to add it to the `Database`. #[derive(Debug)] pub struct PduBuilder { diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 8d862043..80e6e582 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -1,5 +1,8 @@ use crate::Error; -use ruma::identifiers::{DeviceId, UserId}; +use ruma::{ + api::IncomingRequest, + identifiers::{DeviceId, UserId}, +}; use std::{convert::TryInto, ops::Deref}; #[cfg(feature = "conduit_bin")] @@ -16,13 +19,12 @@ use { tokio::io::AsyncReadExt, Request, State, }, - ruma::api::IncomingRequest, std::io::Cursor, }; /// This struct converts rocket requests into ruma structs by converting them into http requests /// first. -pub struct Ruma { +pub struct Ruma { pub body: T, pub sender_id: Option, pub device_id: Option>, @@ -110,7 +112,7 @@ impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { } } -impl Deref for Ruma { +impl Deref for Ruma { type Target = T; fn deref(&self) -> &Self::Target { diff --git a/src/server_server.rs b/src/server_server.rs index f48f502b..e47b50a5 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,14 +1,17 @@ use crate::{client_server, ConduitResult, Database, Error, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; -use ruma::api::federation::{ - directory::get_public_rooms, - discovery::{ - get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, +use ruma::api::{ + client, + federation::{ + directory::get_public_rooms, + discovery::{ + get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, + }, + transactions::send_transaction_message, }, - transactions::send_transaction_message, + OutgoingRequest, }; -use ruma::api::{client, OutgoingRequest}; use serde_json::json; use std::{ collections::BTreeMap, @@ -204,11 +207,11 @@ pub fn get_server_keys_deprecated(db: State<'_, Database>) -> Json { )] pub async fn get_public_rooms_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let Ruma { body: - get_public_rooms::v1::Request { + get_public_rooms::v1::IncomingRequest { room_network: _room_network, // TODO limit, since, @@ -229,7 +232,7 @@ pub async fn get_public_rooms_route( body: client::r0::directory::get_public_rooms_filtered::IncomingRequest { filter: None, limit, - room_network: client::r0::directory::get_public_rooms_filtered::RoomNetwork::Matrix, + room_network: ruma::directory::RoomNetwork::Matrix, server: None, since, }, @@ -268,9 +271,9 @@ pub async fn get_public_rooms_route( feature = "conduit_bin", put("/_matrix/federation/v1/send/<_>", data = "") )] -pub fn send_transaction_message_route( - db: State<'_, Database>, - body: Ruma, +pub fn send_transaction_message_route<'a>( + _db: State<'a, Database>, + body: Ruma, ) -> ConduitResult { dbg!(&*body); Ok(send_transaction_message::v1::Response { diff --git a/src/utils.rs b/src/utils.rs index 8cf1b2ce..77a7d1fa 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,3 +1,4 @@ +use crate::Error; use argon2::{Config, Variant}; use cmp::Ordering; use rand::prelude::*; @@ -90,3 +91,8 @@ pub fn common_elements( .all(|b| b) })) } + +pub fn deserialize<'de, T: serde::Deserialize<'de>>(val: &'de sled::IVec) -> Result { + serde_json::from_slice::(val.as_ref()) + .map_err(|_| Error::bad_database("PDU in db is invalid.")) +} From cb68bf9e0c073aa060b3fe6d9aa14bbfaebc5dae Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Tue, 18 Aug 2020 14:32:38 -0400 Subject: [PATCH 02/51] Use ring crate to generate StatHashes when saving stateid/statehash --- Cargo.lock | 1 + Cargo.toml | 2 +- src/database/rooms.rs | 20 ++++++++++++-------- src/utils.rs | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ffee8ea5..faa9e898 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -295,6 +295,7 @@ dependencies = [ "log", "rand", "reqwest", + "ring", "rocket", "ruma", "rust-argon2 0.8.2", diff --git a/Cargo.toml b/Cargo.toml index 4c14d712..78d8f760 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ image = { version = "0.23.4", default-features = false, features = ["jpeg", "png base64 = "0.12.3" # Used to encode server public key # state-res = { path = "../../state-res" } state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0" } - +ring = "0.16.15" [features] default = ["conduit_bin"] diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 0d363262..6366c8cd 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -4,6 +4,7 @@ pub use edus::RoomEdus; use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; use log::error; +use ring::digest; use ruma::{ api::client::error::ErrorKind, events::{ @@ -21,9 +22,8 @@ use sled::IVec; use state_res::{event_auth, Requester, StateEvent, StateMap, StateStore}; use std::{ - collections::{hash_map::DefaultHasher, BTreeMap, HashMap}, + collections::{BTreeMap, HashMap}, convert::{TryFrom, TryInto}, - hash::{Hash, Hasher}, mem, result::Result as StdResult, }; @@ -285,8 +285,10 @@ impl Rooms { .next() .is_none() { - // TODO use ring crate to hash - return Ok(room_id.as_str().to_owned()); + return utils::string_from_bytes( + digest::digest(&digest::SHA256, room_id.as_bytes()).as_ref(), + ) + .map_err(|_| Error::bad_database("Empty state generated invalid string from hash.")); } let pdu_ids_to_hash = self @@ -304,11 +306,13 @@ impl Rooms { .collect::>>>() })??; - let mut hasher = DefaultHasher::new(); - pdu_ids_to_hash.hash(&mut hasher); - let hash = hasher.finish().to_string(); + let hash = digest::digest( + &digest::SHA256, + &pdu_ids_to_hash.into_iter().flatten().collect::>(), + ); // TODO not sure how you want to hash this - Ok(hash) + utils::string_from_bytes(hash.as_ref()) + .map_err(|_| Error::bad_database("State generated invalid string from hash.")) } /// Checks if a room exists. diff --git a/src/utils.rs b/src/utils.rs index 77a7d1fa..b549153a 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -94,5 +94,5 @@ pub fn common_elements( pub fn deserialize<'de, T: serde::Deserialize<'de>>(val: &'de sled::IVec) -> Result { serde_json::from_slice::(val.as_ref()) - .map_err(|_| Error::bad_database("PDU in db is invalid.")) + .map_err(|_| Error::bad_database("Found invalid bytes as PDU in db.")) } From 846a0098c182272a5669bacf0d27fa988eaa4c23 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Tue, 18 Aug 2020 16:26:03 -0400 Subject: [PATCH 03/51] Split append_pdu -> append_pdu and build_and_append Move all state event appending to append_state_pdu. --- Cargo.lock | 22 ++-- src/client_server/account.rs | 2 +- src/client_server/membership.rs | 37 ++---- src/client_server/message.rs | 2 +- src/client_server/profile.rs | 4 +- src/client_server/redact.rs | 2 +- src/client_server/room.rs | 20 +-- src/client_server/state.rs | 2 +- src/database/rooms.rs | 217 ++++++++++++++++---------------- src/pdu.rs | 28 ++++- 10 files changed, 175 insertions(+), 161 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index faa9e898..3c5d836b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,9 +116,9 @@ checksum = "4af5687fe33aec5e70ef14caac5e0d363e335e5e5d6385fb75978d0c241b1d67" [[package]] name = "async-trait" -version = "0.1.37" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caae68055714ff28740f310927e04f2eba76ff580b16fb18ed90073ee71646f7" +checksum = "6e1a4a2f97ce50c9d0282c1468816208588441492b40d813b2e0419c22c05e7f" dependencies = [ "proc-macro2", "quote", @@ -1194,9 +1194,9 @@ checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" [[package]] name = "once_cell" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" +checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" [[package]] name = "opaque-debug" @@ -1860,9 +1860,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac94b333ee2aac3284c5b8a1b7fb4dd11cba88c244e3fe33cdbd047af0eb693" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" dependencies = [ "base64 0.12.3", "log", @@ -2088,7 +2088,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res#789c8140890e076d38b23fa1147c4ff0500c0d38" +source = "git+https://github.com/ruma/state-res#4e9b428c0db50ac3a3421ced12a6fd202a1c36a3" dependencies = [ "itertools", "js_int", @@ -2281,9 +2281,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed" +checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" [[package]] name = "tokio" @@ -2384,9 +2384,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe233f4227389ab7df5b32649239da7ebe0b281824b4e84b342d04d3fd8c25e" +checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada" dependencies = [ "proc-macro2", "quote", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 9fa1a9c7..9e52f6d2 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -354,7 +354,7 @@ pub fn deactivate_route( third_party_invite: None, }; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index c04cf7f6..824e8718 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -1,6 +1,8 @@ use super::State; use crate::{ - client_server, pdu::PduBuilder, server_server, utils, ConduitResult, Database, Error, Ruma, + client_server, + pdu::{PduBuilder, PduEvent}, + server_server, utils, ConduitResult, Database, Error, Ruma, }; use ruma::{ api::{ @@ -142,24 +144,9 @@ pub async fn join_room_by_id_route( Error::Conflict("Found event_id in sorted events that is not in resolved state") })?; - db.rooms.append_pdu( - PduBuilder { - room_id: pdu.room_id().unwrap_or(&body.room_id).clone(), - sender: pdu.sender().clone(), - event_type: pdu.kind(), - content: pdu.content().clone(), - unsigned: Some( - pdu.unsigned() - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), - ), - state_key: pdu.state_key(), - redacts: pdu.redacts().cloned(), - }, - &db.globals, - &db.account_data, - )?; + // We do not rebuild the PDU in this case only insert to DB + db.rooms + .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; } } @@ -171,7 +158,7 @@ pub async fn join_room_by_id_route( third_party_invite: None, }; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -284,7 +271,7 @@ pub fn leave_room_route( event.membership = member::MembershipState::Leave; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -312,7 +299,7 @@ pub fn invite_user_route( let sender_id = body.sender_id.as_ref().expect("user is authenticated"); if let invite_user::InvitationRecipient::UserId { user_id } = &body.recipient { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -369,7 +356,7 @@ pub fn kick_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; // TODO: reason - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -424,7 +411,7 @@ pub fn ban_user_route( }, )?; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -470,7 +457,7 @@ pub fn unban_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 1b461d24..03832d86 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -25,7 +25,7 @@ pub fn send_message_event_route( let mut unsigned = serde_json::Map::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into()); - let event_id = db.rooms.append_pdu( + let event_id = db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 1313db77..0707b342 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -31,7 +31,7 @@ pub fn set_displayname_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -134,7 +134,7 @@ pub fn set_avatar_url_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index fc65c23f..8708692f 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -18,7 +18,7 @@ pub fn redact_event_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - let event_id = db.rooms.append_pdu( + let event_id = db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 589a2dcd..3ee21b6d 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -56,7 +56,7 @@ pub fn create_room_route( content.room_version = RoomVersionId::Version6; // 1. The room create event - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -71,7 +71,7 @@ pub fn create_room_route( )?; // 2. Let the room creator join - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -120,7 +120,7 @@ pub fn create_room_route( }) .expect("event is valid, we just created it") }; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -144,7 +144,7 @@ pub fn create_room_route( }); // 4.1 Join Rules - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -169,7 +169,7 @@ pub fn create_room_route( )?; // 4.2 History Visibility - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -187,7 +187,7 @@ pub fn create_room_route( )?; // 4.3 Guest Access - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -224,7 +224,7 @@ pub fn create_room_route( continue; } - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -243,7 +243,7 @@ pub fn create_room_route( // 6. Events implied by name and topic if let Some(name) = &body.name { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -264,7 +264,7 @@ pub fn create_room_route( } if let Some(topic) = &body.topic { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -284,7 +284,7 @@ pub fn create_room_route( // 7. Events implied by invite (and TODO: invite_3pid) for user in &body.invite { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 14cc4972..2920de2f 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -57,7 +57,7 @@ pub fn send_state_event_for_key_route( } } - let event_id = db.rooms.append_pdu( + let event_id = db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 6366c8cd..0339b7f9 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -4,6 +4,7 @@ pub use edus::RoomEdus; use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; use log::error; +// TODO if ruma-signatures re-exports `use ruma::signatures::digest;` use ring::digest; use ruma::{ api::client::error::ErrorKind, @@ -99,7 +100,13 @@ impl Rooms { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. /// The incoming event is the `pdu_id` passed to this method. - pub fn append_state_pdu(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result { + pub fn append_state_pdu( + &self, + room_id: &RoomId, + pdu_id: &[u8], + state_key: &str, + kind: &EventType, + ) -> Result { let state_hash = self.new_state_hash_id(room_id)?; let state = self.current_state_pduids(room_id)?; @@ -123,6 +130,13 @@ impl Rooms { // will be everything up to but not including the incoming event. self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + let mut key = room_id.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(kind.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(state_key.as_bytes()); + self.roomstateid_pduid.insert(key, pdu_id)?; + Ok(state_hash) } @@ -535,8 +549,92 @@ impl Rooms { } /// Creates a new persisted data unit and adds it to a room. - #[allow(clippy::blocks_in_if_conditions)] pub fn append_pdu( + &self, + pdu: PduEvent, + globals: &super::globals::Globals, + account_data: &super::account_data::AccountData, + ) -> Result { + let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); + ruma::signatures::hash_and_sign_event( + globals.server_name().as_str(), + globals.keypair(), + &mut pdu_json, + ) + .expect("event is valid, we just created it"); + + self.replace_pdu_leaves(&pdu.room_id, &pdu.event_id)?; + + // Increment the last index and use that + // This is also the next_batch/since value + let index = globals.next_count()?; + + let mut pdu_id = pdu.room_id.as_bytes().to_vec(); + pdu_id.push(0xff); + pdu_id.extend_from_slice(&index.to_be_bytes()); + + self.pduid_pdu.insert(&pdu_id, &*pdu_json.to_string())?; + + self.eventid_pduid + .insert(pdu.event_id.as_bytes(), &*pdu_id)?; + + if let Some(state_key) = &pdu.state_key { + self.append_state_pdu(&pdu.room_id, &pdu_id, state_key, &pdu.kind)?; + } + + match pdu.kind { + EventType::RoomRedaction => { + if let Some(redact_id) = &pdu.redacts { + // TODO: Reason + let _reason = serde_json::from_value::>( + pdu.content, + ) + .expect("Raw::from_value always works.") + .deserialize() + .map_err(|_| { + Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid redaction event content.", + ) + })? + .reason; + + self.redact_pdu(&redact_id)?; + } + } + EventType::RoomMember => { + if let Some(state_key) = &pdu.state_key { + // if the state_key fails + let target_user_id = UserId::try_from(state_key.as_str()) + .expect("This state_key was previously validated"); + // Update our membership info, we do this here incase a user is invited + // and immediately leaves we need the DB to record the invite event for auth + self.update_membership( + &pdu.room_id, + &target_user_id, + serde_json::from_value::(pdu.content).map_err( + |_| { + Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid redaction event content.", + ) + }, + )?, + &pdu.sender, + account_data, + globals, + )?; + } + } + _ => {} + } + self.edus.room_read_set(&pdu.room_id, &pdu.sender, index)?; + + Ok(pdu.event_id) + } + + /// Creates a new persisted data unit and adds it to a room. + pub fn build_and_append_pdu( &self, pdu_builder: PduBuilder, globals: &super::globals::Globals, @@ -618,6 +716,7 @@ impl Rooms { ); // Is the event allowed? + #[allow(clippy::blocks_in_if_conditions)] if !match event_type { EventType::RoomEncryption => { // Don't allow encryption events when it's disabled @@ -687,15 +786,15 @@ impl Rooms { let mut pdu = PduEvent { event_id: EventId::try_from("$thiswillbefilledinlater").expect("we know this is valid"), - room_id: room_id.clone(), - sender: sender.clone(), + room_id, + sender, origin: globals.server_name().to_owned(), origin_server_ts: utils::millis_since_unix_epoch() .try_into() .expect("time is valid"), - kind: event_type.clone(), - content: content.clone(), - state_key: state_key.clone(), + kind: event_type, + content, + state_key, prev_events, depth: depth .try_into() @@ -704,7 +803,7 @@ impl Rooms { .into_iter() .map(|(_, pdu)| pdu.event_id) .collect(), - redacts: redacts.clone(), + redacts, unsigned, hashes: ruma::events::pdu::EventHash { sha256: "aaa".to_owned(), @@ -722,105 +821,7 @@ impl Rooms { )) .expect("ruma's reference hashes are valid event ids"); - let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); - ruma::signatures::hash_and_sign_event( - globals.server_name().as_str(), - globals.keypair(), - &mut pdu_json, - ) - .expect("event is valid, we just created it"); - - self.replace_pdu_leaves(&room_id, &pdu.event_id)?; - - // Increment the last index and use that - // This is also the next_batch/since value - let index = globals.next_count()?; - - let mut pdu_id = room_id.to_string().as_bytes().to_vec(); - pdu_id.push(0xff); - pdu_id.extend_from_slice(&index.to_be_bytes()); - - self.pduid_pdu.insert(&pdu_id, &*pdu_json.to_string())?; - - self.eventid_pduid - .insert(pdu.event_id.to_string(), &*pdu_id)?; - - if let Some(state_key) = &pdu.state_key { - // We call this first because our StateHash relies on the - // state before the new event - self.append_state_pdu(&room_id, &pdu_id)?; - - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(pdu.kind.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pduid.insert(key, pdu_id.as_slice())?; - } - - match event_type { - EventType::RoomRedaction => { - if let Some(redact_id) = &redacts { - // TODO: Reason - let _reason = - serde_json::from_value::>(content) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid redaction event content.", - ) - })? - .reason; - - self.redact_pdu(&redact_id)?; - } - } - EventType::RoomMember => { - if let Some(state_key) = state_key { - // if the state_key fails - let target_user_id = UserId::try_from(state_key) - .expect("This state_key was previously validated"); - // Update our membership info, we do this here incase a user is invited - // and immediately leaves we need the DB to record the invite event for auth - self.update_membership( - &room_id, - &target_user_id, - serde_json::from_value::(content).map_err( - |_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid redaction event content.", - ) - }, - )?, - &sender, - account_data, - globals, - )?; - } - } - EventType::RoomMessage => { - if let Some(body) = content.get("body").and_then(|b| b.as_str()) { - for word in body - .split_terminator(|c: char| !c.is_alphanumeric()) - .map(str::to_lowercase) - { - let mut key = room_id.to_string().as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(word.as_bytes()); - key.push(0xff); - key.extend_from_slice(&pdu_id); - self.tokenids.insert(key, &[])?; - } - } - } - _ => {} - } - self.edus.room_read_set(&room_id, &sender, index)?; - - Ok(pdu.event_id) + self.append_pdu(pdu, globals, account_data) } /// Returns an iterator over all PDUs in a room. @@ -999,7 +1000,7 @@ impl Rooms { if is_ignored { member_content.membership = member::MembershipState::Leave; - self.append_pdu( + self.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: user_id.clone(), diff --git a/src/pdu.rs b/src/pdu.rs index 5485f236..eec8e496 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -9,7 +9,7 @@ use ruma::{ }; use serde::{Deserialize, Serialize}; use serde_json::json; -use std::collections::HashMap; +use std::{collections::HashMap, convert::TryFrom}; #[derive(Deserialize, Serialize)] pub struct PduEvent { @@ -177,6 +177,30 @@ impl PduEvent { } } +impl TryFrom<&state_res::StateEvent> for PduEvent { + type Error = Error; + fn try_from(pdu: &state_res::StateEvent) -> Result { + serde_json::from_value(json!({ + "event_id": pdu.event_id(), + "room_id": pdu.room_id(), + "sender": pdu.sender(), + "origin": pdu.origin(), + "origin_server_ts": pdu.origin_server_ts(), + "event_type": pdu.kind(), + "content": pdu.content(), + "state_key": pdu.state_key(), + "prev_events": pdu.prev_event_ids(), + "depth": pdu.depth(), + "auth_events": pdu.auth_events(), + "redacts": pdu.redacts(), + "unsigned": pdu.unsigned(), + "hashes": pdu.hashes(), + "signatures": pdu.signatures(), + })) + .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) + } +} + impl PduEvent { pub fn convert_for_state_res(&self) -> Result { serde_json::from_value(json!({ @@ -190,11 +214,13 @@ impl PduEvent { "state_key": self.state_key, "prev_events": self.prev_events .iter() + // TODO How do we create one of these .map(|id| (id, EventHash { sha256: "hello".into() })) .collect::>(), "depth": self.depth, "auth_events": self.auth_events .iter() + // TODO How do we create one of these .map(|id| (id, EventHash { sha256: "hello".into() })) .collect::>(), "redacts": self.redacts, From d73c6aa8addcaac5f13cbaa009960659a4365bb2 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 19 Aug 2020 17:27:24 -0400 Subject: [PATCH 04/51] Add roomid_statehash tree, clean up review issues --- src/client_server/membership.rs | 10 +--- src/database.rs | 1 + src/database/rooms.rs | 95 +++++++++++++++++---------------- 3 files changed, 52 insertions(+), 54 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 824e8718..555291e5 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -112,11 +112,7 @@ pub async fn join_room_by_id_route( .room_state .state .iter() - .map(|pdu| pdu.deserialize().map(StateEvent::Full)) - .map(|ev| { - let ev = ev?; - Ok::<_, serde_json::Error>((ev.event_id(), ev)) - }) + .map(|pdu| pdu.deserialize().map(StateEvent::Full).map(|ev| (ev.event_id(), ev))) .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; @@ -140,9 +136,7 @@ pub async fn join_room_by_id_route( for ev_id in &sorted_events_ids { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` - let pdu = event_map.get(ev_id).ok_or_else(|| { - Error::Conflict("Found event_id in sorted events that is not in resolved state") - })?; + let pdu = event_map.get(ev_id).expect("Found event_id in sorted events that is not in resolved state"); // We do not rebuild the PDU in this case only insert to DB db.rooms diff --git a/src/database.rs b/src/database.rs index 6cd65c31..a105058c 100644 --- a/src/database.rs +++ b/src/database.rs @@ -114,6 +114,7 @@ impl Database { stateid_pduid: db.open_tree("stateid_pduid")?, pduid_statehash: db.open_tree("pduid_statehash")?, + roomid_statehash: db.open_tree("roomid_statehash")?, }, account_data: account_data::AccountData { roomuserdataid_accountdata: db.open_tree("roomuserdataid_accountdata")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 0339b7f9..62730054 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -63,6 +63,8 @@ pub struct Rooms { pub(super) pduid_statehash: sled::Tree, // PDU id -> StateHash /// Also holds the full room state minus the latest event. pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + (EventType, StateKey) + /// The room_id -> the latest StateHash + pub(super) roomid_statehash: sled::Tree, } impl StateStore for Rooms { @@ -93,53 +95,7 @@ impl StateStore for Rooms { } } -// These are the methods related to STATE resolution. impl Rooms { - /// Generates a new StateHash and associates it with the incoming event. - /// - /// This adds all current state events (not including the incoming event) - /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. - /// The incoming event is the `pdu_id` passed to this method. - pub fn append_state_pdu( - &self, - room_id: &RoomId, - pdu_id: &[u8], - state_key: &str, - kind: &EventType, - ) -> Result { - let state_hash = self.new_state_hash_id(room_id)?; - let state = self.current_state_pduids(room_id)?; - - let mut key = state_hash.as_bytes().to_vec(); - key.push(0xff); - - // TODO eventually we could avoid writing to the DB so much on every event - // by keeping track of the delta and write that every so often - for ((ev_ty, state_key), pid) in state { - let mut state_id = key.to_vec(); - state_id.extend_from_slice(ev_ty.to_string().as_bytes()); - key.push(0xff); - state_id.extend_from_slice(state_key.expect("state event").as_bytes()); - key.push(0xff); - - self.stateid_pduid.insert(&state_id, &pid)?; - } - - // This event's state does not include the event itself. `current_state_pduids` - // uses `roomstateid_pduid` before the current event is inserted to the tree so the state - // will be everything up to but not including the incoming event. - self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; - - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(kind.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pduid.insert(key, pdu_id)?; - - Ok(state_hash) - } - /// Builds a `StateMap` by iterating over all keys that start /// with `state_hash`, this gives the full state at event "x". pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { @@ -633,6 +589,53 @@ impl Rooms { Ok(pdu.event_id) } + /// Generates a new StateHash and associates it with the incoming event. + /// + /// This adds all current state events (not including the incoming event) + /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. + /// The incoming event is the `pdu_id` passed to this method. + fn append_state_pdu( + &self, + room_id: &RoomId, + pdu_id: &[u8], + state_key: &str, + kind: &EventType, + ) -> Result { + let state_hash = self.new_state_hash_id(room_id)?; + let state = self.current_state_pduids(room_id)?; + + let mut key = state_hash.as_bytes().to_vec(); + key.push(0xff); + + // TODO eventually we could avoid writing to the DB so much on every event + // by keeping track of the delta and write that every so often + for ((ev_ty, state_key), pid) in state { + let mut state_id = key.to_vec(); + state_id.extend_from_slice(ev_ty.to_string().as_bytes()); + key.push(0xff); + state_id.extend_from_slice(state_key.expect("state event").as_bytes()); + key.push(0xff); + + self.stateid_pduid.insert(&state_id, &pid)?; + } + + // This event's state does not include the event itself. `current_state_pduids` + // uses `roomstateid_pduid` before the current event is inserted to the tree so the state + // will be everything up to but not including the incoming event. + self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + + self.roomid_statehash.insert(room_id.as_bytes(), state_hash.as_bytes())?; + + let mut key = room_id.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(kind.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(state_key.as_bytes()); + self.roomstateid_pduid.insert(key, pdu_id)?; + + Ok(state_hash) + } + /// Creates a new persisted data unit and adds it to a room. pub fn build_and_append_pdu( &self, From 64fb0374b60461ad1b7600e92032dfc1e30b39d3 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 19 Aug 2020 19:29:39 -0400 Subject: [PATCH 05/51] Use Vec instead of string for digest bytes and add roomid_statehash --- src/database/rooms.rs | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 62730054..c8ff198b 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -33,7 +33,7 @@ use std::{ /// /// This is created when a state group is added to the database by /// hashing the entire state. -pub type StateHashId = String; +pub type StateHashId = Vec; /// This identifier consists of roomId + count. It represents a /// unique event, it will never be overwritten or removed. @@ -100,7 +100,7 @@ impl Rooms { /// with `state_hash`, this gives the full state at event "x". pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { self.stateid_pduid - .scan_prefix(state_hash.as_bytes()) + .scan_prefix(&state_hash) .values() .map(|pduid| { self.pduid_pdu.get(&pduid?)?.map_or_else( @@ -123,12 +123,12 @@ impl Rooms { pub fn prev_state_hash(&self, current: StateHashId) -> Option { let mut found = false; for pair in self.pduid_statehash.iter().rev() { - let prev = utils::string_from_bytes(&pair.ok()?.1).ok()?; - if current == prev { + let prev = pair.ok()?.1; + if current == prev.as_ref() { found = true; } - if current != prev && found { - return Some(prev); + if current != prev.as_ref() && found { + return Some(prev.to_vec()); } } None @@ -172,17 +172,14 @@ impl Rooms { // We must check here because this method is called outside and before // `append_state_pdu` so the DB can be empty if self.pduid_statehash.scan_prefix(prefix).next().is_none() { - // TODO use ring crate to hash - return Ok(room_id.as_str().to_owned()); + // return the hash of the room_id, this represents a room with no state + return self.new_state_hash_id(room_id); } self.pduid_statehash .iter() .next_back() - .map(|pair| { - utils::string_from_bytes(&pair?.1) - .map_err(|_| Error::bad_database("Invalid state hash string in db.")) - }) + .map(|pair| Ok(pair?.1.to_vec())) .ok_or_else(|| Error::bad_database("No PDU's found for this room."))? } @@ -255,10 +252,9 @@ impl Rooms { .next() .is_none() { - return utils::string_from_bytes( - digest::digest(&digest::SHA256, room_id.as_bytes()).as_ref(), - ) - .map_err(|_| Error::bad_database("Empty state generated invalid string from hash.")); + return Ok(digest::digest(&digest::SHA256, room_id.as_bytes()) + .as_ref() + .to_vec()); } let pdu_ids_to_hash = self @@ -280,9 +276,7 @@ impl Rooms { &digest::SHA256, &pdu_ids_to_hash.into_iter().flatten().collect::>(), ); - // TODO not sure how you want to hash this - utils::string_from_bytes(hash.as_ref()) - .map_err(|_| Error::bad_database("State generated invalid string from hash.")) + Ok(hash.as_ref().to_vec()) } /// Checks if a room exists. @@ -604,7 +598,7 @@ impl Rooms { let state_hash = self.new_state_hash_id(room_id)?; let state = self.current_state_pduids(room_id)?; - let mut key = state_hash.as_bytes().to_vec(); + let mut key = state_hash.to_vec(); key.push(0xff); // TODO eventually we could avoid writing to the DB so much on every event @@ -622,9 +616,10 @@ impl Rooms { // This event's state does not include the event itself. `current_state_pduids` // uses `roomstateid_pduid` before the current event is inserted to the tree so the state // will be everything up to but not including the incoming event. - self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + self.pduid_statehash.insert(pdu_id, state_hash.as_slice())?; - self.roomid_statehash.insert(room_id.as_bytes(), state_hash.as_bytes())?; + self.roomid_statehash + .insert(room_id.as_bytes(), state_hash.as_slice())?; let mut key = room_id.as_bytes().to_vec(); key.push(0xff); From ea2003240400a5870f1f755d8a13bfb618101aed Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 19 Aug 2020 19:30:28 -0400 Subject: [PATCH 06/51] Helper for join_room_by_id route so routes aren't calling routes --- src/client_server/membership.rs | 325 ++++++++++++++++---------------- 1 file changed, 163 insertions(+), 162 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 555291e5..0d9fa125 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -13,14 +13,14 @@ use ruma::{ membership::{ ban_user, forget_room, get_member_events, invite_user, join_room_by_id, join_room_by_id_or_alias, joined_members, joined_rooms, kick_user, leave_room, - unban_user, + unban_user, IncomingThirdPartySigned, }, }, }, federation, }, events::{room::member, EventType}, - EventId, Raw, RoomId, RoomVersionId, + EventId, Raw, RoomId, RoomVersionId, UserId, }; use state_res::StateEvent; @@ -37,136 +37,13 @@ pub async fn join_room_by_id_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - - // Ask a remote server if we don't have this room - if !db.rooms.exists(&body.room_id)? && body.room_id.server_name() != db.globals.server_name() { - let make_join_response = server_server::send_request( - &db, - body.room_id.server_name().to_string(), - federation::membership::create_join_event_template::v1::Request { - room_id: body.room_id.clone(), - user_id: sender_id.clone(), - ver: vec![RoomVersionId::Version5, RoomVersionId::Version6], - }, - ) - .await?; - - let mut join_event_stub_value = - serde_json::from_str::(make_join_response.event.json().get()) - .map_err(|_| { - Error::BadServerResponse("Invalid make_join event json received from server.") - })?; - - let join_event_stub = - join_event_stub_value - .as_object_mut() - .ok_or(Error::BadServerResponse( - "Invalid make join event object received from server.", - ))?; - - join_event_stub.insert( - "origin".to_owned(), - db.globals.server_name().to_owned().to_string().into(), - ); - join_event_stub.insert( - "origin_server_ts".to_owned(), - utils::millis_since_unix_epoch().into(), - ); - - // Generate event id - let event_id = EventId::try_from(&*format!( - "${}", - ruma::signatures::reference_hash(&join_event_stub_value) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); - - // We don't leave the event id into the pdu because that's only allowed in v1 or v2 rooms - let join_event_stub = join_event_stub_value.as_object_mut().unwrap(); - join_event_stub.remove("event_id"); - - ruma::signatures::hash_and_sign_event( - db.globals.server_name().as_str(), - db.globals.keypair(), - &mut join_event_stub_value, - ) - .expect("event is valid, we just created it"); - - let send_join_response = server_server::send_request( - &db, - body.room_id.server_name().to_string(), - federation::membership::create_join_event::v1::Request { - room_id: body.room_id.clone(), - event_id, - pdu_stub: serde_json::from_value(join_event_stub_value) - .expect("Raw::from_value always works"), - }, - ) - .await?; - - dbg!(&send_join_response); - // todo!("Take send_join_response and 'create' the room using that data"); - - let mut event_map = send_join_response - .room_state - .state - .iter() - .map(|pdu| pdu.deserialize().map(StateEvent::Full).map(|ev| (ev.event_id(), ev))) - .collect::, _>>() - .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - - let _auth_chain = send_join_response - .room_state - .auth_chain - .iter() - .flat_map(|pdu| pdu.deserialize().ok()) - .map(StateEvent::Full) - .collect::>(); - - // TODO make StateResolution's methods free functions ? or no self param ? - let sorted_events_ids = state_res::StateResolution::default() - .reverse_topological_power_sort( - &body.room_id, - &event_map.keys().cloned().collect::>(), - &mut event_map, - &db.rooms, - &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort - ); - - for ev_id in &sorted_events_ids { - // this is a `state_res::StateEvent` that holds a `ruma::Pdu` - let pdu = event_map.get(ev_id).expect("Found event_id in sorted events that is not in resolved state"); - - // We do not rebuild the PDU in this case only insert to DB - db.rooms - .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; - } - } - - let event = member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }; - - db.rooms.build_and_append_pdu( - PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; - - Ok(join_room_by_id::Response::new(body.room_id.clone()).into()) + join_room_by_id_helper( + &db, + body.sender_id.as_ref(), + &body.room_id, + body.third_party_signed.as_ref(), + ) + .await } #[cfg_attr( @@ -185,7 +62,7 @@ pub async fn join_room_by_id_or_alias_route( db, Ruma { body: alias::get_alias::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ "room_alias": room_alias }) + serde_json::json!({ "room_alias": room_alias, }) .to_string() .as_bytes() .to_vec(), @@ -202,36 +79,16 @@ pub async fn join_room_by_id_or_alias_route( } }; - // TODO ruma needs to implement the same constructors for the Incoming variants - let tps = if let Some(in_tps) = &body.third_party_signed { - Some(ruma::api::client::r0::membership::ThirdPartySigned { - token: &in_tps.token, - sender: &in_tps.sender, - signatures: in_tps.signatures.clone(), - mxid: &in_tps.mxid, - }) - } else { - None - }; - - let body = Ruma { - sender_id: body.sender_id.clone(), - device_id: body.device_id.clone(), - json_body: None, - body: join_room_by_id::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ - "room_id": room_id, - "third_party_signed": tps, - }) - .to_string() - .as_bytes() - .to_vec(), - )) - .unwrap(), - }; - Ok(join_room_by_id_or_alias::Response { - room_id: join_room_by_id_route(db2, body).await?.0.room_id, + room_id: join_room_by_id_helper( + &db2, + body.sender_id.as_ref(), + &room_id, + body.third_party_signed.as_ref(), + ) + .await? + .0 + .room_id, } .into()) } @@ -568,3 +425,147 @@ pub fn joined_members_route( Ok(joined_members::Response { joined }.into()) } + +async fn join_room_by_id_helper( + db: &Database, + sender_id: Option<&UserId>, + room_id: &RoomId, + _third_party_signed: Option<&IncomingThirdPartySigned>, +) -> ConduitResult { + let sender_id = sender_id.expect("user is authenticated"); + + // Ask a remote server if we don't have this room + if !db.rooms.exists(&room_id)? && room_id.server_name() != db.globals.server_name() { + let make_join_response = server_server::send_request( + &db, + room_id.server_name().to_string(), + federation::membership::create_join_event_template::v1::Request { + room_id: room_id.clone(), + user_id: sender_id.clone(), + ver: vec![RoomVersionId::Version5, RoomVersionId::Version6], + }, + ) + .await?; + + let mut join_event_stub_value = + serde_json::from_str::(make_join_response.event.json().get()) + .map_err(|_| { + Error::BadServerResponse("Invalid make_join event json received from server.") + })?; + + let join_event_stub = + join_event_stub_value + .as_object_mut() + .ok_or(Error::BadServerResponse( + "Invalid make join event object received from server.", + ))?; + + join_event_stub.insert( + "origin".to_owned(), + db.globals.server_name().to_owned().to_string().into(), + ); + join_event_stub.insert( + "origin_server_ts".to_owned(), + utils::millis_since_unix_epoch().into(), + ); + + // Generate event id + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&join_event_stub_value) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + // We don't leave the event id into the pdu because that's only allowed in v1 or v2 rooms + let join_event_stub = join_event_stub_value.as_object_mut().unwrap(); + join_event_stub.remove("event_id"); + + ruma::signatures::hash_and_sign_event( + db.globals.server_name().as_str(), + db.globals.keypair(), + &mut join_event_stub_value, + ) + .expect("event is valid, we just created it"); + + let send_join_response = server_server::send_request( + &db, + room_id.server_name().to_string(), + federation::membership::create_join_event::v1::Request { + room_id: room_id.clone(), + event_id, + pdu_stub: serde_json::from_value(join_event_stub_value) + .expect("Raw::from_value always works"), + }, + ) + .await?; + + dbg!(&send_join_response); + // todo!("Take send_join_response and 'create' the room using that data"); + + let mut event_map = send_join_response + .room_state + .state + .iter() + .map(|pdu| { + pdu.deserialize() + .map(StateEvent::Full) + .map(|ev| (ev.event_id(), ev)) + }) + .collect::, _>>() + .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; + + let _auth_chain = send_join_response + .room_state + .auth_chain + .iter() + .flat_map(|pdu| pdu.deserialize().ok()) + .map(StateEvent::Full) + .collect::>(); + + // TODO make StateResolution's methods free functions ? or no self param ? + let sorted_events_ids = state_res::StateResolution::default() + .reverse_topological_power_sort( + &room_id, + &event_map.keys().cloned().collect::>(), + &mut event_map, + &db.rooms, + &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort + ); + + for ev_id in &sorted_events_ids { + // this is a `state_res::StateEvent` that holds a `ruma::Pdu` + let pdu = event_map + .get(ev_id) + .expect("Found event_id in sorted events that is not in resolved state"); + + // We do not rebuild the PDU in this case only insert to DB + db.rooms + .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; + } + } + + let event = member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }; + + db.rooms.build_and_append_pdu( + PduBuilder { + room_id: room_id.clone(), + sender: sender_id.clone(), + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &db.globals, + &db.account_data, + )?; + + Ok(join_room_by_id::Response::new(room_id.clone()).into()) +} From 5ccdd3694bd86b7048d089d6a2101ae740a1d0f9 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Thu, 20 Aug 2020 12:12:02 -0400 Subject: [PATCH 07/51] Add helper function for get_alias route --- src/client_server/alias.rs | 28 +++++++++++++++++--------- src/client_server/membership.rs | 35 +++++++++------------------------ 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 7dc90783..12bb8df3 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -1,11 +1,14 @@ use super::State; use crate::{server_server, ConduitResult, Database, Error, Ruma}; -use ruma::api::{ - client::{ - error::ErrorKind, - r0::alias::{create_alias, delete_alias, get_alias}, +use ruma::{ + api::{ + client::{ + error::ErrorKind, + r0::alias::{create_alias, delete_alias, get_alias}, + }, + federation, }, - federation, + RoomAliasId, }; #[cfg(feature = "conduit_bin")] @@ -50,12 +53,19 @@ pub async fn get_alias_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - if body.room_alias.server_name() != db.globals.server_name() { + get_alias_helper(db, &body.room_alias).await +} + +pub async fn get_alias_helper( + db: State<'_, Database>, + room_alias: &RoomAliasId, +) -> ConduitResult { + if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( &db, - body.room_alias.server_name().to_string(), + room_alias.server_name().to_string(), federation::query::get_room_information::v1::Request { - room_alias: body.room_alias.to_string(), + room_alias: room_alias.to_string(), }, ) .await?; @@ -65,7 +75,7 @@ pub async fn get_alias_route( let room_id = db .rooms - .id_from_alias(&body.room_alias)? + .id_from_alias(&room_alias)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "Room with alias not found.", diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 0d9fa125..0075861a 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -8,13 +8,10 @@ use ruma::{ api::{ client::{ error::ErrorKind, - r0::{ - alias, - membership::{ - ban_user, forget_room, get_member_events, invite_user, join_room_by_id, - join_room_by_id_or_alias, joined_members, joined_rooms, kick_user, leave_room, - unban_user, IncomingThirdPartySigned, - }, + r0::membership::{ + ban_user, forget_room, get_member_events, invite_user, join_room_by_id, + join_room_by_id_or_alias, joined_members, joined_rooms, kick_user, leave_room, + unban_user, IncomingThirdPartySigned, }, }, federation, @@ -58,24 +55,10 @@ pub async fn join_room_by_id_or_alias_route( let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, Err(room_alias) => { - client_server::get_alias_route( - db, - Ruma { - body: alias::get_alias::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ "room_alias": room_alias, }) - .to_string() - .as_bytes() - .to_vec(), - )) - .unwrap(), - sender_id: body.sender_id.clone(), - device_id: body.device_id.clone(), - json_body: None, - }, - ) - .await? - .0 - .room_id + client_server::get_alias_helper(db, &room_alias) + .await? + .0 + .room_id } }; @@ -495,7 +478,7 @@ async fn join_room_by_id_helper( room_id: room_id.clone(), event_id, pdu_stub: serde_json::from_value(join_event_stub_value) - .expect("Raw::from_value always works"), + .expect("we just created this event"), }, ) .await?; From fe795d38ead8b83c413187f68e83ccea4185c8c7 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Fri, 21 Aug 2020 17:19:18 -0400 Subject: [PATCH 08/51] Replace route calling routes with helpers This fixes the panic from ruma "index out of bounds" --- src/client_server/alias.rs | 4 +- src/client_server/membership.rs | 20 ++--- src/client_server/state.rs | 142 ++++++++++++++++++-------------- 3 files changed, 90 insertions(+), 76 deletions(-) diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 12bb8df3..669f558d 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -53,11 +53,11 @@ pub async fn get_alias_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - get_alias_helper(db, &body.room_alias).await + get_alias_helper(&db, &body.room_alias).await } pub async fn get_alias_helper( - db: State<'_, Database>, + db: &Database, room_alias: &RoomAliasId, ) -> ConduitResult { if room_alias.server_name() != db.globals.server_name() { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 0075861a..996d3c48 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -49,13 +49,12 @@ pub async fn join_room_by_id_route( )] pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, - db2: State<'_, Database>, body: Ruma, ) -> ConduitResult { let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, Err(room_alias) => { - client_server::get_alias_helper(db, &room_alias) + client_server::get_alias_helper(&db, &room_alias) .await? .0 .room_id @@ -64,7 +63,7 @@ pub async fn join_room_by_id_or_alias_route( Ok(join_room_by_id_or_alias::Response { room_id: join_room_by_id_helper( - &db2, + &db, body.sender_id.as_ref(), &room_id, body.third_party_signed.as_ref(), @@ -507,14 +506,13 @@ async fn join_room_by_id_helper( .collect::>(); // TODO make StateResolution's methods free functions ? or no self param ? - let sorted_events_ids = state_res::StateResolution::default() - .reverse_topological_power_sort( - &room_id, - &event_map.keys().cloned().collect::>(), - &mut event_map, - &db.rooms, - &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort - ); + let sorted_events_ids = state_res::StateResolution::reverse_topological_power_sort( + &room_id, + &event_map.keys().cloned().collect::>(), + &mut event_map, + &db.rooms, + &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort + ); for ev_id in &sorted_events_ids { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 2920de2f..867b051e 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -9,8 +9,8 @@ use ruma::{ }, }, events::{AnyStateEventContent, EventContent}, + RoomId, UserId, }; -use std::convert::TryFrom; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; @@ -33,45 +33,14 @@ pub fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = &body.content { - let mut aliases = canonical_alias.alt_aliases.clone(); - - if let Some(alias) = canonical_alias.alias.clone() { - aliases.push(alias); - } - - for alias in aliases { - if alias.server_name() != db.globals.server_name() - || db - .rooms - .id_from_alias(&alias)? - .filter(|room| room == &body.room_id) // Make sure it's the right room - .is_none() - { - return Err(Error::BadRequest( - ErrorKind::Forbidden, - "You are only allowed to send canonical_alias \ - events when it's aliases already exists", - )); - } - } - } - - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), - event_type: body.content.event_type().into(), - content, - unsigned: None, - state_key: Some(body.state_key.clone()), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; - - Ok(send_state_event_for_key::Response::new(event_id).into()) + send_state_event_for_key_helper( + &db, + sender_id, + &body.content, + content, + &body.room_id, + Some(body.state_key.clone()), + ) } #[cfg_attr( @@ -84,34 +53,30 @@ pub fn send_state_event_for_empty_key_route( ) -> ConduitResult { // This just calls send_state_event_for_key_route let Ruma { - body: - send_state_event_for_empty_key::IncomingRequest { - room_id, content, .. - }, + body, sender_id, - device_id, + device_id: _, json_body, } = body; + let json = serde_json::from_str::( + json_body + .as_ref() + .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? + .get(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; + Ok(send_state_event_for_empty_key::Response::new( - send_state_event_for_key_route( - db, - Ruma { - body: send_state_event_for_key::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ - "room_id": room_id, - "state_key": "", - "content": content, - }) - .to_string() - .as_bytes() - .to_vec(), - )) - .unwrap(), - sender_id, - device_id, - json_body, - }, + send_state_event_for_key_helper( + &db, + sender_id + .as_ref() + .expect("no user for send state empty key rout"), + &body.content, + json, + &body.room_id, + None, )? .0 .event_id, @@ -210,3 +175,54 @@ pub fn get_state_events_for_empty_key_route( } .into()) } + +pub fn send_state_event_for_key_helper( + db: &Database, + sender: &UserId, + content: &AnyStateEventContent, + json: serde_json::Value, + room_id: &RoomId, + state_key: Option, +) -> ConduitResult { + let sender_id = sender; + + if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = content { + let mut aliases = canonical_alias.alt_aliases.clone(); + + if let Some(alias) = canonical_alias.alias.clone() { + aliases.push(alias); + } + + for alias in aliases { + if alias.server_name() != db.globals.server_name() + || db + .rooms + .id_from_alias(&alias)? + .filter(|room| room == room_id) // Make sure it's the right room + .is_none() + { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "You are only allowed to send canonical_alias \ + events when it's aliases already exists", + )); + } + } + } + + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + room_id: room_id.clone(), + sender: sender_id.clone(), + event_type: content.event_type().into(), + content: json, + unsigned: None, + state_key, + redacts: None, + }, + &db.globals, + &db.account_data, + )?; + + Ok(send_state_event_for_key::Response::new(event_id).into()) +} From 672bf4f47376eea520c19fa1b9108478ec56974f Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Fri, 21 Aug 2020 20:18:56 -0400 Subject: [PATCH 09/51] Cargo lock update and a few doc additions --- Cargo.lock | 22 +++++++++++----------- src/client_server/config.rs | 6 +----- src/client_server/room.rs | 1 + src/database/rooms.rs | 2 +- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c5d836b..0e5da6f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,9 +144,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" @@ -247,9 +247,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cc" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518" +checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381" [[package]] name = "cfg-if" @@ -960,9 +960,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.74" +version = "0.2.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10" +checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3" [[package]] name = "lock_api" @@ -1358,9 +1358,9 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" +checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" [[package]] name = "proc-macro-crate" @@ -2088,7 +2088,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res#4e9b428c0db50ac3a3421ced12a6fd202a1c36a3" +source = "git+https://github.com/ruma/state-res#d93a965ad17781fa9554bb3cea71673c054b9f3f" dependencies = [ "itertools", "js_int", @@ -2179,9 +2179,9 @@ checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" [[package]] name = "syn" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e69abc24912995b3038597a7a593be5053eb0fb44f3cc5beec0deb421790c1f4" +checksum = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9" dependencies = [ "proc-macro2", "quote", diff --git a/src/client_server/config.rs b/src/client_server/config.rs index 8cb6a0d7..45aec33e 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -56,11 +56,7 @@ pub fn get_global_account_data_route( let data = db .account_data - .get::>( - None, - sender_id, - EventType::try_from(&body.event_type).expect("EventType::try_from can never fail"), - )? + .get::>(None, sender_id, body.event_type.clone().into())? .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?; Ok(get_global_account_data::Response { account_data: data }.into()) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 3ee21b6d..c0603d36 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -195,6 +195,7 @@ pub fn create_room_route( content: match preset { create_room::RoomPreset::PublicChat => { serde_json::to_value(guest_access::GuestAccessEventContent::new( + // In a public room a joining is the only way to access guest_access::GuestAccess::Forbidden, )) .expect("event is valid, we just created it") diff --git a/src/database/rooms.rs b/src/database/rooms.rs index c8ff198b..01297421 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -722,7 +722,7 @@ impl Rooms { } EventType::RoomMember => event_auth::is_membership_change_allowed( // TODO this is a bit of a hack but not sure how to have a type - // declared in `state_res` crate be + // declared in `state_res` crate easily convert to/from conduit::PduEvent Requester { prev_event_ids: prev_events.to_owned(), room_id: &room_id, From 972babbc795c9bfc6f3aee7b6351c0deb7e51b5d Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 19 Aug 2020 18:26:39 +0200 Subject: [PATCH 10/51] fix: set limited to true when skipping messages in /sync --- src/client_server/config.rs | 3 +-- src/client_server/search.rs | 27 +++++++++++++-------------- src/database/rooms.rs | 19 +++++++++++++++++-- src/pdu.rs | 24 +++++++++++++++++++++++- 4 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/client_server/config.rs b/src/client_server/config.rs index 45aec33e..baa9381b 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -5,10 +5,9 @@ use ruma::{ error::ErrorKind, r0::config::{get_global_account_data, set_global_account_data}, }, - events::{custom::CustomEventContent, BasicEvent, EventType}, + events::{custom::CustomEventContent, BasicEvent}, Raw, }; -use std::convert::TryFrom; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; diff --git a/src/client_server/search.rs b/src/client_server/search.rs index dec1ec9f..082711d1 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -14,7 +14,7 @@ use std::collections::BTreeMap; )] pub fn search_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,8 @@ pub fn search_events_route( result: db .rooms .get_pdu_from_id(&result)? - .map(|pdu| pdu.to_room_event()), + // TODO this is an awkward type conversion see method + .map(|pdu| pdu.to_any_event()), }) }) .filter_map(|r| r.ok()) @@ -70,17 +71,15 @@ pub fn search_events_route( Some((skip + limit).to_string()) }; - Ok(search_events::Response { - search_categories: ResultCategories { - room_events: Some(ResultRoomEvents { - count: uint!(0), // TODO - groups: BTreeMap::new(), // TODO - next_batch, - results, - state: BTreeMap::new(), // TODO - highlights: search.1, - }), - }, - } + Ok(search_events::Response::new(ResultCategories { + room_events: Some(ResultRoomEvents { + count: uint!(0), // TODO + groups: BTreeMap::new(), // TODO + next_batch, + results, + state: BTreeMap::new(), // TODO + highlights: search.1, + }), + }) .into()) } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 01297421..d087d652 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -532,7 +532,7 @@ impl Rooms { self.append_state_pdu(&pdu.room_id, &pdu_id, state_key, &pdu.kind)?; } - match pdu.kind { + match &pdu.kind { EventType::RoomRedaction => { if let Some(redact_id) = &pdu.redacts { // TODO: Reason @@ -553,7 +553,7 @@ impl Rooms { } } EventType::RoomMember => { - if let Some(state_key) = &pdu.state_key { + if let Some(state_key) = pdu.state_key.as_ref() { // if the state_key fails let target_user_id = UserId::try_from(state_key.as_str()) .expect("This state_key was previously validated"); @@ -576,6 +576,21 @@ impl Rooms { )?; } } + EventType::RoomMessage => { + if let Some(body) = pdu.content.get("body").and_then(|b| b.as_str()) { + for word in body + .split_terminator(|c: char| !c.is_alphanumeric()) + .map(str::to_lowercase) + { + let mut key = pdu.room_id.to_string().as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(word.as_bytes()); + key.push(0xff); + key.extend_from_slice(&pdu_id); + self.tokenids.insert(key, &[])?; + } + } + } _ => {} } self.edus.room_read_set(&pdu.room_id, &pdu.sender, index)?; diff --git a/src/pdu.rs b/src/pdu.rs index eec8e496..b565a24c 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -2,7 +2,7 @@ use crate::{Error, Result}; use js_int::UInt; use ruma::{ events::{ - pdu::EventHash, room::member::MemberEventContent, AnyRoomEvent, AnyStateEvent, + pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, EventId, Raw, RoomId, ServerName, UserId, @@ -99,6 +99,28 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } + /// This only works for events that are also AnyRoomEvents. + pub fn to_any_event(&self) -> Raw { + let mut json = json!({ + "content": self.content, + "type": self.kind, + "event_id": self.event_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "unsigned": self.unsigned, + "room_id": self.room_id, + }); + + if let Some(state_key) = &self.state_key { + json["state_key"] = json!(state_key); + } + if let Some(redacts) = &self.redacts { + json["redacts"] = json!(redacts); + } + + serde_json::from_value(json).expect("Raw::from_value always works") + } + pub fn to_room_event(&self) -> Raw { let mut json = json!({ "content": self.content, From 3c26166fb59d78cf887645144ffbb108328247df Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Aug 2020 21:22:59 +0200 Subject: [PATCH 11/51] improvement: device list works better The only situation that isn't working yet is sending `left` events for users when the sender leaves the room --- src/client_server/sync.rs | 20 ++++++-------------- src/database/rooms.rs | 9 ++++----- src/utils.rs | 6 ------ 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index ae4c2242..accb199b 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -149,15 +149,7 @@ pub async fn sync_events_route( device_list_updates.extend( db.rooms .room_members(&room_id) - .filter_map(|user_id| { - Some( - UserId::try_from(user_id.ok()?.clone()) - .map_err(|_| { - Error::bad_database("Invalid member event state key in db.") - }) - .ok()?, - ) - }) + .filter_map(|user_id| Some(user_id.ok()?)) .filter(|user_id| { // Don't send key updates from the sender to the sender sender_id != user_id @@ -491,9 +483,7 @@ pub async fn sync_events_route( } for user_id in left_encrypted_users { - // If the user doesn't share an encrypted room with the target anymore, we need to tell - // them - if db + let user_target_encrypted = db .rooms .get_shared_rooms(vec![sender_id.clone(), user_id.clone()]) .filter_map(|r| r.ok()) @@ -505,8 +495,10 @@ pub async fn sync_events_route( .is_some(), ) }) - .all(|encrypted| !encrypted) - { + .all(|encrypted| !encrypted); + // If the user doesn't share an encrypted room with the target anymore, we need to tell + // them + if user_target_encrypted { device_list_left.insert(user_id); } } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index d087d652..575a2bf0 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -75,23 +75,23 @@ impl StateStore for Rooms { .map_err(|e| e.to_string())? .ok_or_else(|| "PDU via room_id and event_id not found in the db.".to_owned())?; - utils::deserialize( + serde_json::from_slice( &self .pduid_pdu .get(pid) .map_err(|e| e.to_string())? .ok_or_else(|| "PDU via pduid not found in db.".to_owned())?, ) + .map_err(|e| e.to_string()) .and_then(|pdu: StateEvent| { // conduit's PDU's always contain a room_id but some // of ruma's do not so this must be an Option if pdu.room_id() == Some(room_id) { Ok(pdu) } else { - Err(Error::bad_database("Found PDU for incorrect room in db.")) + Err("Found PDU for incorrect room in db.".into()) } }) - .map_err(|e| e.to_string()) } } @@ -1207,8 +1207,7 @@ impl Rooms { let roomid_index = key .iter() .enumerate() - .filter(|(_, &b)| b == 0xff) - .nth(0) + .find(|(_, &b)| b == 0xff) .ok_or_else(|| Error::bad_database("Invalid userroomid_joined in db."))? .0 + 1; // +1 because the room id starts AFTER the separator diff --git a/src/utils.rs b/src/utils.rs index b549153a..8cf1b2ce 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,3 @@ -use crate::Error; use argon2::{Config, Variant}; use cmp::Ordering; use rand::prelude::*; @@ -91,8 +90,3 @@ pub fn common_elements( .all(|b| b) })) } - -pub fn deserialize<'de, T: serde::Deserialize<'de>>(val: &'de sled::IVec) -> Result { - serde_json::from_slice::(val.as_ref()) - .map_err(|_| Error::bad_database("Found invalid bytes as PDU in db.")) -} From 1848f08292875030f100bc4ffa987e31c15d6912 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Fri, 21 Aug 2020 21:44:55 -0400 Subject: [PATCH 12/51] Use full sorting algorithm on incoming PDU's in membership --- src/client_server/membership.rs | 50 +++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 996d3c48..3fa3b6ae 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -497,7 +497,7 @@ async fn join_room_by_id_helper( .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - let _auth_chain = send_join_response + let auth_chain = send_join_response .room_state .auth_chain .iter() @@ -505,16 +505,54 @@ async fn join_room_by_id_helper( .map(StateEvent::Full) .collect::>(); - // TODO make StateResolution's methods free functions ? or no self param ? - let sorted_events_ids = state_res::StateResolution::reverse_topological_power_sort( + let power_events = event_map + .values() + .filter(|pdu| pdu.is_power_event()) + .map(|pdu| pdu.event_id()) + .collect::>(); + + // TODO these events are not guaranteed to be sorted but they are resolved, do + // we need the auth_chain + let sorted_power_events = state_res::StateResolution::reverse_topological_power_sort( &room_id, - &event_map.keys().cloned().collect::>(), + &power_events, + &mut event_map, + &db.rooms, + &auth_chain // if we only use it here just build this list in the first place + .iter() + .map(|pdu| pdu.event_id()) + .collect::>(), + ); + + // TODO we may be able to skip this since they are resolved according to spec + let resolved_power = state_res::StateResolution::iterative_auth_check( + room_id, + &RoomVersionId::Version6, + &sorted_power_events, + &BTreeMap::new(), // unconflicted events + &mut event_map, + &db.rooms, + ) + .expect("iterative auth check failed on resolved events"); + // TODO do we need to dedup them + + let events_to_sort = event_map + .keys() + .filter(|id| !sorted_power_events.contains(id)) + .cloned() + .collect::>(); + + let power_level = resolved_power.get(&(EventType::RoomPowerLevels, Some("".into()))); + + let sorted_event_ids = state_res::StateResolution::mainline_sort( + room_id, + &events_to_sort, + power_level, &mut event_map, &db.rooms, - &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort ); - for ev_id in &sorted_events_ids { + for ev_id in &sorted_event_ids { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` let pdu = event_map .get(ev_id) From 27ffe778233370347d50b914862b6e866dae557b Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 23 Aug 2020 08:32:43 -0400 Subject: [PATCH 13/51] Use helper instead of route for get_public_rooms_filtered --- src/client_server/directory.rs | 224 ++++++++++++++++++--------------- src/client_server/sync.rs | 10 +- src/server_server.rs | 25 ++-- 3 files changed, 131 insertions(+), 128 deletions(-) diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 0aace15d..5e03274b 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,7 +14,7 @@ use ruma::{ }, federation, }, - directory::PublicRoomsChunk, + directory::{Filter, PublicRoomsChunk, RoomNetwork}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, @@ -33,17 +33,123 @@ pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - if let Some(other_server) = body - .server + let Ruma { + body: + get_public_rooms_filtered::IncomingRequest { + limit, + server, + since, + filter, + room_network, + }, + .. + } = body; + get_public_rooms_filtered_helper( + &db, + server.as_deref(), + limit, + since.as_deref(), + filter, // This is not used yet + Some(room_network), // This is not used + ) + .await +} + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/client/r0/publicRooms", data = "") +)] +pub async fn get_public_rooms_route( + db: State<'_, Database>, + body: Ruma, +) -> ConduitResult { + let Ruma { + body: + get_public_rooms::IncomingRequest { + limit, + server, + since, + }, + .. + } = body; + + let get_public_rooms_filtered::Response { + chunk, + prev_batch, + next_batch, + total_room_count_estimate, + } = get_public_rooms_filtered_helper( + &db, + server.as_deref(), + limit, + since.as_deref(), + None, // This is not used + None, // This is not used + ) + .await? + .0; + + Ok(get_public_rooms::Response { + chunk, + prev_batch, + next_batch, + total_room_count_estimate, + } + .into()) +} + +#[cfg_attr( + feature = "conduit_bin", + put("/_matrix/client/r0/directory/list/room/<_>", data = "") +)] +pub async fn set_room_visibility_route( + db: State<'_, Database>, + body: Ruma, +) -> ConduitResult { + match body.visibility { + room::Visibility::Public => db.rooms.set_public(&body.room_id, true)?, + room::Visibility::Private => db.rooms.set_public(&body.room_id, false)?, + } + + Ok(set_room_visibility::Response.into()) +} + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/client/r0/directory/list/room/<_>", data = "") +)] +pub async fn get_room_visibility_route( + db: State<'_, Database>, + body: Ruma, +) -> ConduitResult { + Ok(get_room_visibility::Response { + visibility: if db.rooms.is_public_room(&body.room_id)? { + room::Visibility::Public + } else { + room::Visibility::Private + }, + } + .into()) +} + +pub async fn get_public_rooms_filtered_helper( + db: &Database, + server: Option<&str>, + limit: Option, + since: Option<&str>, + _filter: Option, + _network: Option, +) -> ConduitResult { + if let Some(other_server) = server .clone() - .filter(|server| server != db.globals.server_name().as_str()) + .filter(|server| *server != db.globals.server_name().as_str()) { let response = server_server::send_request( &db, - other_server, + other_server.to_owned(), federation::directory::get_public_rooms::v1::Request { - limit: body.limit, - since: body.since.as_deref(), + limit, + since: since.as_deref(), room_network: ruma::directory::RoomNetwork::Matrix, }, ) @@ -73,10 +179,10 @@ pub async fn get_public_rooms_filtered_route( .into()); } - let limit = body.limit.map_or(10, u64::from); - let mut since = 0_u64; + let limit = limit.map_or(10, u64::from); + let mut num_since = 0_u64; - if let Some(s) = &body.since { + if let Some(s) = &since { let mut characters = s.chars(); let backwards = match characters.next() { Some('n') => false, @@ -89,13 +195,13 @@ pub async fn get_public_rooms_filtered_route( } }; - since = characters + num_since = characters .collect::() .parse() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `since` token."))?; if backwards { - since = since.saturating_sub(limit); + num_since = num_since.saturating_sub(limit); } } @@ -217,20 +323,20 @@ pub async fn get_public_rooms_filtered_route( let chunk = all_rooms .into_iter() - .skip(since as usize) + .skip(num_since as usize) .take(limit as usize) .collect::>(); - let prev_batch = if since == 0 { + let prev_batch = if num_since == 0 { None } else { - Some(format!("p{}", since)) + Some(format!("p{}", num_since)) }; let next_batch = if chunk.len() < limit as usize { None } else { - Some(format!("n{}", since + limit)) + Some(format!("n{}", num_since + limit)) }; Ok(get_public_rooms_filtered::Response { @@ -241,89 +347,3 @@ pub async fn get_public_rooms_filtered_route( } .into()) } - -#[cfg_attr( - feature = "conduit_bin", - get("/_matrix/client/r0/publicRooms", data = "") -)] -pub async fn get_public_rooms_route( - db: State<'_, Database>, - body: Ruma, -) -> ConduitResult { - let Ruma { - body: - get_public_rooms::IncomingRequest { - limit, - server, - since, - }, - sender_id, - device_id, - json_body, - } = body; - - let get_public_rooms_filtered::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } = get_public_rooms_filtered_route( - db, - Ruma { - body: get_public_rooms_filtered::IncomingRequest { - filter: None, - limit, - room_network: ruma::directory::RoomNetwork::Matrix, - server, - since, - }, - sender_id, - device_id, - json_body, - }, - ) - .await? - .0; - - Ok(get_public_rooms::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } - .into()) -} - -#[cfg_attr( - feature = "conduit_bin", - put("/_matrix/client/r0/directory/list/room/<_>", data = "") -)] -pub async fn set_room_visibility_route( - db: State<'_, Database>, - body: Ruma, -) -> ConduitResult { - match body.visibility { - room::Visibility::Public => db.rooms.set_public(&body.room_id, true)?, - room::Visibility::Private => db.rooms.set_public(&body.room_id, false)?, - } - - Ok(set_room_visibility::Response.into()) -} - -#[cfg_attr( - feature = "conduit_bin", - get("/_matrix/client/r0/directory/list/room/<_>", data = "") -)] -pub async fn get_room_visibility_route( - db: State<'_, Database>, - body: Ruma, -) -> ConduitResult { - Ok(get_room_visibility::Response { - visibility: if db.rooms.is_public_room(&body.room_id)? { - room::Visibility::Public - } else { - room::Visibility::Private - }, - } - .into()) -} diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index accb199b..ccb25d19 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -415,15 +415,7 @@ pub async fn sync_events_route( device_list_left.extend( db.rooms .room_members(&room_id) - .filter_map(|user_id| { - Some( - UserId::try_from(user_id.ok()?.clone()) - .map_err(|_| { - Error::bad_database("Invalid member event state key in db.") - }) - .ok()?, - ) - }) + .filter_map(|user_id| Some(user_id.ok()?)) .filter(|user_id| { // Don't send key updates from the sender to the sender sender_id != user_id diff --git a/src/server_server.rs b/src/server_server.rs index e47b50a5..ac4407ba 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -216,9 +216,7 @@ pub async fn get_public_rooms_route( limit, since, }, - sender_id, - device_id, - json_body, + .. } = body; let client::r0::directory::get_public_rooms_filtered::Response { @@ -226,20 +224,13 @@ pub async fn get_public_rooms_route( prev_batch, next_batch, total_room_count_estimate, - } = client_server::get_public_rooms_filtered_route( - db, - Ruma { - body: client::r0::directory::get_public_rooms_filtered::IncomingRequest { - filter: None, - limit, - room_network: ruma::directory::RoomNetwork::Matrix, - server: None, - since, - }, - sender_id, - device_id, - json_body, - }, + } = client_server::get_public_rooms_filtered_helper( + &db, + None, + limit, + since.as_deref(), + None, + Some(ruma::directory::RoomNetwork::Matrix), ) .await? .0; From d9a29e3e5c2b8390bb1a4b99a631b3ac615b0763 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Tue, 25 Aug 2020 15:30:25 -0400 Subject: [PATCH 14/51] Fix state for empty key route Replace None with Some("") for state_key --- src/client_server/room.rs | 2 +- src/client_server/state.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index c0603d36..7f4a15fa 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -195,7 +195,7 @@ pub fn create_room_route( content: match preset { create_room::RoomPreset::PublicChat => { serde_json::to_value(guest_access::GuestAccessEventContent::new( - // In a public room a joining is the only way to access + // In a public room, joining is the only way to access guest_access::GuestAccess::Forbidden, )) .expect("event is valid, we just created it") diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 867b051e..12c5cacb 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -76,7 +76,7 @@ pub fn send_state_event_for_empty_key_route( &body.content, json, &body.room_id, - None, + Some("".into()), )? .0 .event_id, From f46c2d1eec808658f671a4845b0ecd39221c8826 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 26 Aug 2020 11:15:52 -0400 Subject: [PATCH 15/51] Fix review issues, move state-res to spec-comp branch --- Cargo.lock | 79 ++++++++++++++------------------- Cargo.toml | 2 +- src/client_server/directory.rs | 31 ++++--------- src/client_server/membership.rs | 16 +++---- src/client_server/room.rs | 1 - src/client_server/state.rs | 35 ++++++++------- src/client_server/sync.rs | 4 +- src/database/rooms.rs | 76 +++++++++++++++---------------- 8 files changed, 106 insertions(+), 138 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e5da6f6..62b13d00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,7 +157,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.4.0", + "miniz_oxide 0.4.1", "object", "rustc-demangle", ] @@ -168,12 +168,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" -[[package]] -name = "base64" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" - [[package]] name = "base64" version = "0.12.3" @@ -265,7 +259,7 @@ checksum = "942f72db697d8767c22d46a598e01f2d3b475501ea43d0db4f16d90259182d0b" dependencies = [ "num-integer", "num-traits", - "time 0.1.43", + "time 0.1.44", ] [[package]] @@ -287,7 +281,7 @@ checksum = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd" name = "conduit" version = "0.1.0" dependencies = [ - "base64 0.12.3", + "base64", "directories", "http", "image", @@ -298,7 +292,7 @@ dependencies = [ "ring", "rocket", "ruma", - "rust-argon2 0.8.2", + "rust-argon2", "serde", "serde_json", "sled", @@ -320,7 +314,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ "aes-gcm", - "base64 0.12.3", + "base64", "hkdf", "percent-encoding", "rand", @@ -486,9 +480,9 @@ checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" [[package]] name = "encoding_rs" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171" +checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" dependencies = [ "cfg-if", ] @@ -672,7 +666,7 @@ checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.9.0+wasi-snapshot-preview1", ] [[package]] @@ -816,7 +810,7 @@ dependencies = [ "itoa", "pin-project", "socket2", - "time 0.1.43", + "time 0.1.44", "tokio", "tower-service", "tracing", @@ -1057,9 +1051,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" +checksum = "4d7559a8a40d0f97e1edea3220f698f78b1c5ab67532e49f68fde3910323b722" dependencies = [ "adler", ] @@ -1462,13 +1456,13 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_users" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b23093265f8d200fa7b4c2c76297f47e681c655f6f1285a8780d6a022f7431" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" dependencies = [ "getrandom", "redox_syscall", - "rust-argon2 0.7.0", + "rust-argon2", ] [[package]] @@ -1527,11 +1521,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12427a5577082c24419c9c417db35cfeb65962efc7675bb6b0d5f1f9d315bfe6" +checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" dependencies = [ - "base64 0.12.3", + "base64", "bytes", "encoding_rs", "futures-core", @@ -1813,31 +1807,19 @@ name = "ruma-signatures" version = "0.6.0-dev.1" source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ - "base64 0.12.3", + "base64", "ring", "serde_json", "untrusted", ] -[[package]] -name = "rust-argon2" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017" -dependencies = [ - "base64 0.11.0", - "blake2b_simd", - "constant_time_eq", - "crossbeam-utils", -] - [[package]] name = "rust-argon2" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19" dependencies = [ - "base64 0.12.3", + "base64", "blake2b_simd", "constant_time_eq", "crossbeam-utils", @@ -1864,7 +1846,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" dependencies = [ - "base64 0.12.3", + "base64", "log", "ring", "sct", @@ -2072,9 +2054,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0437cfb83762844799a60e1e3b489d5ceb6a650fbacb86437badc1b6d87b246" +checksum = "33a71ea1ea5f8747d1af1979bfb7e65c3a025a70609f04ceb78425bc5adad8e6" dependencies = [ "version_check", ] @@ -2088,7 +2070,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res#d93a965ad17781fa9554bb3cea71673c054b9f3f" +source = "git+https://github.com/ruma/state-res?branch=spec-comp#17958665f6592af3ef478024fd1d75c384a30e7f" dependencies = [ "itertools", "js_int", @@ -2233,11 +2215,12 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi 0.3.9", ] @@ -2395,9 +2378,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db63662723c316b43ca36d833707cc93dff82a02ba3d7e354f342682cc8b3545" +checksum = "4f0e00789804e99b20f12bc7003ca416309d28a6f495d6af58d1e2c2842461b5" dependencies = [ "lazy_static", ] @@ -2550,6 +2533,12 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasm-bindgen" version = "0.2.67" diff --git a/Cargo.toml b/Cargo.toml index 78d8f760..15cee725 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key # state-res = { path = "../../state-res" } -state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0" } +state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } ring = "0.16.15" [features] diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 5e03274b..3b106866 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -63,26 +63,11 @@ pub async fn get_public_rooms_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - let Ruma { - body: - get_public_rooms::IncomingRequest { - limit, - server, - since, - }, - .. - } = body; - - let get_public_rooms_filtered::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } = get_public_rooms_filtered_helper( + let response = get_public_rooms_filtered_helper( &db, - server.as_deref(), - limit, - since.as_deref(), + body.body.server.as_deref(), + body.body.limit, + body.body.since.as_deref(), None, // This is not used None, // This is not used ) @@ -90,10 +75,10 @@ pub async fn get_public_rooms_route( .0; Ok(get_public_rooms::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, + chunk: response.chunk, + prev_batch: response.prev_batch, + next_batch: response.next_batch, + total_room_count_estimate: response.total_room_count_estimate, } .into()) } diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 3fa3b6ae..90683e61 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -483,12 +483,12 @@ async fn join_room_by_id_helper( .await?; dbg!(&send_join_response); - // todo!("Take send_join_response and 'create' the room using that data"); let mut event_map = send_join_response .room_state .state .iter() + .chain(send_join_response.room_state.auth_chain.iter()) .map(|pdu| { pdu.deserialize() .map(StateEvent::Full) @@ -497,14 +497,6 @@ async fn join_room_by_id_helper( .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - let auth_chain = send_join_response - .room_state - .auth_chain - .iter() - .flat_map(|pdu| pdu.deserialize().ok()) - .map(StateEvent::Full) - .collect::>(); - let power_events = event_map .values() .filter(|pdu| pdu.is_power_event()) @@ -518,9 +510,11 @@ async fn join_room_by_id_helper( &power_events, &mut event_map, &db.rooms, - &auth_chain // if we only use it here just build this list in the first place + &send_join_response + .room_state + .auth_chain .iter() - .map(|pdu| pdu.event_id()) + .filter_map(|pdu| Some(StateEvent::Full(pdu.deserialize().ok()?).event_id())) .collect::>(), ); diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 7f4a15fa..3ee21b6d 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -195,7 +195,6 @@ pub fn create_room_route( content: match preset { create_room::RoomPreset::PublicChat => { serde_json::to_value(guest_access::GuestAccessEventContent::new( - // In a public room, joining is the only way to access guest_access::GuestAccess::Forbidden, )) .expect("event is valid, we just created it") diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 12c5cacb..e7d2bcf3 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -1,5 +1,5 @@ use super::State; -use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Ruma}; +use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Result, Ruma}; use ruma::{ api::client::{ error::ErrorKind, @@ -9,7 +9,7 @@ use ruma::{ }, }, events::{AnyStateEventContent, EventContent}, - RoomId, UserId, + EventId, RoomId, UserId, }; #[cfg(feature = "conduit_bin")] @@ -33,13 +33,16 @@ pub fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - send_state_event_for_key_helper( - &db, - sender_id, - &body.content, - content, - &body.room_id, - Some(body.state_key.clone()), + Ok( + send_state_event_for_key::Response::new(send_state_event_for_key_helper( + &db, + sender_id, + &body.content, + content, + &body.room_id, + Some(body.state_key.clone()), + )?) + .into(), ) } @@ -67,8 +70,8 @@ pub fn send_state_event_for_empty_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - Ok(send_state_event_for_empty_key::Response::new( - send_state_event_for_key_helper( + Ok( + send_state_event_for_empty_key::Response::new(send_state_event_for_key_helper( &db, sender_id .as_ref() @@ -77,11 +80,9 @@ pub fn send_state_event_for_empty_key_route( json, &body.room_id, Some("".into()), - )? - .0 - .event_id, + )?) + .into(), ) - .into()) } #[cfg_attr( @@ -183,7 +184,7 @@ pub fn send_state_event_for_key_helper( json: serde_json::Value, room_id: &RoomId, state_key: Option, -) -> ConduitResult { +) -> Result { let sender_id = sender; if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = content { @@ -224,5 +225,5 @@ pub fn send_state_event_for_key_helper( &db.account_data, )?; - Ok(send_state_event_for_key::Response::new(event_id).into()) + Ok(event_id) } diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index ccb25d19..74329605 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -475,7 +475,7 @@ pub async fn sync_events_route( } for user_id in left_encrypted_users { - let user_target_encrypted = db + let still_share_encrypted_room = db .rooms .get_shared_rooms(vec![sender_id.clone(), user_id.clone()]) .filter_map(|r| r.ok()) @@ -490,7 +490,7 @@ pub async fn sync_events_route( .all(|encrypted| !encrypted); // If the user doesn't share an encrypted room with the target anymore, we need to tell // them - if user_target_encrypted { + if still_share_encrypted_room { device_list_left.insert(user_id); } } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 575a2bf0..66af7363 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -4,7 +4,6 @@ pub use edus::RoomEdus; use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; use log::error; -// TODO if ruma-signatures re-exports `use ruma::signatures::digest;` use ring::digest; use ruma::{ api::client::error::ErrorKind, @@ -96,9 +95,9 @@ impl StateStore for Rooms { } impl Rooms { - /// Builds a `StateMap` by iterating over all keys that start - /// with `state_hash`, this gives the full state at event "x". - pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { + /// Builds a StateMap by iterating over all keys that start + /// with state_hash, this gives the full state for the given state_hash. + pub fn state_full(&self, state_hash: StateHashId) -> Result> { self.stateid_pduid .scan_prefix(&state_hash) .values() @@ -242,8 +241,6 @@ impl Rooms { /// Generate a new StateHash. /// /// A unique hash made from hashing the current states pduid's. - /// Because `append_state_pdu` handles the empty state db case it does not - /// have to be here. fn new_state_hash_id(&self, room_id: &RoomId) -> Result { // Use hashed roomId as the first StateHash key for first state event in room if self @@ -281,7 +278,7 @@ impl Rooms { /// Checks if a room exists. pub fn exists(&self, room_id: &RoomId) -> Result { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); // Look for PDUs in that room. @@ -300,7 +297,7 @@ impl Rooms { let mut hashmap = HashMap::new(); for pdu in self.roomstateid_pduid - .scan_prefix(&room_id.to_string().as_bytes()) + .scan_prefix(&room_id.as_bytes()) .values() .map(|value| { Ok::<_, Error>( @@ -322,13 +319,13 @@ impl Rooms { Ok(hashmap) } - /// Returns the all state entries for this type. + /// Returns all state entries for this type. pub fn room_state_type( &self, room_id: &RoomId, event_type: &EventType, ) -> Result> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(&event_type.to_string().as_bytes()); @@ -357,7 +354,7 @@ impl Rooms { Ok(hashmap) } - /// Returns a single PDU in `room_id` with key (`event_type`, `state_key`). + /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`). pub fn room_state_get( &self, room_id: &RoomId, @@ -459,7 +456,7 @@ impl Rooms { /// Returns the leaf pdus of a room. pub fn get_pdu_leaves(&self, room_id: &RoomId) -> Result> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); let mut events = Vec::new(); @@ -582,7 +579,7 @@ impl Rooms { .split_terminator(|c: char| !c.is_alphanumeric()) .map(str::to_lowercase) { - let mut key = pdu.room_id.to_string().as_bytes().to_vec(); + let mut key = pdu.room_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(word.as_bytes()); key.push(0xff); @@ -752,7 +749,10 @@ impl Rooms { }) .collect::>>()?, ) - .ok_or(Error::Conflict("Found incoming PDU with invalid data."))?, + .map_err(|e| { + log::error!("{}", e); + Error::Conflict("Found incoming PDU with invalid data.") + })?, EventType::RoomCreate => prev_events.is_empty(), // Not allow any of the following events if the sender is not joined. _ if sender_membership != member::MembershipState::Join => false, @@ -982,13 +982,13 @@ impl Rooms { globals: &super::globals::Globals, ) -> Result<()> { let membership = member_content.membership; - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); - let mut roomuser_id = room_id.to_string().as_bytes().to_vec(); + let mut roomuser_id = room_id.as_bytes().to_vec(); roomuser_id.push(0xff); - roomuser_id.extend_from_slice(user_id.to_string().as_bytes()); + roomuser_id.extend_from_slice(user_id.as_bytes()); match &membership { member::MembershipState::Join => { @@ -1051,9 +1051,9 @@ impl Rooms { /// Makes a user forget a room. pub fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); self.userroomid_left.remove(userroom_id)?; @@ -1069,8 +1069,8 @@ impl Rooms { if let Some(room_id) = room_id { // New alias self.alias_roomid - .insert(alias.alias(), &*room_id.to_string())?; - let mut aliasid = room_id.to_string().as_bytes().to_vec(); + .insert(alias.alias(), room_id.as_bytes())?; + let mut aliasid = room_id.as_bytes().to_vec(); aliasid.extend_from_slice(&globals.next_count()?.to_be_bytes()); self.aliasid_alias.insert(aliasid, &*alias.alias())?; } else { @@ -1105,7 +1105,7 @@ impl Rooms { } pub fn room_aliases(&self, room_id: &RoomId) -> impl Iterator> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); self.aliasid_alias @@ -1119,16 +1119,16 @@ impl Rooms { pub fn set_public(&self, room_id: &RoomId, public: bool) -> Result<()> { if public { - self.publicroomids.insert(room_id.to_string(), &[])?; + self.publicroomids.insert(room_id.as_bytes(), &[])?; } else { - self.publicroomids.remove(room_id.to_string())?; + self.publicroomids.remove(room_id.as_bytes())?; } Ok(()) } pub fn is_public_room(&self, room_id: &RoomId) -> Result { - Ok(self.publicroomids.contains_key(room_id.to_string())?) + Ok(self.publicroomids.contains_key(room_id.as_bytes())?) } pub fn public_rooms(&self) -> impl Iterator> { @@ -1147,7 +1147,7 @@ impl Rooms { room_id: &RoomId, search_string: &str, ) -> Result<(impl Iterator + 'a, Vec)> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); let words = search_string @@ -1233,7 +1233,7 @@ impl Rooms { /// Returns an iterator over all joined members of a room. pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> { self.roomuserid_joined - .scan_prefix(room_id.to_string()) + .scan_prefix(room_id.as_bytes()) .keys() .map(|key| { Ok(UserId::try_from( @@ -1254,7 +1254,7 @@ impl Rooms { /// Returns an iterator over all invited members of a room. pub fn room_members_invited(&self, room_id: &RoomId) -> impl Iterator> { self.roomuserid_invited - .scan_prefix(room_id.to_string()) + .scan_prefix(room_id.as_bytes()) .keys() .map(|key| { Ok(UserId::try_from( @@ -1275,7 +1275,7 @@ impl Rooms { /// Returns an iterator over all rooms this user joined. pub fn rooms_joined(&self, user_id: &UserId) -> impl Iterator> { self.userroomid_joined - .scan_prefix(user_id.to_string()) + .scan_prefix(user_id.as_bytes()) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1296,7 +1296,7 @@ impl Rooms { /// Returns an iterator over all rooms a user was invited to. pub fn rooms_invited(&self, user_id: &UserId) -> impl Iterator> { self.userroomid_invited - .scan_prefix(&user_id.to_string()) + .scan_prefix(&user_id.as_bytes()) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1317,7 +1317,7 @@ impl Rooms { /// Returns an iterator over all rooms a user left. pub fn rooms_left(&self, user_id: &UserId) -> impl Iterator> { self.userroomid_left - .scan_prefix(&user_id.to_string()) + .scan_prefix(&user_id.as_bytes()) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1336,25 +1336,25 @@ impl Rooms { } pub fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); Ok(self.userroomid_joined.get(userroom_id)?.is_some()) } pub fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); Ok(self.userroomid_invited.get(userroom_id)?.is_some()) } pub fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); Ok(self.userroomid_left.get(userroom_id)?.is_some()) } From 3b40f3d60ef9839c72763169f6ae1daed7b04895 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Thu, 27 Aug 2020 16:10:20 -0400 Subject: [PATCH 16/51] Update state-res crate --- Cargo.lock | 2 +- src/database/rooms.rs | 73 +++++++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62b13d00..eebaddc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2070,7 +2070,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res?branch=spec-comp#17958665f6592af3ef478024fd1d75c384a30e7f" +source = "git+https://github.com/ruma/state-res?branch=spec-comp#394d26744a6586ccdc01838964bb27dab289eee5" dependencies = [ "itertools", "js_int", diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 66af7363..ee070b38 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -19,13 +19,12 @@ use ruma::{ EventId, Raw, RoomAliasId, RoomId, UserId, }; use sled::IVec; -use state_res::{event_auth, Requester, StateEvent, StateMap, StateStore}; +use state_res::{event_auth, Error as StateError, Requester, StateEvent, StateMap, StateStore}; use std::{ collections::{BTreeMap, HashMap}, convert::{TryFrom, TryInto}, mem, - result::Result as StdResult, }; /// The unique identifier of each state group. @@ -67,28 +66,32 @@ pub struct Rooms { } impl StateStore for Rooms { - fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> StdResult { + fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> state_res::Result { let pid = self .eventid_pduid .get(event_id.as_bytes()) - .map_err(|e| e.to_string())? - .ok_or_else(|| "PDU via room_id and event_id not found in the db.".to_owned())?; + .map_err(StateError::custom)? + .ok_or_else(|| { + StateError::NotFound("PDU via room_id and event_id not found in the db.".into()) + })?; serde_json::from_slice( &self .pduid_pdu .get(pid) - .map_err(|e| e.to_string())? - .ok_or_else(|| "PDU via pduid not found in db.".to_owned())?, + .map_err(StateError::custom)? + .ok_or_else(|| StateError::NotFound("PDU via pduid not found in db.".into()))?, ) - .map_err(|e| e.to_string()) + .map_err(Into::into) .and_then(|pdu: StateEvent| { // conduit's PDU's always contain a room_id but some // of ruma's do not so this must be an Option if pdu.room_id() == Some(room_id) { Ok(pdu) } else { - Err("Found PDU for incorrect room in db.".into()) + Err(StateError::NotFound( + "Found PDU for incorrect room in db.".into(), + )) } }) } @@ -732,27 +735,37 @@ impl Rooms { // Don't allow encryption events when it's disabled !globals.encryption_disabled() } - EventType::RoomMember => event_auth::is_membership_change_allowed( - // TODO this is a bit of a hack but not sure how to have a type - // declared in `state_res` crate easily convert to/from conduit::PduEvent - Requester { - prev_event_ids: prev_events.to_owned(), - room_id: &room_id, - content: &content, - state_key: Some(state_key.to_owned()), - sender: &sender, - }, - &auth_events - .iter() - .map(|((ty, key), pdu)| { - Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) - }) - .collect::>>()?, - ) - .map_err(|e| { - log::error!("{}", e); - Error::Conflict("Found incoming PDU with invalid data.") - })?, + EventType::RoomMember => { + let prev_event = self + .get_pdu(prev_events.iter().next().ok_or(Error::BadRequest( + ErrorKind::Unknown, + "Membership can't be the first event", + ))?)? + .map(|pdu| pdu.convert_for_state_res()) + .transpose()?; + event_auth::valid_membership_change( + // TODO this is a bit of a hack but not sure how to have a type + // declared in `state_res` crate easily convert to/from conduit::PduEvent + Requester { + prev_event_ids: prev_events.to_owned(), + room_id: &room_id, + content: &content, + state_key: Some(state_key.to_owned()), + sender: &sender, + }, + prev_event.as_ref(), + &auth_events + .iter() + .map(|((ty, key), pdu)| { + Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) + }) + .collect::>>()?, + ) + .map_err(|e| { + log::error!("{}", e); + Error::Conflict("Found incoming PDU with invalid data.") + })? + } EventType::RoomCreate => prev_events.is_empty(), // Not allow any of the following events if the sender is not joined. _ if sender_membership != member::MembershipState::Join => false, From 2a63d0955ab06d1d1cbcae8fdf078d02a49fa65b Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 30 Aug 2020 16:08:47 -0400 Subject: [PATCH 17/51] Sort and authenticate the events from /send_join response --- src/client_server/membership.rs | 49 +++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 90683e61..6d1931bf 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -497,17 +497,18 @@ async fn join_room_by_id_helper( .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - let power_events = event_map + let control_events = event_map .values() .filter(|pdu| pdu.is_power_event()) .map(|pdu| pdu.event_id()) .collect::>(); - // TODO these events are not guaranteed to be sorted but they are resolved, do - // we need the auth_chain - let sorted_power_events = state_res::StateResolution::reverse_topological_power_sort( + // These events are not guaranteed to be sorted but they are resolved according to spec + // we auth them anyways to weed out faulty/malicious server. The following is basically the + // full state resolution algorithm. + let sorted_control_events = state_res::StateResolution::reverse_topological_power_sort( &room_id, - &power_events, + &control_events, &mut event_map, &db.rooms, &send_join_response @@ -518,26 +519,31 @@ async fn join_room_by_id_helper( .collect::>(), ); - // TODO we may be able to skip this since they are resolved according to spec - let resolved_power = state_res::StateResolution::iterative_auth_check( + // Auth check each event against the "partial" state created by the preceding events + let resolved_control_events = state_res::StateResolution::iterative_auth_check( room_id, &RoomVersionId::Version6, - &sorted_power_events, - &BTreeMap::new(), // unconflicted events + &sorted_control_events, + &BTreeMap::new(), // We have no "clean/resolved" events to add (these extend the `resolved_control_events`) &mut event_map, &db.rooms, ) .expect("iterative auth check failed on resolved events"); - // TODO do we need to dedup them + // This removes the control events that failed auth, leaving the resolved + // to be mainline sorted let events_to_sort = event_map .keys() - .filter(|id| !sorted_power_events.contains(id)) + .filter(|id| { + !sorted_control_events.contains(id) + || resolved_control_events.values().any(|rid| *id == rid) + }) .cloned() .collect::>(); - let power_level = resolved_power.get(&(EventType::RoomPowerLevels, Some("".into()))); - + let power_level = + resolved_control_events.get(&(EventType::RoomPowerLevels, Some("".into()))); + // Sort the remaining non control events let sorted_event_ids = state_res::StateResolution::mainline_sort( room_id, &events_to_sort, @@ -546,7 +552,22 @@ async fn join_room_by_id_helper( &db.rooms, ); - for ev_id in &sorted_event_ids { + let resolved_events = state_res::StateResolution::iterative_auth_check( + room_id, + &RoomVersionId::Version6, + &sorted_event_ids, + &resolved_control_events, + &mut event_map, + &db.rooms, + ) + .expect("iterative auth check failed on resolved events"); + + // filter the events that failed the auth check keeping the remaining events + // sorted correctly + for ev_id in sorted_event_ids + .iter() + .filter(|id| resolved_events.values().any(|rid| rid == *id)) + { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` let pdu = event_map .get(ev_id) From 1e8fbd8d50c1bf0fe80cb77b8df8b5a45c3698f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 8 Sep 2020 17:32:03 +0200 Subject: [PATCH 18/51] Update ruma version --- Cargo.lock | 18 +--------- Cargo.toml | 9 ++--- src/client_server/account.rs | 10 +++--- src/client_server/alias.rs | 8 ++--- src/client_server/backup.rs | 10 +++--- src/client_server/config.rs | 4 +-- src/client_server/context.rs | 2 +- src/client_server/device.rs | 8 ++--- src/client_server/directory.rs | 14 ++++---- src/client_server/filter.rs | 2 +- src/client_server/keys.rs | 18 +++++----- src/client_server/media.rs | 8 ++--- src/client_server/membership.rs | 32 +++++++++--------- src/client_server/message.rs | 8 ++--- src/client_server/presence.rs | 2 +- src/client_server/profile.rs | 10 +++--- src/client_server/read_marker.rs | 4 +-- src/client_server/redact.rs | 2 +- src/client_server/room.rs | 51 ++++++++++------------------- src/client_server/search.rs | 22 ++++++++----- src/client_server/session.rs | 2 +- src/client_server/state.rs | 6 ++-- src/client_server/sync.rs | 2 +- src/client_server/tag.rs | 6 ++-- src/client_server/to_device.rs | 2 +- src/client_server/typing.rs | 8 ++--- src/client_server/user_directory.rs | 2 +- src/database/media.rs | 4 +-- src/database/users.rs | 6 ++-- src/error.rs | 6 ++-- src/pdu.rs | 2 +- src/ruma_wrapper.rs | 22 ++++++++----- src/server_server.rs | 6 ++-- 33 files changed, 147 insertions(+), 169 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eebaddc0..c88c5786 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1631,7 +1631,6 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1647,7 +1646,6 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "http", "percent-encoding", @@ -1662,7 +1660,6 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1673,7 +1670,6 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-api", "ruma-common", @@ -1686,7 +1682,6 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "assign", "http", @@ -1705,9 +1700,9 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", + "ruma-api", "ruma-identifiers", "ruma-serde", "serde", @@ -1718,7 +1713,6 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-common", @@ -1733,7 +1727,6 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1744,7 +1737,6 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-api", @@ -1759,7 +1751,6 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1771,7 +1762,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro2", "quote", @@ -1782,18 +1772,14 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ - "ruma-serde", "serde", - "serde_json", "strum", ] [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "form_urlencoded", "itoa", @@ -1805,7 +1791,6 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "base64", "ring", @@ -2070,7 +2055,6 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res?branch=spec-comp#394d26744a6586ccdc01838964bb27dab289eee5" dependencies = [ "itertools", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 15cee725..5d354333 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,9 @@ edition = "2018" # TODO: This can become optional as soon as proper configs are supported #rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } - -ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers +#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers +#ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fixes" } # Used for matrix spec type definitions and helpers +ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } tokio = "0.2.22" # Used for long polling sled = "0.32.0" # Used for storing data permanently log = "0.4.8" # Used for emitting log entries @@ -31,8 +32,8 @@ reqwest = "0.10.6" # Used to send requests thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key -# state-res = { path = "../../state-res" } -state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } +#state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } +state-res = { path = "../state-res" } ring = "0.16.15" [features] diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 9e52f6d2..cb77a15d 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -33,7 +33,7 @@ const GUEST_NAME_LENGTH: usize = 10; )] pub fn get_register_available_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { // Validate user id let user_id = UserId::parse_with_server_name(body.username.clone(), db.globals.server_name()) @@ -75,7 +75,7 @@ pub fn get_register_available_route( )] pub fn register_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { if db.globals.registration_disabled() { return Err(Error::BadRequest( @@ -84,7 +84,7 @@ pub fn register_route( )); } - let is_guest = matches!(body.kind, Some(RegistrationKind::Guest)); + let is_guest = body.kind == RegistrationKind::Guest; let mut missing_username = false; @@ -223,7 +223,7 @@ pub fn register_route( )] pub fn change_password_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -305,7 +305,7 @@ pub fn whoami_route(body: Ruma) -> ConduitResult, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 669f558d..1d30261b 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -20,7 +20,7 @@ use rocket::{delete, get, put}; )] pub fn create_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { if db.rooms.id_from_alias(&body.room_alias)?.is_some() { return Err(Error::Conflict("Alias already exists.")); @@ -38,7 +38,7 @@ pub fn create_alias_route( )] pub fn delete_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { db.rooms.set_alias(&body.room_alias, None, &db.globals)?; @@ -51,7 +51,7 @@ pub fn delete_alias_route( )] pub async fn get_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { get_alias_helper(&db, &body.room_alias).await } @@ -65,7 +65,7 @@ pub async fn get_alias_helper( &db, room_alias.server_name().to_string(), federation::query::get_room_information::v1::Request { - room_alias: room_alias.to_string(), + room_alias, }, ) .await?; diff --git a/src/client_server/backup.rs b/src/client_server/backup.rs index a1049649..8966c017 100644 --- a/src/client_server/backup.rs +++ b/src/client_server/backup.rs @@ -33,7 +33,7 @@ pub fn create_backup_route( )] pub fn update_backup_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); db.key_backups @@ -75,7 +75,7 @@ pub fn get_latest_backup_route( )] pub fn get_backup_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let algorithm = db @@ -90,7 +90,7 @@ pub fn get_backup_route( algorithm, count: (db.key_backups.count_keys(sender_id, &body.version)? as u32).into(), etag: db.key_backups.get_etag(sender_id, &body.version)?, - version: body.version.clone(), + version: body.version.to_owned(), } .into()) } @@ -102,7 +102,7 @@ pub fn get_backup_route( )] pub fn add_backup_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -132,7 +132,7 @@ pub fn add_backup_keys_route( )] pub fn get_backup_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/config.rs b/src/client_server/config.rs index baa9381b..515ad160 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -18,7 +18,7 @@ use rocket::{get, put}; )] pub fn set_global_account_data_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -49,7 +49,7 @@ pub fn set_global_account_data_route( )] pub fn get_global_account_data_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 7b1fad91..95937264 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -12,7 +12,7 @@ use rocket::get; )] pub fn get_context_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/device.rs b/src/client_server/device.rs index 89033f06..6352d0d1 100644 --- a/src/client_server/device.rs +++ b/src/client_server/device.rs @@ -37,7 +37,7 @@ pub fn get_devices_route( )] pub fn get_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,7 @@ pub fn get_device_route( )] pub fn update_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -80,7 +80,7 @@ pub fn update_device_route( )] pub fn delete_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -127,7 +127,7 @@ pub fn delete_device_route( )] pub fn delete_devices_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 3b106866..34feb718 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,7 +14,7 @@ use ruma::{ }, federation, }, - directory::{Filter, PublicRoomsChunk, RoomNetwork}, + directory::{IncomingFilter, PublicRoomsChunk, IncomingRoomNetwork}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, @@ -31,7 +31,7 @@ use rocket::{get, post, put}; )] pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let Ruma { body: @@ -61,7 +61,7 @@ pub async fn get_public_rooms_filtered_route( )] pub async fn get_public_rooms_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let response = get_public_rooms_filtered_helper( &db, @@ -89,7 +89,7 @@ pub async fn get_public_rooms_route( )] pub async fn set_room_visibility_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { match body.visibility { room::Visibility::Public => db.rooms.set_public(&body.room_id, true)?, @@ -105,7 +105,7 @@ pub async fn set_room_visibility_route( )] pub async fn get_room_visibility_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { Ok(get_room_visibility::Response { visibility: if db.rooms.is_public_room(&body.room_id)? { @@ -122,8 +122,8 @@ pub async fn get_public_rooms_filtered_helper( server: Option<&str>, limit: Option, since: Option<&str>, - _filter: Option, - _network: Option, + _filter: Option, + _network: Option, ) -> ConduitResult { if let Some(other_server) = server .clone() diff --git a/src/client_server/filter.rs b/src/client_server/filter.rs index 4322de3e..4b1c3a00 100644 --- a/src/client_server/filter.rs +++ b/src/client_server/filter.rs @@ -7,7 +7,7 @@ use rocket::{get, post}; #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/user/<_>/filter/<_>"))] pub fn get_filter_route() -> ConduitResult { // TODO - Ok(get_filter::Response::new(filter::FilterDefinition { + Ok(get_filter::Response::new(filter::IncomingFilterDefinition { event_fields: None, event_format: None, account_data: None, diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index 33115296..0e7b1eff 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -11,7 +11,7 @@ use ruma::{ uiaa::{AuthFlow, UiaaInfo}, }, }, - encryption::UnsignedDeviceInfo, + encryption::IncomingUnsignedDeviceInfo, }; use std::collections::{BTreeMap, HashSet}; @@ -24,7 +24,7 @@ use rocket::{get, post}; )] pub fn upload_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,7 @@ pub fn upload_keys_route( )] pub fn get_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -78,9 +78,9 @@ pub fn get_keys_route( Error::bad_database("all_device_keys contained nonexistent device.") })?; - keys.unsigned = Some(UnsignedDeviceInfo { + keys.unsigned = IncomingUnsignedDeviceInfo { device_display_name: metadata.display_name, - }); + }; container.insert(device_id, keys); } @@ -97,9 +97,9 @@ pub fn get_keys_route( ), )?; - keys.unsigned = Some(UnsignedDeviceInfo { + keys.unsigned = IncomingUnsignedDeviceInfo { device_display_name: metadata.display_name, - }); + }; container.insert(device_id.clone(), keys); } @@ -167,7 +167,7 @@ pub fn claim_keys_route( )] pub fn upload_signing_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -280,7 +280,7 @@ pub fn upload_signatures_route( )] pub fn get_key_changes_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 79c1f080..038012e2 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -27,7 +27,7 @@ pub fn get_media_config_route( )] pub fn create_content_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let mxc = format!( "mxc://{}/{}", @@ -36,7 +36,7 @@ pub fn create_content_route( ); db.media.create( mxc.clone(), - body.filename.as_ref(), + &body.filename, &body.content_type, &body.file, )?; @@ -53,7 +53,7 @@ pub fn create_content_route( )] pub fn get_content_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _server_name: String, _media_id: String, ) -> ConduitResult { @@ -85,7 +85,7 @@ pub fn get_content_route( )] pub fn get_content_thumbnail_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _server_name: String, _media_id: String, ) -> ConduitResult { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 6d1931bf..606e4700 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -32,7 +32,7 @@ use rocket::{get, post}; )] pub async fn join_room_by_id_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { join_room_by_id_helper( &db, @@ -49,7 +49,7 @@ pub async fn join_room_by_id_route( )] pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, @@ -81,7 +81,7 @@ pub async fn join_room_by_id_or_alias_route( )] pub fn leave_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -127,11 +127,11 @@ pub fn leave_room_route( )] pub fn invite_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - if let invite_user::InvitationRecipient::UserId { user_id } = &body.recipient { + if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), @@ -165,7 +165,7 @@ pub fn invite_user_route( )] pub fn kick_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -212,7 +212,7 @@ pub fn kick_user_route( )] pub fn ban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -267,7 +267,7 @@ pub fn ban_user_route( )] pub fn unban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -313,7 +313,7 @@ pub fn unban_user_route( )] pub fn forget_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -348,7 +348,7 @@ pub fn joined_rooms_route( )] pub fn get_member_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -376,7 +376,7 @@ pub fn get_member_events_route( )] pub fn joined_members_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -422,9 +422,9 @@ async fn join_room_by_id_helper( &db, room_id.server_name().to_string(), federation::membership::create_join_event_template::v1::Request { - room_id: room_id.clone(), - user_id: sender_id.clone(), - ver: vec![RoomVersionId::Version5, RoomVersionId::Version6], + room_id, + user_id: sender_id, + ver: &[RoomVersionId::Version5, RoomVersionId::Version6], }, ) .await?; @@ -474,8 +474,8 @@ async fn join_room_by_id_helper( &db, room_id.server_name().to_string(), federation::membership::create_join_event::v1::Request { - room_id: room_id.clone(), - event_id, + room_id, + event_id: &event_id, pdu_stub: serde_json::from_value(join_event_stub_value) .expect("we just created this event"), }, diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 03832d86..09c3517f 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -18,7 +18,7 @@ use rocket::{get, put}; )] pub fn send_message_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -53,7 +53,7 @@ pub fn send_message_event_route( )] pub fn get_message_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -96,7 +96,7 @@ pub fn get_message_events_route( .collect::>(); let mut resp = get_message_events::Response::new(); - resp.start = Some(body.from.clone()); + resp.start = Some(body.from.to_owned()); resp.end = end_token; resp.chunk = events_after; resp.state = Vec::new(); @@ -120,7 +120,7 @@ pub fn get_message_events_route( .collect::>(); let mut resp = get_message_events::Response::new(); - resp.start = Some(body.from.clone()); + resp.start = Some(body.from.to_owned()); resp.end = start_token; resp.chunk = events_before; resp.state = Vec::new(); diff --git a/src/client_server/presence.rs b/src/client_server/presence.rs index 0b6a51f4..d105eb6a 100644 --- a/src/client_server/presence.rs +++ b/src/client_server/presence.rs @@ -12,7 +12,7 @@ use rocket::put; )] pub fn set_presence_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 0707b342..386d8988 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -21,7 +21,7 @@ use std::convert::TryInto; )] pub fn set_displayname_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -98,7 +98,7 @@ pub fn set_displayname_route( )] pub fn get_displayname_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { Ok(get_display_name::Response { displayname: db.users.displayname(&body.user_id)?, @@ -112,7 +112,7 @@ pub fn get_displayname_route( )] pub fn set_avatar_url_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -201,7 +201,7 @@ pub fn set_avatar_url_route( )] pub fn get_avatar_url_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { Ok(get_avatar_url::Response { avatar_url: db.users.avatar_url(&body.user_id)?, @@ -215,7 +215,7 @@ pub fn get_avatar_url_route( )] pub fn get_profile_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let avatar_url = db.users.avatar_url(&body.user_id)?; let displayname = db.users.displayname(&body.user_id)?; diff --git a/src/client_server/read_marker.rs b/src/client_server/read_marker.rs index ff72765f..023eece2 100644 --- a/src/client_server/read_marker.rs +++ b/src/client_server/read_marker.rs @@ -15,7 +15,7 @@ use std::{collections::BTreeMap, time::SystemTime}; )] pub fn set_read_marker_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -52,7 +52,7 @@ pub fn set_read_marker_route( ); let mut receipt_content = BTreeMap::new(); receipt_content.insert( - event.clone(), + event.to_owned(), ruma::events::receipt::Receipts { read: Some(user_receipts), }, diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 8708692f..cd1b4438 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -14,7 +14,7 @@ use rocket::put; )] pub fn redact_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 3ee21b6d..9918123f 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -22,7 +22,7 @@ use rocket::{get, post}; )] pub fn create_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -48,11 +48,8 @@ pub fn create_room_route( })?; let mut content = ruma::events::room::create::CreateEventContent::new(sender_id.clone()); - content.federate = body.creation_content.as_ref().map_or(true, |c| c.federate); - content.predecessor = body - .creation_content - .as_ref() - .and_then(|c| c.predecessor.clone()); + content.federate = body.creation_content.federate; + content.predecessor = body.creation_content.predecessor.clone(); content.room_version = RoomVersionId::Version6; // 1. The room create event @@ -80,7 +77,7 @@ pub fn create_room_route( membership: member::MembershipState::Join, displayname: db.users.displayname(&sender_id)?, avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: body.is_direct, + is_direct: Some(body.is_direct), third_party_invite: None, }) .expect("event is valid, we just created it"), @@ -137,8 +134,7 @@ pub fn create_room_route( // 4. Events set by preset // Figure out preset. We need it for preset specific events - let visibility = body.visibility.unwrap_or(room::Visibility::Private); - let preset = body.preset.unwrap_or_else(|| match visibility { + let preset = body.preset.unwrap_or_else(|| match body.visibility { room::Visibility::Private => create_room::RoomPreset::PrivateChat, room::Visibility::Public => create_room::RoomPreset::PublicChat, }); @@ -213,32 +209,19 @@ pub fn create_room_route( )?; // 5. Events listed in initial_state - for create_room::InitialStateEvent { - event_type, - state_key, - content, - } in &body.initial_state - { + for event in &body.initial_state { + let pdu_builder = serde_json::from_str::( + &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"), + ).map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; + // Silently skip encryption events if they are not allowed - if event_type == &EventType::RoomEncryption && db.globals.encryption_disabled() { + if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() + { continue; } - db.rooms.build_and_append_pdu( - PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), - event_type: event_type.clone(), - content: serde_json::from_str(content.get()).map_err(|_| { - Error::BadRequest(ErrorKind::BadJson, "Invalid initial_state content.") - })?, - unsigned: None, - state_key: state_key.clone(), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu(pdu_builder, &db.globals, &db.account_data)?; } // 6. Events implied by name and topic @@ -293,7 +276,7 @@ pub fn create_room_route( membership: member::MembershipState::Invite, displayname: db.users.displayname(&user)?, avatar_url: db.users.avatar_url(&user)?, - is_direct: body.is_direct, + is_direct: Some(body.is_direct), third_party_invite: None, }) .expect("event is valid, we just created it"), @@ -311,7 +294,7 @@ pub fn create_room_route( db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?; } - if let Some(room::Visibility::Public) = body.visibility { + if body.visibility == room::Visibility::Public { db.rooms.set_public(&room_id, true)?; } @@ -324,7 +307,7 @@ pub fn create_room_route( )] pub fn get_room_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/search.rs b/src/client_server/search.rs index 082711d1..2967e001 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -1,11 +1,10 @@ use super::State; use crate::{ConduitResult, Database, Error, Ruma}; -use js_int::uint; use ruma::api::client::{error::ErrorKind, r0::search::search_events}; #[cfg(feature = "conduit_bin")] use rocket::post; -use search_events::{ResultCategories, ResultRoomEvents, SearchResult}; +use search_events::{EventContextResult, ResultCategories, ResultRoomEvents, SearchResult}; use std::collections::BTreeMap; #[cfg_attr( @@ -14,7 +13,7 @@ use std::collections::BTreeMap; )] pub fn search_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -51,13 +50,18 @@ pub fn search_events_route( .0 .map(|result| { Ok::<_, Error>(SearchResult { - context: None, + context: EventContextResult { + end: None, + events_after: Vec::new(), + events_before: Vec::new(), + profile_info: BTreeMap::new(), + start: None, + }, rank: None, result: db .rooms .get_pdu_from_id(&result)? - // TODO this is an awkward type conversion see method - .map(|pdu| pdu.to_any_event()), + .map(|pdu| pdu.to_room_event()), }) }) .filter_map(|r| r.ok()) @@ -72,14 +76,14 @@ pub fn search_events_route( }; Ok(search_events::Response::new(ResultCategories { - room_events: Some(ResultRoomEvents { - count: uint!(0), // TODO + room_events: ResultRoomEvents { + count: None, // TODO? maybe not groups: BTreeMap::new(), // TODO next_batch, results, state: BTreeMap::new(), // TODO highlights: search.1, - }), + }, }) .into()) } diff --git a/src/client_server/session.rs b/src/client_server/session.rs index 948b455b..9cd051ce 100644 --- a/src/client_server/session.rs +++ b/src/client_server/session.rs @@ -36,7 +36,7 @@ pub fn get_login_types_route() -> ConduitResult { )] pub fn login_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { // Validate login method let user_id = diff --git a/src/client_server/state.rs b/src/client_server/state.rs index e7d2bcf3..75463cba 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -21,7 +21,7 @@ use rocket::{get, put}; )] pub fn send_state_event_for_key_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -40,7 +40,7 @@ pub fn send_state_event_for_key_route( &body.content, content, &body.room_id, - Some(body.state_key.clone()), + Some(body.state_key.to_owned()), )?) .into(), ) @@ -52,7 +52,7 @@ pub fn send_state_event_for_key_route( )] pub fn send_state_event_for_empty_key_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { // This just calls send_state_event_for_key_route let Ruma { diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 74329605..167ee75e 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -31,7 +31,7 @@ use std::{ )] pub async fn sync_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/tag.rs b/src/client_server/tag.rs index 99ee6e34..d04dd3ae 100644 --- a/src/client_server/tag.rs +++ b/src/client_server/tag.rs @@ -15,7 +15,7 @@ use rocket::{delete, get, put}; )] pub fn update_tag_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -49,7 +49,7 @@ pub fn update_tag_route( )] pub fn delete_tag_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -80,7 +80,7 @@ pub fn delete_tag_route( )] pub fn get_tags_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/to_device.rs b/src/client_server/to_device.rs index ca423fef..fe741011 100644 --- a/src/client_server/to_device.rs +++ b/src/client_server/to_device.rs @@ -14,7 +14,7 @@ use rocket::put; )] pub fn send_event_to_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/typing.rs b/src/client_server/typing.rs index 7eba13e1..b15121c0 100644 --- a/src/client_server/typing.rs +++ b/src/client_server/typing.rs @@ -1,5 +1,6 @@ use super::State; use crate::{utils, ConduitResult, Database, Ruma}; +use create_typing_event::Typing; use ruma::api::client::r0::typing::create_typing_event; #[cfg(feature = "conduit_bin")] @@ -11,16 +12,15 @@ use rocket::put; )] pub fn create_typing_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - if body.typing { + if let Typing::Yes(duration) = body.state { db.rooms.edus.roomactive_add( &sender_id, &body.room_id, - body.timeout.map(|d| d.as_millis() as u64).unwrap_or(30000) - + utils::millis_since_unix_epoch(), + duration.as_millis() as u64 + utils::millis_since_unix_epoch(), &db.globals, )?; } else { diff --git a/src/client_server/user_directory.rs b/src/client_server/user_directory.rs index f47643c1..dcf48fe3 100644 --- a/src/client_server/user_directory.rs +++ b/src/client_server/user_directory.rs @@ -11,7 +11,7 @@ use rocket::post; )] pub fn search_users_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let limit = u64::from(body.limit) as usize; diff --git a/src/database/media.rs b/src/database/media.rs index e9dcb4a0..63fa11c6 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -16,7 +16,7 @@ impl Media { pub fn create( &self, mxc: String, - filename: Option<&String>, + filename: &Option, content_type: &str, file: &[u8], ) -> Result<()> { @@ -25,7 +25,7 @@ impl Media { key.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail key.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail key.push(0xff); - key.extend_from_slice(filename.map(|f| f.as_bytes()).unwrap_or_default()); + key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default()); key.push(0xff); key.extend_from_slice(content_type.as_bytes()); diff --git a/src/database/users.rs b/src/database/users.rs index 1b6a6812..10e1ef30 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -8,7 +8,7 @@ use ruma::{ keys::{CrossSigningKey, OneTimeKey}, }, }, - encryption::DeviceKeys, + encryption::IncomingDeviceKeys, events::{AnyToDeviceEvent, EventType}, DeviceId, DeviceKeyAlgorithm, DeviceKeyId, Raw, UserId, }; @@ -395,7 +395,7 @@ impl Users { &self, user_id: &UserId, device_id: &DeviceId, - device_keys: &DeviceKeys, + device_keys: &IncomingDeviceKeys, rooms: &super::rooms::Rooms, globals: &super::globals::Globals, ) -> Result<()> { @@ -625,7 +625,7 @@ impl Users { &self, user_id: &UserId, device_id: &DeviceId, - ) -> Result> { + ) -> Result> { let mut key = user_id.to_string().as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(device_id.as_bytes()); diff --git a/src/error.rs b/src/error.rs index 623aa0ef..f521da43 100644 --- a/src/error.rs +++ b/src/error.rs @@ -70,14 +70,14 @@ where use ErrorKind::*; let (kind, status_code) = match self { Self::BadRequest(kind, _) => ( - kind, + kind.clone(), match kind { Forbidden | GuestAccessForbidden | ThreepidAuthFailed | ThreepidDenied => { StatusCode::FORBIDDEN } - Unauthorized | UnknownToken | MissingToken => StatusCode::UNAUTHORIZED, + Unauthorized | UnknownToken { .. } | MissingToken => StatusCode::UNAUTHORIZED, NotFound => StatusCode::NOT_FOUND, - LimitExceeded => StatusCode::TOO_MANY_REQUESTS, + LimitExceeded { .. } => StatusCode::TOO_MANY_REQUESTS, UserDeactivated => StatusCode::FORBIDDEN, TooLarge => StatusCode::PAYLOAD_TOO_LARGE, _ => StatusCode::BAD_REQUEST, diff --git a/src/pdu.rs b/src/pdu.rs index b565a24c..15264842 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -255,7 +255,7 @@ impl PduEvent { } /// Build the start of a PDU in order to add it to the `Database`. -#[derive(Debug)] +#[derive(Debug, Deserialize)] pub struct PduBuilder { pub room_id: RoomId, pub sender: UserId, diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 80e6e582..734d2144 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -1,9 +1,9 @@ use crate::Error; use ruma::{ - api::IncomingRequest, + api::{Outgoing, OutgoingRequest}, identifiers::{DeviceId, UserId}, }; -use std::{convert::TryInto, ops::Deref}; +use std::{convert::TryFrom, convert::TryInto, ops::Deref}; #[cfg(feature = "conduit_bin")] use { @@ -24,15 +24,21 @@ use { /// This struct converts rocket requests into ruma structs by converting them into http requests /// first. -pub struct Ruma { - pub body: T, +pub struct Ruma { + pub body: T::Incoming, pub sender_id: Option, pub device_id: Option>, pub json_body: Option>, // This is None when body is not a valid string } #[cfg(feature = "conduit_bin")] -impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { +impl<'a, T: Outgoing + OutgoingRequest> FromTransformedData<'a> for Ruma +where + ::Incoming: TryFrom>> + std::fmt::Debug, + <::Incoming as std::convert::TryFrom< + http::request::Request>, + >>::Error: std::fmt::Debug, +{ type Error = (); // TODO: Better error handling type Owned = Data; type Borrowed = Self::Owned; @@ -93,7 +99,7 @@ impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { let http_request = http_request.body(body.clone()).unwrap(); log::info!("{:?}", http_request); - match T::try_from(http_request) { + match ::Incoming::try_from(http_request) { Ok(t) => Success(Ruma { body: t, sender_id: user_id, @@ -112,8 +118,8 @@ impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { } } -impl Deref for Ruma { - type Target = T; +impl Deref for Ruma { + type Target = T::Incoming; fn deref(&self) -> &Self::Target { &self.body diff --git a/src/server_server.rs b/src/server_server.rs index ac4407ba..d39abe64 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -207,7 +207,7 @@ pub fn get_server_keys_deprecated(db: State<'_, Database>) -> Json { )] pub async fn get_public_rooms_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let Ruma { body: @@ -230,7 +230,7 @@ pub async fn get_public_rooms_route( limit, since.as_deref(), None, - Some(ruma::directory::RoomNetwork::Matrix), + Some(ruma::directory::IncomingRoomNetwork::Matrix), ) .await? .0; @@ -264,7 +264,7 @@ pub async fn get_public_rooms_route( )] pub fn send_transaction_message_route<'a>( _db: State<'a, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { dbg!(&*body); Ok(send_transaction_message::v1::Response { From 12a8c9badd202f3019cab71a1f3a3134c8fb75ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 12 Sep 2020 21:30:07 +0200 Subject: [PATCH 19/51] fix: join rooms over federation --- Cargo.lock | 140 +++++++------ Cargo.toml | 8 +- src/client_server/account.rs | 4 +- src/client_server/alias.rs | 4 +- src/client_server/directory.rs | 2 +- src/client_server/media.rs | 8 +- src/client_server/membership.rs | 106 +++++----- src/client_server/message.rs | 5 +- src/client_server/profile.rs | 8 +- src/client_server/redact.rs | 4 +- src/client_server/room.rs | 51 ++--- src/client_server/search.rs | 2 +- src/client_server/state.rs | 4 +- src/database.rs | 1 - src/database/rooms.rs | 342 ++++++++++++++------------------ src/pdu.rs | 112 ++++++----- src/server_server.rs | 1 + 17 files changed, 396 insertions(+), 406 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c88c5786..dc215c39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,9 +116,9 @@ checksum = "4af5687fe33aec5e70ef14caac5e0d363e335e5e5d6385fb75978d0c241b1d67" [[package]] name = "async-trait" -version = "0.1.38" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1a4a2f97ce50c9d0282c1468816208588441492b40d813b2e0419c22c05e7f" +checksum = "687c230d85c0a52504709705fc8a53e4a692b83a2184f03dae73e38e1e93a783" dependencies = [ "proc-macro2", "quote", @@ -223,9 +223,9 @@ checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" [[package]] name = "bytemuck" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7a1029718df60331e557c9e83a55523c955e5dd2a7bfeffad6bbd50b538ae9" +checksum = "41aa2ec95ca3b5c54cf73c91acf06d24f4495d5f1b1c12506ae3483d646177ac" [[package]] name = "byteorder" @@ -301,6 +301,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "const_fn" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -319,7 +325,7 @@ dependencies = [ "percent-encoding", "rand", "sha2", - "time 0.2.16", + "time 0.2.19", "version_check", ] @@ -660,9 +666,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ "cfg-if", "libc", @@ -721,12 +727,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.8.2" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b62f79061a0bc2e046024cb7ba44b08419ed238ecbd9adbd787434b9e8c25" -dependencies = [ - "autocfg", -] +checksum = "00d63df3d41950fb462ed38308eea019113ad1508da725bbedcd0fa5a85ef5f7" [[package]] name = "heck" @@ -843,9 +846,9 @@ dependencies = [ [[package]] name = "image" -version = "0.23.8" +version = "0.23.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "543904170510c1b5fb65140485d84de4a57fddb2ed685481e9020ce3d2c9f64c" +checksum = "974e194911d1f7efe3cd8a8f9db3b767e43536327e899e8bc9a12ef5711b74d2" dependencies = [ "bytemuck", "byteorder", @@ -859,9 +862,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b45e59b16c76b11bf9738fd5d38879d3bd28ad292d7b313608becb17ae2df9" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" dependencies = [ "autocfg", "hashbrown", @@ -920,9 +923,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.44" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a7e2c92a4804dd459b86c339278d0fe87cf93757fae222c3fa3ae75458bc73" +checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" dependencies = [ "wasm-bindgen", ] @@ -954,9 +957,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.76" +version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3" +checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" [[package]] name = "lock_api" @@ -1120,9 +1123,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.34" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" dependencies = [ "cfg-if", "libc", @@ -1379,9 +1382,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.19" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" +checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c" dependencies = [ "unicode-xid", ] @@ -1587,7 +1590,7 @@ dependencies = [ "rocket_codegen", "rocket_http", "state", - "time 0.2.16", + "time 0.2.19", "tokio", "toml", "version_check", @@ -1622,7 +1625,7 @@ dependencies = [ "ref-cast", "smallvec", "state", - "time 0.2.16", + "time 0.2.19", "tokio", "tokio-rustls", "unicode-xid", @@ -1631,6 +1634,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1646,6 +1650,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "http", "percent-encoding", @@ -1660,6 +1665,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1670,6 +1676,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "ruma-api", "ruma-common", @@ -1682,6 +1689,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "assign", "http", @@ -1700,6 +1708,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "js_int", "ruma-api", @@ -1713,6 +1722,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "js_int", "ruma-common", @@ -1727,6 +1737,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1737,6 +1748,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "js_int", "ruma-api", @@ -1751,6 +1763,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1762,6 +1775,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "proc-macro2", "quote", @@ -1772,6 +1786,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "serde", "strum", @@ -1780,6 +1795,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "form_urlencoded", "itoa", @@ -1791,6 +1807,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "base64", "ring", @@ -1910,18 +1927,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.115" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54c9a88f2da7238af84b5101443f0c0d0a3bbdc455e34a5c9497b1903ed55d5" +checksum = "96fe57af81d28386a513cbc6858332abc6117cfdb5999647c6444b8f43a370a5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.115" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609feed1d0a73cc36a0182a840a9b37b4a82f0b1150369f0536a9e3f2a31dc48" +checksum = "f630a6370fd8e457873b4bd2ffdae75408bc291ba72be773772a4c2a065d9ae8" dependencies = [ "proc-macro2", "quote", @@ -2021,9 +2038,9 @@ checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" [[package]] name = "socket2" -version = "0.3.12" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" dependencies = [ "cfg-if", "libc", @@ -2055,6 +2072,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#0081081604b051d412a2365b68357e064c33320c" dependencies = [ "itertools", "js_int", @@ -2139,15 +2157,15 @@ dependencies = [ [[package]] name = "subtle" -version = "2.2.3" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" +checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd" [[package]] name = "syn" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9" +checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350" dependencies = [ "proc-macro2", "quote", @@ -2210,11 +2228,11 @@ dependencies = [ [[package]] name = "time" -version = "0.2.16" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a51cadc5b1eec673a685ff7c33192ff7b7603d0b75446fb354939ee615acb15" +checksum = "80c1a1fd93112fc50b11c43a1def21f926be3c18884fad676ea879572da070a1" dependencies = [ - "cfg-if", + "const_fn", "libc", "standback", "stdweb", @@ -2288,9 +2306,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "228139ddd4fea3fa345a29233009635235833e52807af7ea6448ead03890d6a9" +checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", "rustls", @@ -2362,9 +2380,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f0e00789804e99b20f12bc7003ca416309d28a6f495d6af58d1e2c2842461b5" +checksum = "5bcf46c1f1f06aeea2d6b81f3c863d0930a596c86ad1920d4e5bad6dd1d7119a" dependencies = [ "lazy_static", ] @@ -2382,9 +2400,9 @@ dependencies = [ [[package]] name = "tracing-serde" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6ccba2f8f16e0ed268fc765d9b7ff22e965e7185d32f8f1ec8294fe17d86e79" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" dependencies = [ "serde", "tracing-core", @@ -2392,9 +2410,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abd165311cc4d7a555ad11cc77a37756df836182db0d81aac908c8184c584f40" +checksum = "82bb5079aa76438620837198db8a5c529fb9878c730bc2b28179b0241cf04c10" dependencies = [ "ansi_term", "chrono", @@ -2525,9 +2543,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0563a9a4b071746dd5aedbc3a28c6fe9be4586fb3fbadb67c400d4f53c6b16c" +checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" dependencies = [ "cfg-if", "serde", @@ -2537,9 +2555,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc71e4c5efa60fb9e74160e89b93353bc24059999c0ae0fb03affc39770310b0" +checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" dependencies = [ "bumpalo", "lazy_static", @@ -2552,9 +2570,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95f8d235a77f880bcef268d379810ea6c0af2eacfa90b1ad5af731776e0c4699" +checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" dependencies = [ "cfg-if", "js-sys", @@ -2564,9 +2582,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97c57cefa5fa80e2ba15641578b44d36e7a64279bc5ed43c6dbaf329457a2ed2" +checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2574,9 +2592,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841a6d1c35c6f596ccea1f82504a192a60378f64b3bb0261904ad8f2f5657556" +checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" dependencies = [ "proc-macro2", "quote", @@ -2587,15 +2605,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93b162580e34310e5931c4b792560108b10fd14d64915d7fff8ff00180e70092" +checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" [[package]] name = "web-sys" -version = "0.3.44" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda38f4e5ca63eda02c059d243aa25b5f35ab98451e518c51612cd0f1bd19a47" +checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 5d354333..1b7a7007 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,8 +16,8 @@ edition = "2018" #rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } #ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers -#ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fixes" } # Used for matrix spec type definitions and helpers -ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } +ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fed-fixes" } # Used for matrix spec type definitions and helpers +#ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } tokio = "0.2.22" # Used for long polling sled = "0.32.0" # Used for storing data permanently log = "0.4.8" # Used for emitting log entries @@ -32,8 +32,8 @@ reqwest = "0.10.6" # Used to send requests thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key -#state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } -state-res = { path = "../state-res" } +state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] } +#state-res = { path = "../state-res", features = ["unstable-pre-spec"] } ring = "0.16.15" [features] diff --git a/src/client_server/account.rs b/src/client_server/account.rs index cb77a15d..3db933c1 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -356,14 +356,14 @@ pub fn deactivate_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 1d30261b..bfdaecac 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -64,9 +64,7 @@ pub async fn get_alias_helper( let response = server_server::send_request( &db, room_alias.server_name().to_string(), - federation::query::get_room_information::v1::Request { - room_alias, - }, + federation::query::get_room_information::v1::Request { room_alias }, ) .await?; diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 34feb718..2764d2c5 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,7 +14,7 @@ use ruma::{ }, federation, }, - directory::{IncomingFilter, PublicRoomsChunk, IncomingRoomNetwork}, + directory::{IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 038012e2..d0774472 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -34,12 +34,8 @@ pub fn create_content_route( db.globals.server_name(), utils::random_string(MXC_LENGTH) ); - db.media.create( - mxc.clone(), - &body.filename, - &body.content_type, - &body.file, - )?; + db.media + .create(mxc.clone(), &body.filename, &body.content_type, &body.file)?; Ok(create_content::Response { content_uri: mxc }.into()) } diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 606e4700..ea2271bb 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -4,6 +4,7 @@ use crate::{ pdu::{PduBuilder, PduEvent}, server_server, utils, ConduitResult, Database, Error, Ruma, }; +use log::warn; use ruma::{ api::{ client::{ @@ -20,8 +21,7 @@ use ruma::{ EventId, Raw, RoomId, RoomVersionId, UserId, }; use state_res::StateEvent; - -use std::{collections::BTreeMap, convert::TryFrom}; +use std::{collections::BTreeMap, convert::TryFrom, sync::Arc}; #[cfg(feature = "conduit_bin")] use rocket::{get, post}; @@ -106,14 +106,14 @@ pub fn leave_room_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -134,8 +134,6 @@ pub fn invite_user_route( if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Invite, @@ -149,6 +147,8 @@ pub fn invite_user_route( state_key: Some(user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -191,14 +191,14 @@ pub fn kick_user_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -246,14 +246,14 @@ pub fn ban_user_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -292,14 +292,14 @@ pub fn unban_user_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -473,7 +473,7 @@ async fn join_room_by_id_helper( let send_join_response = server_server::send_request( &db, room_id.server_name().to_string(), - federation::membership::create_join_event::v1::Request { + federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, pdu_stub: serde_json::from_value(join_event_stub_value) @@ -482,25 +482,39 @@ async fn join_room_by_id_helper( ) .await?; - dbg!(&send_join_response); - let mut event_map = send_join_response .room_state .state .iter() .chain(send_join_response.room_state.auth_chain.iter()) .map(|pdu| { - pdu.deserialize() - .map(StateEvent::Full) - .map(|ev| (ev.event_id(), ev)) + let mut value = serde_json::from_str(pdu.json().get()) + .expect("converting raw jsons to values always works"); + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&value) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + value + .as_object_mut() + .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? + .insert("event_id".to_owned(), event_id.to_string().into()); + + serde_json::from_value::(value) + .map(|ev| (event_id, Arc::new(ev))) + .map_err(|e| { + warn!("{}", e); + Error::BadServerResponse("Invalid PDU bytes in send_join response.") + }) }) - .collect::, _>>() - .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; + .collect::>, _>>()?; let control_events = event_map .values() .filter(|pdu| pdu.is_power_event()) - .map(|pdu| pdu.event_id()) + .map(|pdu| pdu.event_id().clone()) .collect::>(); // These events are not guaranteed to be sorted but they are resolved according to spec @@ -515,7 +529,9 @@ async fn join_room_by_id_helper( .room_state .auth_chain .iter() - .filter_map(|pdu| Some(StateEvent::Full(pdu.deserialize().ok()?).event_id())) + .filter_map(|pdu| { + Some(StateEvent::Full(pdu.deserialize().ok()?).event_id().clone()) + }) .collect::>(), ); @@ -575,31 +591,31 @@ async fn join_room_by_id_helper( // We do not rebuild the PDU in this case only insert to DB db.rooms - .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; + .append_pdu(PduEvent::from(&**pdu), &db.globals, &db.account_data)?; } - } - - let event = member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }; + } else { + let event = member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }; - db.rooms.build_and_append_pdu( - PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.account_data, + )?; + } Ok(join_room_by_id::Response::new(room_id.clone()).into()) } diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 09c3517f..025331e6 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -27,11 +27,10 @@ pub fn send_message_event_route( let event_id = db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: body.content.event_type().into(), content: serde_json::from_str( body.json_body + .as_ref() .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? .get(), ) @@ -40,6 +39,8 @@ pub fn send_message_event_route( state_key: None, redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 386d8988..2a2e05ef 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -33,8 +33,6 @@ pub fn set_displayname_route( let room_id = room_id?; db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(ruma::events::room::member::MemberEventContent { displayname: body.displayname.clone(), @@ -62,6 +60,8 @@ pub fn set_displayname_route( state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -136,8 +136,6 @@ pub fn set_avatar_url_route( let room_id = room_id?; db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(ruma::events::room::member::MemberEventContent { avatar_url: body.avatar_url.clone(), @@ -165,6 +163,8 @@ pub fn set_avatar_url_route( state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index cd1b4438..51173489 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -20,8 +20,6 @@ pub fn redact_event_route( let event_id = db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomRedaction, content: serde_json::to_value(redaction::RedactionEventContent { reason: body.reason.clone(), @@ -31,6 +29,8 @@ pub fn redact_event_route( state_key: None, redacts: Some(body.event_id.clone()), }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 9918123f..9a83f81b 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -55,14 +55,14 @@ pub fn create_room_route( // 1. The room create event db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomCreate, content: serde_json::to_value(content).expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -70,8 +70,6 @@ pub fn create_room_route( // 2. Let the room creator join db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Join, @@ -85,6 +83,8 @@ pub fn create_room_route( state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -119,14 +119,14 @@ pub fn create_room_route( }; db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomPowerLevels, content: power_levels_content, unsigned: None, state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -142,8 +142,6 @@ pub fn create_room_route( // 4.1 Join Rules db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomJoinRules, content: match preset { create_room::RoomPreset::PublicChat => serde_json::to_value( @@ -160,6 +158,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -167,8 +167,6 @@ pub fn create_room_route( // 4.2 History Visibility db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomHistoryVisibility, content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new( history_visibility::HistoryVisibility::Shared, @@ -178,6 +176,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -185,8 +185,6 @@ pub fn create_room_route( // 4.3 Guest Access db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomGuestAccess, content: match preset { create_room::RoomPreset::PublicChat => { @@ -204,6 +202,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -212,24 +212,27 @@ pub fn create_room_route( for event in &body.initial_state { let pdu_builder = serde_json::from_str::( &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"), - ).map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; // Silently skip encryption events if they are not allowed - if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() - { + if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() { continue; } - db.rooms - .build_and_append_pdu(pdu_builder, &db.globals, &db.account_data)?; + db.rooms.build_and_append_pdu( + pdu_builder, + &sender_id, + &room_id, + &db.globals, + &db.account_data, + )?; } // 6. Events implied by name and topic if let Some(name) = &body.name { db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomName, content: serde_json::to_value( name::NameEventContent::new(name.clone()).map_err(|_| { @@ -241,6 +244,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -249,8 +254,6 @@ pub fn create_room_route( if let Some(topic) = &body.topic { db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomTopic, content: serde_json::to_value(topic::TopicEventContent { topic: topic.clone(), @@ -260,6 +263,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -269,8 +274,6 @@ pub fn create_room_route( for user in &body.invite { db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Invite, @@ -284,6 +287,8 @@ pub fn create_room_route( state_key: Some(user.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/search.rs b/src/client_server/search.rs index 2967e001..3b03e7ac 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -77,7 +77,7 @@ pub fn search_events_route( Ok(search_events::Response::new(ResultCategories { room_events: ResultRoomEvents { - count: None, // TODO? maybe not + count: None, // TODO? maybe not groups: BTreeMap::new(), // TODO next_batch, results, diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 75463cba..1fe3cd6c 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -213,14 +213,14 @@ pub fn send_state_event_for_key_helper( let event_id = db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: content.event_type().into(), content: json, unsigned: None, state_key, redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/database.rs b/src/database.rs index a105058c..0d18020b 100644 --- a/src/database.rs +++ b/src/database.rs @@ -97,7 +97,6 @@ impl Database { }, pduid_pdu: db.open_tree("pduid_pdu")?, eventid_pduid: db.open_tree("eventid_pduid")?, - roomstateid_pduid: db.open_tree("roomstateid_pduid")?, roomid_pduleaves: db.open_tree("roomid_pduleaves")?, alias_roomid: db.open_tree("alias_roomid")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index ee070b38..60333782 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -25,6 +25,7 @@ use std::{ collections::{BTreeMap, HashMap}, convert::{TryFrom, TryInto}, mem, + sync::Arc, }; /// The unique identifier of each state group. @@ -33,10 +34,6 @@ use std::{ /// hashing the entire state. pub type StateHashId = Vec; -/// This identifier consists of roomId + count. It represents a -/// unique event, it will never be overwritten or removed. -pub type PduId = IVec; - pub struct Rooms { pub edus: edus::RoomEdus, pub(super) pduid_pdu: sled::Tree, // PduId = RoomId + Count @@ -54,22 +51,22 @@ pub struct Rooms { pub(super) roomuserid_invited: sled::Tree, pub(super) userroomid_left: sled::Tree, - // STATE TREES - /// This holds the full current state, including the latest event. - pub(super) roomstateid_pduid: sled::Tree, // RoomStateId = Room + StateType + StateKey - /// This holds the full room state minus the latest event. - pub(super) pduid_statehash: sled::Tree, // PDU id -> StateHash - /// Also holds the full room state minus the latest event. - pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + (EventType, StateKey) - /// The room_id -> the latest StateHash + /// Remember the current state hash of a room. pub(super) roomid_statehash: sled::Tree, + /// Remember the state hash at events in the past. + pub(super) pduid_statehash: sled::Tree, + /// The state for a given state hash. + pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + EventType + StateKey } impl StateStore for Rooms { - fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> state_res::Result { + fn get_event( + &self, + room_id: &RoomId, + event_id: &EventId, + ) -> state_res::Result> { let pid = self - .eventid_pduid - .get(event_id.as_bytes()) + .get_pdu_id(event_id) .map_err(StateError::custom)? .ok_or_else(|| { StateError::NotFound("PDU via room_id and event_id not found in the db.".into()) @@ -87,7 +84,7 @@ impl StateStore for Rooms { // conduit's PDU's always contain a room_id but some // of ruma's do not so this must be an Option if pdu.room_id() == Some(room_id) { - Ok(pdu) + Ok(Arc::new(pdu)) } else { Err(StateError::NotFound( "Found PDU for incorrect room in db.".into(), @@ -136,53 +133,12 @@ impl Rooms { None } - /// Fetch the current State using the `roomstateid_pduid` tree. - pub fn current_state_pduids(&self, room_id: &RoomId) -> Result> { - // TODO this could also scan roomstateid_pduid if we passed in room_id ? - self.roomstateid_pduid - .scan_prefix(room_id.as_bytes()) - .values() - .map(|pduid| { - let pduid = &pduid?; - self.pduid_pdu.get(pduid)?.map_or_else( - || { - Err(Error::bad_database( - "Failed to find current state of pduid's.", - )) - }, - |b| { - Ok(( - serde_json::from_slice::(&b) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - pduid.clone(), - )) - }, - ) - }) - .map(|pair| { - let (pdu, id) = pair?; - Ok(((pdu.kind, pdu.state_key), id)) - }) - .collect::>>() - } - /// Returns the last state hash key added to the db. - pub fn current_state_hash(&self, room_id: &RoomId) -> Result { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xff); - - // We must check here because this method is called outside and before - // `append_state_pdu` so the DB can be empty - if self.pduid_statehash.scan_prefix(prefix).next().is_none() { - // return the hash of the room_id, this represents a room with no state - return self.new_state_hash_id(room_id); - } - - self.pduid_statehash - .iter() - .next_back() - .map(|pair| Ok(pair?.1.to_vec())) - .ok_or_else(|| Error::bad_database("No PDU's found for this room."))? + pub fn current_state_hash(&self, room_id: &RoomId) -> Result> { + Ok(self + .roomid_statehash + .get(room_id.as_bytes())? + .map(|bytes| bytes.to_vec())) } /// This fetches auth event_ids from the current state using the @@ -243,39 +199,11 @@ impl Rooms { /// Generate a new StateHash. /// - /// A unique hash made from hashing the current states pduid's. - fn new_state_hash_id(&self, room_id: &RoomId) -> Result { - // Use hashed roomId as the first StateHash key for first state event in room - if self - .pduid_statehash - .scan_prefix(room_id.as_bytes()) - .next() - .is_none() - { - return Ok(digest::digest(&digest::SHA256, room_id.as_bytes()) - .as_ref() - .to_vec()); - } - - let pdu_ids_to_hash = self - .pduid_statehash - .scan_prefix(room_id.as_bytes()) - .values() - .next_back() - .unwrap() // We just checked if the tree was empty - .map(|hash| { - self.stateid_pduid - .scan_prefix(hash) - .values() - // pduid is roomId + count so just hash the whole thing - .map(|pid| Ok(pid?.to_vec())) - .collect::>>>() - })??; - - let hash = digest::digest( - &digest::SHA256, - &pdu_ids_to_hash.into_iter().flatten().collect::>(), - ); + /// A unique hash made from hashing all PDU ids of the state joined with 0xff. + fn calculate_hash(&self, pdu_id_bytes: &[&[u8]]) -> Result { + // We only hash the pdu's event ids, not the whole pdu + let bytes = pdu_id_bytes.join(&0xff); + let hash = digest::digest(&digest::SHA256, &bytes); Ok(hash.as_ref().to_vec()) } @@ -297,29 +225,38 @@ impl Rooms { &self, room_id: &RoomId, ) -> Result> { - let mut hashmap = HashMap::new(); - for pdu in - self.roomstateid_pduid - .scan_prefix(&room_id.as_bytes()) + if let Some(current_state_hash) = self.current_state_hash(room_id)? { + let mut prefix = current_state_hash; + prefix.push(0xff); + + let mut hashmap = HashMap::new(); + for pdu in self + .stateid_pduid + .scan_prefix(prefix) .values() - .map(|value| { + .map(|pdu_id| { Ok::<_, Error>( serde_json::from_slice::( - &self.pduid_pdu.get(value?)?.ok_or_else(|| { - Error::bad_database("PDU not found for ID in db.") + &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { + Error::bad_database("PDU in state not found in database.") })?, ) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + .map_err(|_| { + Error::bad_database("Invalid PDU bytes in current room state.") + })?, ) }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert((pdu.kind.clone(), state_key), pdu); + { + let pdu = pdu?; + let state_key = pdu.state_key.clone().ok_or_else(|| { + Error::bad_database("Room state contains event without state_key.") + })?; + hashmap.insert((pdu.kind.clone(), state_key), pdu); + } + Ok(hashmap) + } else { + Ok(HashMap::new()) } - Ok(hashmap) } /// Returns all state entries for this type. @@ -328,33 +265,40 @@ impl Rooms { room_id: &RoomId, event_type: &EventType, ) -> Result> { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xff); - prefix.extend_from_slice(&event_type.to_string().as_bytes()); + if let Some(current_state_hash) = self.current_state_hash(room_id)? { + let mut prefix = current_state_hash; + prefix.push(0xff); + prefix.extend_from_slice(&event_type.to_string().as_bytes()); + prefix.push(0xff); - let mut hashmap = HashMap::new(); - for pdu in - self.roomstateid_pduid + let mut hashmap = HashMap::new(); + for pdu in self + .stateid_pduid .scan_prefix(&prefix) .values() - .map(|value| { + .map(|pdu_id| { Ok::<_, Error>( serde_json::from_slice::( - &self.pduid_pdu.get(value?)?.ok_or_else(|| { - Error::bad_database("PDU not found for ID in db.") + &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { + Error::bad_database("PDU in state not found in database.") })?, ) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + .map_err(|_| { + Error::bad_database("Invalid PDU bytes in current room state.") + })?, ) }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert(state_key, pdu); + { + let pdu = pdu?; + let state_key = pdu.state_key.clone().ok_or_else(|| { + Error::bad_database("Room state contains event without state_key.") + })?; + hashmap.insert(state_key, pdu); + } + Ok(hashmap) + } else { + Ok(HashMap::new()) } - Ok(hashmap) } /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`). @@ -364,23 +308,24 @@ impl Rooms { event_type: &EventType, state_key: &str, ) -> Result> { - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(&event_type.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(&state_key.as_bytes()); - - self.roomstateid_pduid.get(&key)?.map_or(Ok(None), |value| { - Ok::<_, Error>(Some( - serde_json::from_slice::( - &self - .pduid_pdu - .get(value)? - .ok_or_else(|| Error::bad_database("PDU not found for ID in db."))?, - ) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - )) - }) + if let Some(current_state_hash) = self.current_state_hash(room_id)? { + let mut key = current_state_hash; + key.push(0xff); + key.extend_from_slice(&event_type.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(&state_key.as_bytes()); + + self.stateid_pduid.get(&key)?.map_or(Ok(None), |pdu_id| { + Ok::<_, Error>(Some( + serde_json::from_slice::(&self.pduid_pdu.get(pdu_id)?.ok_or_else( + || Error::bad_database("PDU in state not found in database."), + )?) + .map_err(|_| Error::bad_database("Invalid PDU bytes in current room state."))?, + )) + }) + } else { + Ok(None) + } } /// Returns the `count` of this pdu's id. @@ -528,8 +473,8 @@ impl Rooms { self.eventid_pduid .insert(pdu.event_id.as_bytes(), &*pdu_id)?; - if let Some(state_key) = &pdu.state_key { - self.append_state_pdu(&pdu.room_id, &pdu_id, state_key, &pdu.kind)?; + if pdu.state_key.is_some() { + self.append_to_state(&pdu_id, &pdu)?; } match &pdu.kind { @@ -603,59 +548,69 @@ impl Rooms { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. /// The incoming event is the `pdu_id` passed to this method. - fn append_state_pdu( - &self, - room_id: &RoomId, - pdu_id: &[u8], - state_key: &str, - kind: &EventType, - ) -> Result { - let state_hash = self.new_state_hash_id(room_id)?; - let state = self.current_state_pduids(room_id)?; - - let mut key = state_hash.to_vec(); - key.push(0xff); - - // TODO eventually we could avoid writing to the DB so much on every event - // by keeping track of the delta and write that every so often - for ((ev_ty, state_key), pid) in state { - let mut state_id = key.to_vec(); - state_id.extend_from_slice(ev_ty.to_string().as_bytes()); - key.push(0xff); - state_id.extend_from_slice(state_key.expect("state event").as_bytes()); - key.push(0xff); + fn append_to_state(&self, new_pdu_id: &[u8], new_pdu: &PduEvent) -> Result { + let old_state = + if let Some(old_state_hash) = self.roomid_statehash.get(new_pdu.room_id.as_bytes())? { + // Store state for event. The state does not include the event itself. + // Instead it's the state before the pdu, so the room's old state. + self.pduid_statehash.insert(new_pdu_id, &old_state_hash)?; + if new_pdu.state_key.is_none() { + return Ok(old_state_hash.to_vec()); + } - self.stateid_pduid.insert(&state_id, &pid)?; - } + let mut prefix = old_state_hash.to_vec(); + prefix.push(0xff); + self.stateid_pduid + .scan_prefix(&prefix) + .filter_map(|pdu| pdu.map_err(|e| error!("{}", e)).ok()) + .map(|(k, v)| (k.subslice(prefix.len(), k.len() - prefix.len()), v)) + .collect::>() + } else { + HashMap::new() + }; + + if let Some(state_key) = &new_pdu.state_key { + let mut new_state = old_state; + let mut pdu_key = new_pdu.kind.as_str().as_bytes().to_vec(); + pdu_key.push(0xff); + pdu_key.extend_from_slice(state_key.as_bytes()); + new_state.insert(pdu_key.into(), new_pdu_id.into()); - // This event's state does not include the event itself. `current_state_pduids` - // uses `roomstateid_pduid` before the current event is inserted to the tree so the state - // will be everything up to but not including the incoming event. - self.pduid_statehash.insert(pdu_id, state_hash.as_slice())?; + let new_state_hash = + self.calculate_hash(&new_state.values().map(|b| &**b).collect::>())?; - self.roomid_statehash - .insert(room_id.as_bytes(), state_hash.as_slice())?; + let mut key = new_state_hash.to_vec(); + key.push(0xff); + + // TODO: we could avoid writing to the DB on every state event by keeping + // track of the delta and write that every so often + for (key_without_prefix, pdu_id) in new_state { + let mut state_id = key.clone(); + state_id.extend_from_slice(&key_without_prefix); + self.stateid_pduid.insert(&state_id, &pdu_id)?; + } - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(kind.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pduid.insert(key, pdu_id)?; + self.roomid_statehash + .insert(new_pdu.room_id.as_bytes(), &*new_state_hash)?; - Ok(state_hash) + Ok(new_state_hash) + } else { + Err(Error::bad_database( + "Tried to insert non-state event into room without a state.", + )) + } } /// Creates a new persisted data unit and adds it to a room. pub fn build_and_append_pdu( &self, pdu_builder: PduBuilder, + sender: &UserId, + room_id: &RoomId, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, ) -> Result { let PduBuilder { - room_id, - sender, event_type, content, unsigned, @@ -741,8 +696,7 @@ impl Rooms { ErrorKind::Unknown, "Membership can't be the first event", ))?)? - .map(|pdu| pdu.convert_for_state_res()) - .transpose()?; + .map(|pdu| pdu.convert_for_state_res()); event_auth::valid_membership_change( // TODO this is a bit of a hack but not sure how to have a type // declared in `state_res` crate easily convert to/from conduit::PduEvent @@ -753,11 +707,12 @@ impl Rooms { state_key: Some(state_key.to_owned()), sender: &sender, }, - prev_event.as_ref(), + prev_event, + None, &auth_events .iter() .map(|((ty, key), pdu)| { - Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) + Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res())) }) .collect::>>()?, ) @@ -812,9 +767,8 @@ impl Rooms { let mut pdu = PduEvent { event_id: EventId::try_from("$thiswillbefilledinlater").expect("we know this is valid"), - room_id, - sender, - origin: globals.server_name().to_owned(), + room_id: room_id.clone(), + sender: sender.clone(), origin_server_ts: utils::millis_since_unix_epoch() .try_into() .expect("time is valid"), @@ -834,7 +788,7 @@ impl Rooms { hashes: ruma::events::pdu::EventHash { sha256: "aaa".to_owned(), }, - signatures: HashMap::new(), + signatures: BTreeMap::new(), }; // Generate event id @@ -1028,8 +982,6 @@ impl Rooms { self.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: user_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member_content) .expect("event is valid, we just created it"), @@ -1037,6 +989,8 @@ impl Rooms { state_key: Some(user_id.to_string()), redacts: None, }, + &user_id, + &room_id, globals, account_data, )?; diff --git a/src/pdu.rs b/src/pdu.rs index 15264842..f23e688c 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -5,18 +5,17 @@ use ruma::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, - EventId, Raw, RoomId, ServerName, UserId, + EventId, Raw, RoomId, ServerKeyId, ServerName, UserId, }; use serde::{Deserialize, Serialize}; use serde_json::json; -use std::{collections::HashMap, convert::TryFrom}; +use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::UNIX_EPOCH}; -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, Debug)] pub struct PduEvent { pub event_id: EventId, pub room_id: RoomId, pub sender: UserId, - pub origin: Box, pub origin_server_ts: UInt, #[serde(rename = "type")] pub kind: EventType, @@ -31,7 +30,7 @@ pub struct PduEvent { #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] pub unsigned: serde_json::Map, pub hashes: EventHash, - pub signatures: HashMap>, + pub signatures: BTreeMap, BTreeMap>, } impl PduEvent { @@ -199,66 +198,69 @@ impl PduEvent { } } -impl TryFrom<&state_res::StateEvent> for PduEvent { - type Error = Error; - fn try_from(pdu: &state_res::StateEvent) -> Result { - serde_json::from_value(json!({ - "event_id": pdu.event_id(), - "room_id": pdu.room_id(), - "sender": pdu.sender(), - "origin": pdu.origin(), - "origin_server_ts": pdu.origin_server_ts(), - "event_type": pdu.kind(), - "content": pdu.content(), - "state_key": pdu.state_key(), - "prev_events": pdu.prev_event_ids(), - "depth": pdu.depth(), - "auth_events": pdu.auth_events(), - "redacts": pdu.redacts(), - "unsigned": pdu.unsigned(), - "hashes": pdu.hashes(), - "signatures": pdu.signatures(), - })) - .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) +impl From<&state_res::StateEvent> for PduEvent { + fn from(pdu: &state_res::StateEvent) -> Self { + Self { + event_id: pdu.event_id().clone(), + room_id: pdu.room_id().unwrap().clone(), + sender: pdu.sender().clone(), + origin_server_ts: (pdu + .origin_server_ts() + .duration_since(UNIX_EPOCH) + .expect("time is valid") + .as_millis() as u64) + .try_into() + .expect("time is valid"), + kind: pdu.kind(), + content: pdu.content().clone(), + state_key: pdu.state_key(), + prev_events: pdu.prev_event_ids(), + depth: pdu.depth().clone(), + auth_events: pdu.auth_events(), + redacts: pdu.redacts().cloned(), + unsigned: pdu.unsigned().clone().into_iter().collect(), + hashes: pdu.hashes().clone(), + signatures: pdu.signatures(), + } } } impl PduEvent { - pub fn convert_for_state_res(&self) -> Result { - serde_json::from_value(json!({ - "event_id": self.event_id, - "room_id": self.room_id, - "sender": self.sender, - "origin": self.origin, - "origin_server_ts": self.origin_server_ts, - "type": self.kind, - "content": self.content, - "state_key": self.state_key, - "prev_events": self.prev_events - .iter() - // TODO How do we create one of these - .map(|id| (id, EventHash { sha256: "hello".into() })) - .collect::>(), - "depth": self.depth, - "auth_events": self.auth_events - .iter() - // TODO How do we create one of these - .map(|id| (id, EventHash { sha256: "hello".into() })) - .collect::>(), - "redacts": self.redacts, - "unsigned": self.unsigned, - "hashes": self.hashes, - "signatures": self.signatures, - })) - .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) + pub fn convert_for_state_res(&self) -> Arc { + Arc::new( + serde_json::from_value(json!({ + "event_id": self.event_id, + "room_id": self.room_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "type": self.kind, + "content": self.content, + "state_key": self.state_key, + "prev_events": self.prev_events + .iter() + // TODO How do we create one of these + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "depth": self.depth, + "auth_events": self.auth_events + .iter() + // TODO How do we create one of these + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "redacts": self.redacts, + "unsigned": self.unsigned, + "hashes": self.hashes, + "signatures": self.signatures, + })) + .expect("all conduit PDUs are state events"), + ) } } /// Build the start of a PDU in order to add it to the `Database`. #[derive(Debug, Deserialize)] pub struct PduBuilder { - pub room_id: RoomId, - pub sender: UserId, + #[serde(rename = "type")] pub event_type: EventType, pub content: serde_json::Value, pub unsigned: Option>, diff --git a/src/server_server.rs b/src/server_server.rs index d39abe64..ffa5c5be 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -139,6 +139,7 @@ where .unwrap() .into_iter() .collect(); + Ok( T::IncomingResponse::try_from(http_response.body(body).unwrap()) .expect("TODO: error handle other server errors"), From 1f2843498863065ca2775bd0a02a37db3f8f0002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 12 Sep 2020 22:41:33 +0200 Subject: [PATCH 20/51] feat: hacky transactions --- src/server_server.rs | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index ffa5c5be..d7d0e23f 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,16 +1,19 @@ use crate::{client_server, ConduitResult, Database, Error, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; -use ruma::api::{ - client, - federation::{ - directory::get_public_rooms, - discovery::{ - get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, +use ruma::{ + api::{ + client, + federation::{ + directory::get_public_rooms, + discovery::{ + get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, + }, + transactions::send_transaction_message, }, - transactions::send_transaction_message, + OutgoingRequest, }, - OutgoingRequest, + EventId, }; use serde_json::json; use std::{ @@ -264,10 +267,29 @@ pub async fn get_public_rooms_route( put("/_matrix/federation/v1/send/<_>", data = "") )] pub fn send_transaction_message_route<'a>( - _db: State<'a, Database>, + db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { dbg!(&*body); + for pdu in &body.pdus { + let mut value = serde_json::to_value(pdu).expect("all ruma pdus are json values"); + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&value).expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + value + .as_object_mut() + .expect("ruma pdus are json objects") + .insert("event_id".to_owned(), event_id.to_string().into()); + + db.rooms.append_pdu( + serde_json::from_value(value).expect("all ruma pdus are conduit pdus"), + &db.globals, + &db.account_data, + )?; + } Ok(send_transaction_message::v1::Response { pdus: BTreeMap::new(), } From af53485d70d474468b02d10badd6959fa05f1068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 12 Sep 2020 23:38:52 +0200 Subject: [PATCH 21/51] fix: avoid pdus without event ids --- src/client_server/membership.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index ea2271bb..20cf315b 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -502,8 +502,10 @@ async fn join_room_by_id_helper( .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? .insert("event_id".to_owned(), event_id.to_string().into()); + dbg!(&value); + serde_json::from_value::(value) - .map(|ev| (event_id, Arc::new(ev))) + .map(|ev| (dbg!(&ev).event_id().clone(), Arc::new(ev))) .map_err(|e| { warn!("{}", e); Error::BadServerResponse("Invalid PDU bytes in send_join response.") @@ -520,19 +522,14 @@ async fn join_room_by_id_helper( // These events are not guaranteed to be sorted but they are resolved according to spec // we auth them anyways to weed out faulty/malicious server. The following is basically the // full state resolution algorithm. + let event_ids = event_map.keys().cloned().collect::>(); + let sorted_control_events = state_res::StateResolution::reverse_topological_power_sort( &room_id, &control_events, &mut event_map, &db.rooms, - &send_join_response - .room_state - .auth_chain - .iter() - .filter_map(|pdu| { - Some(StateEvent::Full(pdu.deserialize().ok()?).event_id().clone()) - }) - .collect::>(), + &event_ids, ); // Auth check each event against the "partial" state created by the preceding events From 1f292c09f2e8a0679467ec5f3b699918d60cea64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 13 Sep 2020 22:24:36 +0200 Subject: [PATCH 22/51] improvement: better federation joins --- Cargo.lock | 30 ++++++------ src/client_server/membership.rs | 81 +++++++++++++++++++++---------- src/database/rooms.rs | 86 +++++++++++++++++++++------------ src/server_server.rs | 18 ++++--- 4 files changed, 136 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 646cdcc6..6ffd3473 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1634,7 +1634,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1650,7 +1650,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "http", "percent-encoding", @@ -1665,7 +1665,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "ruma-api", "ruma-common", @@ -1689,7 +1689,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "assign", "http", @@ -1708,7 +1708,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "js_int", "ruma-api", @@ -1722,7 +1722,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "js_int", "ruma-common", @@ -1737,7 +1737,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1748,7 +1748,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "js_int", "ruma-api", @@ -1763,7 +1763,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1775,7 +1775,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "proc-macro2", "quote", @@ -1786,7 +1786,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "serde", "strum", @@ -1795,7 +1795,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "form_urlencoded", "itoa", @@ -1807,7 +1807,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "base64", "ring", @@ -2072,7 +2072,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#0081081604b051d412a2365b68357e064c33320c" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#a9186476b748c901fbf4356414247a0b3ac01b5f" dependencies = [ "itertools", "js_int", diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 20cf315b..9285648c 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -2,7 +2,7 @@ use super::State; use crate::{ client_server, pdu::{PduBuilder, PduEvent}, - server_server, utils, ConduitResult, Database, Error, Ruma, + server_server, utils, ConduitResult, Database, Error, Result, Ruma, }; use log::warn; use ruma::{ @@ -17,11 +17,14 @@ use ruma::{ }, federation, }, + events::pdu::Pdu, events::{room::member, EventType}, EventId, Raw, RoomId, RoomVersionId, UserId, }; use state_res::StateEvent; -use std::{collections::BTreeMap, convert::TryFrom, sync::Arc}; +use std::{ + collections::BTreeMap, collections::HashMap, collections::HashSet, convert::TryFrom, sync::Arc, +}; #[cfg(feature = "conduit_bin")] use rocket::{get, post}; @@ -482,36 +485,49 @@ async fn join_room_by_id_helper( ) .await?; - let mut event_map = send_join_response + let add_event_id = |pdu: &Raw| { + let mut value = serde_json::from_str(pdu.json().get()) + .expect("converting raw jsons to values always works"); + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&value) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + value + .as_object_mut() + .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? + .insert("event_id".to_owned(), event_id.to_string().into()); + + Ok((event_id, value)) + }; + + let room_state = send_join_response.room_state.state.iter().map(add_event_id); + + let state_events = room_state + .clone() + .map(|pdu: Result<(EventId, serde_json::Value)>| Ok(pdu?.0)) + .collect::>>()?; + + let auth_chain = send_join_response .room_state - .state + .auth_chain .iter() - .chain(send_join_response.room_state.auth_chain.iter()) - .map(|pdu| { - let mut value = serde_json::from_str(pdu.json().get()) - .expect("converting raw jsons to values always works"); - let event_id = EventId::try_from(&*format!( - "${}", - ruma::signatures::reference_hash(&value) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); - - value - .as_object_mut() - .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? - .insert("event_id".to_owned(), event_id.to_string().into()); - - dbg!(&value); + .map(add_event_id); + let mut event_map = room_state + .chain(auth_chain) + .map(|r| { + let (event_id, value) = r?; serde_json::from_value::(value) - .map(|ev| (dbg!(&ev).event_id().clone(), Arc::new(ev))) + .map(|ev| (event_id, Arc::new(ev))) .map_err(|e| { warn!("{}", e); Error::BadServerResponse("Invalid PDU bytes in send_join response.") }) }) - .collect::>, _>>()?; + .collect::>>>()?; let control_events = event_map .values() @@ -575,6 +591,8 @@ async fn join_room_by_id_helper( ) .expect("iterative auth check failed on resolved events"); + let mut state = HashMap::new(); + // filter the events that failed the auth check keeping the remaining events // sorted correctly for ev_id in sorted_event_ids @@ -587,9 +605,22 @@ async fn join_room_by_id_helper( .expect("Found event_id in sorted events that is not in resolved state"); // We do not rebuild the PDU in this case only insert to DB - db.rooms - .append_pdu(PduEvent::from(&**pdu), &db.globals, &db.account_data)?; + let pdu_id = + db.rooms + .append_pdu(&PduEvent::from(&**pdu), &db.globals, &db.account_data)?; + + if state_events.contains(ev_id) { + state.insert( + ( + pdu.kind(), + pdu.state_key().expect("State events have a state key"), + ), + pdu_id, + ); + } } + + db.rooms.force_state(room_id, state)?; } else { let event = member::MemberEventContent { membership: member::MembershipState::Join, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 87f4dcd4..b538c854 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -220,6 +220,31 @@ impl Rooms { .is_some()) } + /// Returns the full room state. + pub fn force_state( + &self, + room_id: &RoomId, + state: HashMap<(EventType, String), Vec>, + ) -> Result<()> { + let state_hash = + self.calculate_hash(&state.values().map(|pdu_id| &**pdu_id).collect::>())?; + let mut prefix = state_hash.clone(); + prefix.push(0xff); + + for ((event_type, state_key), pdu_id) in state { + let mut state_id = prefix.clone(); + state_id.extend_from_slice(&event_type.as_str().as_bytes()); + state_id.push(0xff); + state_id.extend_from_slice(&state_key.as_bytes()); + self.stateid_pduid.insert(state_id, pdu_id)?; + } + + self.roomid_statehash + .insert(room_id.as_bytes(), &*state_hash)?; + + Ok(()) + } + /// Returns the full room state. pub fn room_state_full( &self, @@ -446,10 +471,10 @@ impl Rooms { /// Creates a new persisted data unit and adds it to a room. pub fn append_pdu( &self, - pdu: PduEvent, + pdu: &PduEvent, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, - ) -> Result { + ) -> Result> { let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); ruma::signatures::hash_and_sign_event( globals.server_name().as_str(), @@ -473,10 +498,6 @@ impl Rooms { self.eventid_pduid .insert(pdu.event_id.as_bytes(), &*pdu_id)?; - if pdu.state_key.is_some() { - self.append_to_state(&pdu_id, &pdu)?; - } - match pdu.kind { EventType::RoomRedaction => { if let Some(redact_id) = &pdu.redacts { @@ -484,23 +505,22 @@ impl Rooms { } } EventType::RoomMember => { - if let Some(state_key) = pdu.state_key { + if let Some(state_key) = &pdu.state_key { // if the state_key fails - let target_user_id = UserId::try_from(state_key) + let target_user_id = UserId::try_from(state_key.clone()) .expect("This state_key was previously validated"); // Update our membership info, we do this here incase a user is invited // and immediately leaves we need the DB to record the invite event for auth self.update_membership( &pdu.room_id, &target_user_id, - serde_json::from_value::(pdu.content).map_err( - |_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid redaction event content.", - ) - }, - )?, + serde_json::from_value::(pdu.content.clone()) + .map_err(|_| { + Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid redaction event content.", + ) + })?, &pdu.sender, account_data, globals, @@ -528,7 +548,7 @@ impl Rooms { self.edus .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; - Ok(pdu.event_id) + Ok(pdu_id) } /// Generates a new StateHash and associates it with the incoming event. @@ -789,7 +809,13 @@ impl Rooms { )) .expect("ruma's reference hashes are valid event ids"); - self.append_pdu(pdu, globals, account_data) + let pdu_id = self.append_pdu(&pdu, globals, account_data)?; + + if pdu.state_key.is_some() { + self.append_to_state(&pdu_id, &pdu)?; + } + + Ok(pdu.event_id) } /// Returns an iterator over all PDUs in a room. @@ -953,19 +979,17 @@ impl Rooms { self.roomuseroncejoinedids.insert(&userroom_id, &[])?; // Check if the room has a predecessor - if let Some(predecessor) = serde_json::from_value::< - Raw, - >( - self.room_state_get(&room_id, &EventType::RoomCreate, "")? - .ok_or_else(|| { - Error::bad_database("Found room without m.room.create event.") - })? - .content, - ) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid room event in database."))? - .predecessor + if let Some(predecessor) = self + .room_state_get(&room_id, &EventType::RoomCreate, "")? + .and_then(|create| { + serde_json::from_value::< + Raw, + >(create.content) + .expect("Raw::from_value always works") + .deserialize() + .ok() + }) + .and_then(|content| content.predecessor) { // Copy user settings from predecessor to the current room: // - Push rules diff --git a/src/server_server.rs b/src/server_server.rs index d7d0e23f..6634d5a2 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,4 +1,4 @@ -use crate::{client_server, ConduitResult, Database, Error, Result, Ruma}; +use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ @@ -270,9 +270,11 @@ pub fn send_transaction_message_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { - dbg!(&*body); + //dbg!(&*body); for pdu in &body.pdus { - let mut value = serde_json::to_value(pdu).expect("all ruma pdus are json values"); + let mut value = serde_json::from_str(pdu.json().get()) + .expect("converting raw jsons to values always works"); + let event_id = EventId::try_from(&*format!( "${}", ruma::signatures::reference_hash(&value).expect("ruma can calculate reference hashes") @@ -284,11 +286,11 @@ pub fn send_transaction_message_route<'a>( .expect("ruma pdus are json objects") .insert("event_id".to_owned(), event_id.to_string().into()); - db.rooms.append_pdu( - serde_json::from_value(value).expect("all ruma pdus are conduit pdus"), - &db.globals, - &db.account_data, - )?; + let pdu = + serde_json::from_value::(value).expect("all ruma pdus are conduit pdus"); + if db.rooms.exists(&pdu.room_id)? { + db.rooms.append_pdu(&pdu, &db.globals, &db.account_data)?; + } } Ok(send_transaction_message::v1::Response { pdus: BTreeMap::new(), From c5313b3e8f8d024c540d0428a6c75828d82f95c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 11:00:31 +0200 Subject: [PATCH 23/51] improvement: try out multiple servers when joining remote rooms --- Cargo.lock | 35 +++++++++++----------- src/client_server/alias.rs | 4 +-- src/client_server/directory.rs | 12 ++++---- src/client_server/membership.rs | 52 +++++++++++++++++++++------------ src/server_server.rs | 10 +++---- 5 files changed, 65 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ffd3473..28a4395e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,7 +157,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.4.1", + "miniz_oxide 0.4.2", "object", "rustc-demangle", ] @@ -1054,11 +1054,12 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d7559a8a40d0f97e1edea3220f698f78b1c5ab67532e49f68fde3910323b722" +checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" dependencies = [ "adler", + "autocfg", ] [[package]] @@ -1634,7 +1635,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1650,7 +1651,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "http", "percent-encoding", @@ -1665,7 +1666,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1676,7 +1677,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "ruma-api", "ruma-common", @@ -1689,7 +1690,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "assign", "http", @@ -1708,7 +1709,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "js_int", "ruma-api", @@ -1722,7 +1723,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "js_int", "ruma-common", @@ -1737,7 +1738,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1748,7 +1749,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "js_int", "ruma-api", @@ -1763,7 +1764,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1775,7 +1776,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "proc-macro2", "quote", @@ -1786,7 +1787,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "serde", "strum", @@ -1795,7 +1796,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "form_urlencoded", "itoa", @@ -1807,7 +1808,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "base64", "ring", diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index bfdaecac..0ec43f57 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -63,7 +63,7 @@ pub async fn get_alias_helper( if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( &db, - room_alias.server_name().to_string(), + room_alias.server_name(), federation::query::get_room_information::v1::Request { room_alias }, ) .await?; @@ -79,5 +79,5 @@ pub async fn get_alias_helper( "Room with alias not found.", ))?; - Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_string()]).into()) + Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_owned()]).into()) } diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 2764d2c5..a68d8dd1 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -19,7 +19,7 @@ use ruma::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, }, - Raw, + Raw, ServerName, }; #[cfg(feature = "conduit_bin")] @@ -65,9 +65,9 @@ pub async fn get_public_rooms_route( ) -> ConduitResult { let response = get_public_rooms_filtered_helper( &db, - body.body.server.as_deref(), - body.body.limit, - body.body.since.as_deref(), + body.server.as_deref(), + body.limit, + body.since.as_deref(), None, // This is not used None, // This is not used ) @@ -119,7 +119,7 @@ pub async fn get_room_visibility_route( pub async fn get_public_rooms_filtered_helper( db: &Database, - server: Option<&str>, + server: Option<&ServerName>, limit: Option, since: Option<&str>, _filter: Option, @@ -131,7 +131,7 @@ pub async fn get_public_rooms_filtered_helper( { let response = server_server::send_request( &db, - other_server.to_owned(), + other_server, federation::directory::get_public_rooms::v1::Request { limit, since: since.as_deref(), diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 9285648c..8d194024 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -19,7 +19,7 @@ use ruma::{ }, events::pdu::Pdu, events::{room::member, EventType}, - EventId, Raw, RoomId, RoomVersionId, UserId, + EventId, Raw, RoomId, RoomVersionId, ServerName, UserId, }; use state_res::StateEvent; use std::{ @@ -41,6 +41,7 @@ pub async fn join_room_by_id_route( &db, body.sender_id.as_ref(), &body.room_id, + &[body.room_id.server_name().to_owned()], body.third_party_signed.as_ref(), ) .await @@ -54,13 +55,12 @@ pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { - Ok(room_id) => room_id, + let (servers, room_id) = match RoomId::try_from(body.room_id_or_alias.clone()) { + Ok(room_id) => (vec![room_id.server_name().to_owned()], room_id), Err(room_alias) => { - client_server::get_alias_helper(&db, &room_alias) - .await? - .0 - .room_id + let response = client_server::get_alias_helper(&db, &room_alias).await?; + + (response.0.servers, response.0.room_id) } }; @@ -69,6 +69,7 @@ pub async fn join_room_by_id_or_alias_route( &db, body.sender_id.as_ref(), &room_id, + &servers, body.third_party_signed.as_ref(), ) .await? @@ -415,22 +416,37 @@ async fn join_room_by_id_helper( db: &Database, sender_id: Option<&UserId>, room_id: &RoomId, + servers: &[Box], _third_party_signed: Option<&IncomingThirdPartySigned>, ) -> ConduitResult { let sender_id = sender_id.expect("user is authenticated"); // Ask a remote server if we don't have this room if !db.rooms.exists(&room_id)? && room_id.server_name() != db.globals.server_name() { - let make_join_response = server_server::send_request( - &db, - room_id.server_name().to_string(), - federation::membership::create_join_event_template::v1::Request { - room_id, - user_id: sender_id, - ver: &[RoomVersionId::Version5, RoomVersionId::Version6], - }, - ) - .await?; + let mut make_join_response_and_server = Err(Error::BadServerResponse( + "No server available to assist in joining.", + )); + + for remote_server in servers { + let make_join_response = server_server::send_request( + &db, + remote_server, + federation::membership::create_join_event_template::v1::Request { + room_id, + user_id: sender_id, + ver: &[RoomVersionId::Version5, RoomVersionId::Version6], + }, + ) + .await; + + make_join_response_and_server = make_join_response.map(|r| (r, remote_server)); + + if make_join_response_and_server.is_ok() { + break; + } + } + + let (make_join_response, remote_server) = make_join_response_and_server?; let mut join_event_stub_value = serde_json::from_str::(make_join_response.event.json().get()) @@ -475,7 +491,7 @@ async fn join_room_by_id_helper( let send_join_response = server_server::send_request( &db, - room_id.server_name().to_string(), + remote_server, federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, diff --git a/src/server_server.rs b/src/server_server.rs index 6634d5a2..fc1da001 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -13,7 +13,7 @@ use ruma::{ }, OutgoingRequest, }, - EventId, + EventId, ServerName, }; use serde_json::json; use std::{ @@ -44,16 +44,16 @@ pub async fn request_well_known(db: &crate::Database, destination: &str) -> Opti pub async fn send_request( db: &crate::Database, - destination: String, + destination: &ServerName, request: T, ) -> Result where T: Debug, { let actual_destination = "https://".to_owned() - + &request_well_known(db, &destination) + + &request_well_known(db, &destination.as_str()) .await - .unwrap_or(destination.clone() + ":8448"); + .unwrap_or(destination.as_str().to_owned() + ":8448"); let mut http_request = request .try_into_http_request(&actual_destination, Some("")) @@ -82,7 +82,7 @@ where "origin".to_owned(), db.globals.server_name().as_str().into(), ); - request_map.insert("destination".to_owned(), destination.into()); + request_map.insert("destination".to_owned(), destination.as_str().into()); let mut request_json = request_map.into(); ruma::signatures::sign_json( From 4e44fedbcd823876862315ac3590cc3d21a2825e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 11:42:16 +0200 Subject: [PATCH 24/51] fix: room list over federation --- Cargo.lock | 28 ++++++------ src/client_server/directory.rs | 32 +++++--------- src/server_server.rs | 80 +++++++++++++++++++++++----------- 3 files changed, 80 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28a4395e..865540c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1635,7 +1635,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1651,7 +1651,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "http", "percent-encoding", @@ -1666,7 +1666,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1677,7 +1677,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "ruma-api", "ruma-common", @@ -1690,7 +1690,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "assign", "http", @@ -1709,7 +1709,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "js_int", "ruma-api", @@ -1723,7 +1723,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "js_int", "ruma-common", @@ -1738,7 +1738,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1749,7 +1749,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "js_int", "ruma-api", @@ -1764,7 +1764,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1776,7 +1776,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "proc-macro2", "quote", @@ -1787,7 +1787,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "serde", "strum", @@ -1796,7 +1796,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "form_urlencoded", "itoa", @@ -1808,7 +1808,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "base64", "ring", diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index a68d8dd1..f30825d1 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,6 +14,7 @@ use ruma::{ }, federation, }, + directory::RoomNetwork, directory::{IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, @@ -33,24 +34,13 @@ pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - let Ruma { - body: - get_public_rooms_filtered::IncomingRequest { - limit, - server, - since, - filter, - room_network, - }, - .. - } = body; get_public_rooms_filtered_helper( &db, - server.as_deref(), - limit, - since.as_deref(), - filter, // This is not used yet - Some(room_network), // This is not used + body.server.as_deref(), + body.limit, + body.since.as_deref(), + &body.filter, + &body.room_network, ) .await } @@ -68,8 +58,8 @@ pub async fn get_public_rooms_route( body.server.as_deref(), body.limit, body.since.as_deref(), - None, // This is not used - None, // This is not used + &IncomingFilter::default(), + &IncomingRoomNetwork::Matrix, ) .await? .0; @@ -122,8 +112,8 @@ pub async fn get_public_rooms_filtered_helper( server: Option<&ServerName>, limit: Option, since: Option<&str>, - _filter: Option, - _network: Option, + _filter: &IncomingFilter, + _network: &IncomingRoomNetwork, ) -> ConduitResult { if let Some(other_server) = server .clone() @@ -135,7 +125,7 @@ pub async fn get_public_rooms_filtered_helper( federation::directory::get_public_rooms::v1::Request { limit, since: since.as_deref(), - room_network: ruma::directory::RoomNetwork::Matrix, + room_network: RoomNetwork::Matrix, }, ) .await?; diff --git a/src/server_server.rs b/src/server_server.rs index fc1da001..6c53aed2 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -2,8 +2,8 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Rum use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ + api::federation::directory::get_public_rooms_filtered, api::{ - client, federation::{ directory::get_public_rooms, discovery::{ @@ -13,6 +13,7 @@ use ruma::{ }, OutgoingRequest, }, + directory::{IncomingFilter, IncomingRoomNetwork}, EventId, ServerName, }; use serde_json::json; @@ -209,38 +210,67 @@ pub fn get_server_keys_deprecated(db: State<'_, Database>) -> Json { feature = "conduit_bin", post("/_matrix/federation/v1/publicRooms", data = "") )] +pub async fn get_public_rooms_filtered_route( + db: State<'_, Database>, + body: Ruma>, +) -> ConduitResult { + let response = client_server::get_public_rooms_filtered_helper( + &db, + None, + body.limit, + body.since.as_deref(), + &body.filter, + &body.room_network, + ) + .await? + .0; + + Ok(get_public_rooms_filtered::v1::Response { + chunk: response + .chunk + .into_iter() + .map(|c| { + // Convert ruma::api::federation::directory::get_public_rooms::v1::PublicRoomsChunk + // to ruma::api::client::r0::directory::PublicRoomsChunk + Ok::<_, Error>( + serde_json::from_str( + &serde_json::to_string(&c) + .expect("PublicRoomsChunk::to_string always works"), + ) + .expect("federation and client-server PublicRoomsChunk are the same type"), + ) + }) + .filter_map(|r| r.ok()) + .collect(), + prev_batch: response.prev_batch, + next_batch: response.next_batch, + total_room_count_estimate: response.total_room_count_estimate, + } + .into()) +} + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/federation/v1/publicRooms", data = "") +)] pub async fn get_public_rooms_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - let Ruma { - body: - get_public_rooms::v1::IncomingRequest { - room_network: _room_network, // TODO - limit, - since, - }, - .. - } = body; - - let client::r0::directory::get_public_rooms_filtered::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } = client_server::get_public_rooms_filtered_helper( + let response = client_server::get_public_rooms_filtered_helper( &db, None, - limit, - since.as_deref(), - None, - Some(ruma::directory::IncomingRoomNetwork::Matrix), + body.limit, + body.since.as_deref(), + &IncomingFilter::default(), + &IncomingRoomNetwork::Matrix, ) .await? .0; Ok(get_public_rooms::v1::Response { - chunk: chunk + chunk: response + .chunk .into_iter() .map(|c| { // Convert ruma::api::federation::directory::get_public_rooms::v1::PublicRoomsChunk @@ -255,9 +285,9 @@ pub async fn get_public_rooms_route( }) .filter_map(|r| r.ok()) .collect(), - prev_batch, - next_batch, - total_room_count_estimate, + prev_batch: response.prev_batch, + next_batch: response.next_batch, + total_room_count_estimate: response.total_room_count_estimate, } .into()) } From aa5e9e607ecc739d0d991ea7221dadd0125f6d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 14:20:38 +0200 Subject: [PATCH 25/51] feat: download media and thumbnails over federation --- src/client_server/media.rs | 69 +++++++++++++++++++++++++++++++++++--- src/database/media.rs | 28 ++++++++++++++-- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index d0774472..8f337435 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -1,5 +1,7 @@ use super::State; -use crate::{database::media::FileMeta, utils, ConduitResult, Database, Error, Ruma}; +use crate::{ + database::media::FileMeta, server_server, utils, ConduitResult, Database, Error, Ruma, +}; use ruma::api::client::{ error::ErrorKind, r0::media::{create_content, get_content, get_content_thumbnail, get_media_config}, @@ -35,7 +37,7 @@ pub fn create_content_route( utils::random_string(MXC_LENGTH) ); db.media - .create(mxc.clone(), &body.filename, &body.content_type, &body.file)?; + .create(mxc.clone(), &body.filename.as_deref(), &body.content_type, &body.file)?; Ok(create_content::Response { content_uri: mxc }.into()) } @@ -47,19 +49,25 @@ pub fn create_content_route( data = "" ) )] -pub fn get_content_route( +pub async fn get_content_route( db: State<'_, Database>, body: Ruma>, _server_name: String, _media_id: String, ) -> ConduitResult { + let mxc = format!( + "mxc://{}/{}", + db.globals.server_name(), + utils::random_string(MXC_LENGTH) + ); + if let Some(FileMeta { filename, content_type, file, }) = db .media - .get(format!("mxc://{}/{}", body.server_name, body.media_id))? + .get(&mxc)? { Ok(get_content::Response { file, @@ -67,6 +75,26 @@ pub fn get_content_route( content_disposition: filename.unwrap_or_default(), // TODO: Spec says this should be optional } .into()) + } else if body.allow_remote { + let get_content_response = server_server::send_request( + &db, + body.server_name.as_ref(), + get_content::Request { + allow_remote: false, + server_name: &body.server_name, + media_id: &body.media_id, + }, + ) + .await?; + + db.media.create( + mxc, + &Some(&get_content_response.content_disposition), + &get_content_response.content_type, + &get_content_response.file, + )?; + + Ok(get_content_response.into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } @@ -79,7 +107,7 @@ pub fn get_content_route( data = "" ) )] -pub fn get_content_thumbnail_route( +pub async fn get_content_thumbnail_route( db: State<'_, Database>, body: Ruma>, _server_name: String, @@ -97,6 +125,37 @@ pub fn get_content_thumbnail_route( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, )? { Ok(get_content_thumbnail::Response { file, content_type }.into()) + } else if body.allow_remote { + let get_thumbnail_response = server_server::send_request( + &db, + body.server_name.as_ref(), + get_content_thumbnail::Request { + allow_remote: false, + height: body.height, + width: body.width, + method: body.method, + server_name: &body.server_name, + media_id: &body.media_id, + }, + ) + .await?; + + let mxc = format!( + "mxc://{}/{}", + db.globals.server_name(), + utils::random_string(MXC_LENGTH) + ); + + db.media.upload_thumbnail( + mxc, + &None, + &get_thumbnail_response.content_type, + body.width.try_into().expect("all UInts are valid u32s"), + body.height.try_into().expect("all UInts are valid u32s"), + &get_thumbnail_response.file, + )?; + + Ok(get_thumbnail_response.into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } diff --git a/src/database/media.rs b/src/database/media.rs index 63fa11c6..869d5d80 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -16,7 +16,7 @@ impl Media { pub fn create( &self, mxc: String, - filename: &Option, + filename: &Option<&str>, content_type: &str, file: &[u8], ) -> Result<()> { @@ -34,8 +34,32 @@ impl Media { Ok(()) } + /// Uploads or replaces a file thumbnail. + pub fn upload_thumbnail( + &self, + mxc: String, + filename: &Option, + content_type: &str, + width: u32, + height: u32, + file: &[u8], + ) -> Result<()> { + let mut key = mxc.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(&width.to_be_bytes()); + key.extend_from_slice(&height.to_be_bytes()); + key.push(0xff); + key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default()); + key.push(0xff); + key.extend_from_slice(content_type.as_bytes()); + + self.mediaid_file.insert(key, file)?; + + Ok(()) + } + /// Downloads a file. - pub fn get(&self, mxc: String) -> Result> { + pub fn get(&self, mxc: &str) -> Result> { let mut prefix = mxc.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail From d1099e9224f3b47d9de9135ab751edd9152dc3b0 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 25 Aug 2020 11:49:51 +0200 Subject: [PATCH 26/51] Update dependencies --- Cargo.lock | 274 +------------------------------------ Cargo.toml | 57 +++++--- src/client_server/media.rs | 22 +-- 3 files changed, 56 insertions(+), 297 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 865540c8..bde0b0dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,80 +1,11 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "addr2line" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" - [[package]] name = "adler32" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" -[[package]] -name = "aead" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" -dependencies = [ - "generic-array", -] - -[[package]] -name = "aes" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7001367fde4c768a19d1029f0a8be5abd9308e1119846d5bd9ad26297b8faf5" -dependencies = [ - "aes-soft", - "aesni", - "block-cipher", -] - -[[package]] -name = "aes-gcm" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f5007801316299f922a6198d1d09a0bae95786815d066d5880d13f7c45ead1" -dependencies = [ - "aead", - "aes", - "block-cipher", - "ghash", - "subtle", -] - -[[package]] -name = "aes-soft" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4925647ee64e5056cf231608957ce7c81e12d6d6e316b9ce1404778cc1d35fa7" -dependencies = [ - "block-cipher", - "byteorder", - "opaque-debug 0.2.3", -] - -[[package]] -name = "aesni" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050d39b0b7688b3a3254394c3e30a9d66c41dcf9b05b0e2dbdc623f6505d264" -dependencies = [ - "block-cipher", - "opaque-debug 0.2.3", -] - [[package]] name = "ansi_term" version = "0.12.1" @@ -90,12 +21,6 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" -[[package]] -name = "array-init" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30bbe2f5e3d117f55bd8c7a1f9191e4a5deba9f15f595bbea4f670c59c765db" - [[package]] name = "arrayref" version = "0.3.6" @@ -148,20 +73,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" -[[package]] -name = "backtrace" -version = "0.3.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide 0.4.2", - "object", - "rustc-demangle", -] - [[package]] name = "base-x" version = "0.2.6" @@ -197,24 +108,6 @@ dependencies = [ "constant_time_eq", ] -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - -[[package]] -name = "block-cipher" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa136449e765dc7faa244561ccae839c394048667929af599b5d931ebe7b7f10" -dependencies = [ - "generic-array", -] - [[package]] name = "bumpalo" version = "3.4.0" @@ -319,12 +212,7 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ - "aes-gcm", - "base64", - "hkdf", "percent-encoding", - "rand", - "sha2", "time 0.2.19", "version_check", ] @@ -345,12 +233,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" -[[package]] -name = "cpuid-bool" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" - [[package]] name = "crc32fast" version = "1.2.0" @@ -386,16 +268,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "crypto-mac" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "deflate" version = "0.8.6" @@ -436,22 +308,12 @@ dependencies = [ "syn", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "directories" -version = "2.0.2" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "551a778172a450d7fc12e629ca3b0428d00f6afa9a43da1b630d54604e97371c" +checksum = "f8fed639d60b58d0f53498ab13d26f621fd77569cc6edb031f4cc36a2ad9da0f" dependencies = [ - "cfg-if", "dirs-sys", ] @@ -654,16 +516,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "generic-array" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.1.15" @@ -675,15 +527,6 @@ dependencies = [ "wasi 0.9.0+wasi-snapshot-preview1", ] -[[package]] -name = "ghash" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6e27f0689a6e15944bdce7e45425efb87eaa8ab0c6e87f11d0987a9133e2531" -dependencies = [ - "polyval", -] - [[package]] name = "gif" version = "0.10.3" @@ -694,12 +537,6 @@ dependencies = [ "lzw", ] -[[package]] -name = "gimli" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" - [[package]] name = "glob" version = "0.3.0" @@ -749,26 +586,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hkdf" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe1149865383e4526a43aee8495f9a325f0b806c63ce6427d06336a590abbbc9" -dependencies = [ - "digest", - "hmac", -] - -[[package]] -name = "hmac" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" -dependencies = [ - "crypto-mac", - "digest", -] - [[package]] name = "http" version = "0.2.1" @@ -1052,16 +869,6 @@ dependencies = [ "adler32", ] -[[package]] -name = "miniz_oxide" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" -dependencies = [ - "adler", - "autocfg", -] - [[package]] name = "mio" version = "0.6.22" @@ -1184,30 +991,12 @@ dependencies = [ "libc", ] -[[package]] -name = "object" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" - [[package]] name = "once_cell" version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" version = "0.10.30" @@ -1341,17 +1130,7 @@ dependencies = [ "bitflags", "crc32fast", "deflate", - "miniz_oxide 0.3.7", -] - -[[package]] -name = "polyval" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a50142b55ab3ed0e9f68dfb3709f1d90d29da24e91033f28b96330643107dc" -dependencies = [ - "cfg-if", - "universal-hash", + "miniz_oxide", ] [[package]] @@ -1828,12 +1607,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "rustc-demangle" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" - [[package]] name = "rustc_version" version = "0.2.3" @@ -1975,19 +1748,6 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" -[[package]] -name = "sha2" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1" -dependencies = [ - "block-buffer", - "cfg-if", - "cpuid-bool", - "digest", - "opaque-debug 0.3.0", -] - [[package]] name = "sharded-slab" version = "0.0.9" @@ -2015,12 +1775,10 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "sled" -version = "0.32.1" +version = "0.34.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3dbbb8ee10611bd1d020767c27599ccbbf8365f7e0ed7e54429cc8b9433ad8" +checksum = "f72c064e63fbca3138ad07f3588c58093f1684f3a99f60dcfa6d46b87e60fde7" dependencies = [ - "array-init", - "backtrace", "crc32fast", "crossbeam-epoch", "crossbeam-utils", @@ -2156,12 +1914,6 @@ dependencies = [ "syn", ] -[[package]] -name = "subtle" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd" - [[package]] name = "syn" version = "1.0.40" @@ -2436,12 +2188,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" -[[package]] -name = "typenum" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" - [[package]] name = "unicase" version = "2.6.0" @@ -2481,16 +2227,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" -[[package]] -name = "universal-hash" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "untrusted" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index 1b7a7007..60296a2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,28 +12,49 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +# Used to handle requests # TODO: This can become optional as soon as proper configs are supported -#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests -rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } -#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fed-fixes" } # Used for matrix spec type definitions and helpers +#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", default-features = false, features = ["tls"] } # Used to handle requests +rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] } + +# Used for matrix spec type definitions and helpers +#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } +ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fed-fixes" } #ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } -tokio = "0.2.22" # Used for long polling -sled = "0.32.0" # Used for storing data permanently -log = "0.4.8" # Used for emitting log entries -http = "0.2.1" # Used for rocket<->ruma conversions -directories = "2.0.2" # Used to find data directory for default db path -js_int = "0.1.5" # Used for number types for ruma -serde_json = { version = "1.0.53", features = ["raw_value"] } # Used for ruma wrapper -serde = "1.0.111" # Used for pdu definition -rand = "0.7.3" # Used for secure identifiers -rust-argon2 = "0.8.2" # Used to hash passwords -reqwest = "0.10.6" # Used to send requests -thiserror = "1.0.19" # Used for conduit::Error type -image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images -base64 = "0.12.3" # Used to encode server public key + +# Used when doing state resolution state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] } #state-res = { path = "../state-res", features = ["unstable-pre-spec"] } + +# Used for long polling +tokio = "0.2.22" +# Used for storing data permanently +sled = "0.34.4" +# Used for emitting log entries +log = "0.4.11" +# Used for rocket<->ruma conversions +http = "0.2.1" +# Used to find data directory for default db path +directories = "3.0.1" +# Used for number types for ruma +js_int = "0.1.9" +# Used for ruma wrapper +serde_json = { version = "1.0.57", features = ["raw_value"] } +# Used for pdu definition +serde = "1.0.116" +# Used for secure identifiers +rand = "0.7.3" +# Used to hash passwords +rust-argon2 = "0.8.2" +# Used to send requests +reqwest = "0.10.8" +# Used for conduit::Error type +thiserror = "1.0.20" +# Used to generate thumbnails for images +image = { version = "0.23.9", default-features = false, features = ["jpeg", "png", "gif"] } +# Used to encode server public key +base64 = "0.12.3" +# Used when hashing the state ring = "0.16.15" [features] diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 8f337435..f897a678 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -36,8 +36,12 @@ pub fn create_content_route( db.globals.server_name(), utils::random_string(MXC_LENGTH) ); - db.media - .create(mxc.clone(), &body.filename.as_deref(), &body.content_type, &body.file)?; + db.media.create( + mxc.clone(), + &body.filename.as_deref(), + &body.content_type, + &body.file, + )?; Ok(create_content::Response { content_uri: mxc }.into()) } @@ -55,19 +59,17 @@ pub async fn get_content_route( _server_name: String, _media_id: String, ) -> ConduitResult { - let mxc = format!( - "mxc://{}/{}", - db.globals.server_name(), - utils::random_string(MXC_LENGTH) - ); + let mxc = format!( + "mxc://{}/{}", + db.globals.server_name(), + utils::random_string(MXC_LENGTH) + ); if let Some(FileMeta { filename, content_type, file, - }) = db - .media - .get(&mxc)? + }) = db.media.get(&mxc)? { Ok(get_content::Response { file, From 9f05ef926af6f0c7c7d2886b99b87ccd6e218e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 16:23:15 +0200 Subject: [PATCH 27/51] fix: filter public room dir --- src/client_server/directory.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index f30825d1..871a780f 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,6 +14,7 @@ use ruma::{ }, federation, }, + directory::Filter, directory::RoomNetwork, directory::{IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk}, events::{ @@ -112,7 +113,7 @@ pub async fn get_public_rooms_filtered_helper( server: Option<&ServerName>, limit: Option, since: Option<&str>, - _filter: &IncomingFilter, + filter: &IncomingFilter, _network: &IncomingRoomNetwork, ) -> ConduitResult { if let Some(other_server) = server @@ -122,9 +123,12 @@ pub async fn get_public_rooms_filtered_helper( let response = server_server::send_request( &db, other_server, - federation::directory::get_public_rooms::v1::Request { + federation::directory::get_public_rooms_filtered::v1::Request { limit, since: since.as_deref(), + filter: Filter { + generic_search_term: filter.generic_search_term.as_deref(), + }, room_network: RoomNetwork::Matrix, }, ) From f7816b11de0889fca761f55510a3313dcfa78a42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 20:23:19 +0200 Subject: [PATCH 28/51] feat: send messages over federation --- Cargo.lock | 32 ++--- src/client_server/account.rs | 4 +- src/client_server/alias.rs | 2 +- src/client_server/directory.rs | 2 +- src/client_server/media.rs | 4 +- src/client_server/membership.rs | 206 +++++++++++++++++--------------- src/client_server/message.rs | 4 +- src/client_server/profile.rs | 8 +- src/client_server/redact.rs | 4 +- src/client_server/room.rs | 37 +++--- src/client_server/state.rs | 12 +- src/database.rs | 1 + src/database/rooms.rs | 177 ++++++++++++++++++--------- src/pdu.rs | 27 ++++- src/server_server.rs | 22 ++-- 15 files changed, 324 insertions(+), 218 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bde0b0dd..e0de2a7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1414,7 +1414,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1430,7 +1430,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "http", "percent-encoding", @@ -1445,7 +1445,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1456,7 +1456,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "ruma-api", "ruma-common", @@ -1469,7 +1469,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "assign", "http", @@ -1488,7 +1488,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "js_int", "ruma-api", @@ -1502,7 +1502,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "js_int", "ruma-common", @@ -1517,7 +1517,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1528,7 +1528,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "js_int", "ruma-api", @@ -1543,7 +1543,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1555,7 +1555,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "proc-macro2", "quote", @@ -1566,7 +1566,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "serde", "strum", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "form_urlencoded", "itoa", @@ -1587,7 +1587,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "base64", "ring", @@ -1916,9 +1916,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.40" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350" +checksum = "6690e3e9f692504b941dc6c3b188fd28df054f7fb8469ab40680df52fdcc842b" dependencies = [ "proc-macro2", "quote", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 3db933c1..2ec9282f 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -303,7 +303,7 @@ pub fn whoami_route(body: Ruma) -> ConduitResult, body: Ruma>, ) -> ConduitResult { @@ -366,7 +366,7 @@ pub fn deactivate_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // Remove devices and mark account as deactivated diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 0ec43f57..c5c514e0 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -62,7 +62,7 @@ pub async fn get_alias_helper( ) -> ConduitResult { if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( - &db, + &db.globals, room_alias.server_name(), federation::query::get_room_information::v1::Request { room_alias }, ) diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 871a780f..372ce983 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -121,7 +121,7 @@ pub async fn get_public_rooms_filtered_helper( .filter(|server| *server != db.globals.server_name().as_str()) { let response = server_server::send_request( - &db, + &db.globals, other_server, federation::directory::get_public_rooms_filtered::v1::Request { limit, diff --git a/src/client_server/media.rs b/src/client_server/media.rs index f897a678..8f7a9b96 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -79,7 +79,7 @@ pub async fn get_content_route( .into()) } else if body.allow_remote { let get_content_response = server_server::send_request( - &db, + &db.globals, body.server_name.as_ref(), get_content::Request { allow_remote: false, @@ -129,7 +129,7 @@ pub async fn get_content_thumbnail_route( Ok(get_content_thumbnail::Response { file, content_type }.into()) } else if body.allow_remote { let get_thumbnail_response = server_server::send_request( - &db, + &db.globals, body.server_name.as_ref(), get_content_thumbnail::Request { allow_remote: false, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 8d194024..18fb5a9e 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -83,7 +83,7 @@ pub async fn join_room_by_id_or_alias_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/leave", data = "") )] -pub fn leave_room_route( +pub async fn leave_room_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -108,19 +108,21 @@ pub fn leave_room_route( event.membership = member::MembershipState::Leave; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(leave_room::Response::new().into()) } @@ -129,33 +131,35 @@ pub fn leave_room_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/invite", data = "") )] -pub fn invite_user_route( +pub async fn invite_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user_id)?, - avatar_url: db.users.avatar_url(&user_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user_id)?, + avatar_url: db.users.avatar_url(&user_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(invite_user::Response.into()) } else { @@ -167,7 +171,7 @@ pub fn invite_user_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/kick", data = "") )] -pub fn kick_user_route( +pub async fn kick_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -193,19 +197,21 @@ pub fn kick_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; // TODO: reason - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(kick_user::Response::new().into()) } @@ -214,7 +220,7 @@ pub fn kick_user_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/ban", data = "") )] -pub fn ban_user_route( +pub async fn ban_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -248,19 +254,21 @@ pub fn ban_user_route( }, )?; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(ban_user::Response::new().into()) } @@ -269,7 +277,7 @@ pub fn ban_user_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/unban", data = "") )] -pub fn unban_user_route( +pub async fn unban_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -294,19 +302,21 @@ pub fn unban_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(unban_user::Response::new().into()) } @@ -429,7 +439,7 @@ async fn join_room_by_id_helper( for remote_server in servers { let make_join_response = server_server::send_request( - &db, + &db.globals, remote_server, federation::membership::create_join_event_template::v1::Request { room_id, @@ -490,7 +500,7 @@ async fn join_room_by_id_helper( .expect("event is valid, we just created it"); let send_join_response = server_server::send_request( - &db, + &db.globals, remote_server, federation::membership::create_join_event::v2::Request { room_id, @@ -621,9 +631,12 @@ async fn join_room_by_id_helper( .expect("Found event_id in sorted events that is not in resolved state"); // We do not rebuild the PDU in this case only insert to DB - let pdu_id = - db.rooms - .append_pdu(&PduEvent::from(&**pdu), &db.globals, &db.account_data)?; + let pdu_id = db.rooms.append_pdu( + &PduEvent::from(&**pdu), + &serde_json::to_value(&**pdu).expect("PDU is valid value"), + &db.globals, + &db.account_data, + )?; if state_events.contains(ev_id) { state.insert( @@ -646,19 +659,22 @@ async fn join_room_by_id_helper( third_party_invite: None, }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.account_data, + ) + .await?; } Ok(join_room_by_id::Response::new(room_id.clone()).into()) diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 8a09aba5..4ba0d9fd 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -17,7 +17,7 @@ use rocket::{get, put}; feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/send/<_>/<_>", data = "") )] -pub fn send_message_event_route( +pub async fn send_message_event_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -67,7 +67,7 @@ pub fn send_message_event_route( &body.room_id, &db.globals, &db.account_data, - )?; + ).await?; db.transaction_ids .add_txnid(sender_id, device_id, &body.txn_id, event_id.as_bytes())?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index c1c0253e..be893e1b 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -19,7 +19,7 @@ use std::convert::TryInto; feature = "conduit_bin", put("/_matrix/client/r0/profile/<_>/displayname", data = "") )] -pub fn set_displayname_route( +pub async fn set_displayname_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -64,7 +64,7 @@ pub fn set_displayname_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // Presence update db.rooms.edus.update_presence( @@ -110,7 +110,7 @@ pub fn get_displayname_route( feature = "conduit_bin", put("/_matrix/client/r0/profile/<_>/avatar_url", data = "") )] -pub fn set_avatar_url_route( +pub async fn set_avatar_url_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -167,7 +167,7 @@ pub fn set_avatar_url_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // Presence update db.rooms.edus.update_presence( diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 51173489..701fc00d 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -12,7 +12,7 @@ use rocket::put; feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/redact/<_>/<_>", data = "") )] -pub fn redact_event_route( +pub async fn redact_event_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -33,7 +33,7 @@ pub fn redact_event_route( &body.room_id, &db.globals, &db.account_data, - )?; + ).await?; Ok(redact_event::Response { event_id }.into()) } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index a5280cf5..0e5c5716 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -20,7 +20,7 @@ use rocket::{get, post}; feature = "conduit_bin", post("/_matrix/client/r0/createRoom", data = "") )] -pub fn create_room_route( +pub async fn create_room_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -65,7 +65,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 2. Let the room creator join db.rooms.build_and_append_pdu( @@ -87,7 +87,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 3. Power levels let mut users = BTreeMap::new(); @@ -129,7 +129,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 4. Events set by preset @@ -162,7 +162,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 4.2 History Visibility db.rooms.build_and_append_pdu( @@ -180,7 +180,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 4.3 Guest Access db.rooms.build_and_append_pdu( @@ -206,7 +206,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 5. Events listed in initial_state for event in &body.initial_state { @@ -226,7 +226,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // 6. Events implied by name and topic @@ -248,7 +248,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } if let Some(topic) = &body.topic { @@ -267,7 +267,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // 7. Events implied by invite (and TODO: invite_3pid) @@ -291,7 +291,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // Homeserver specific stuff @@ -337,7 +337,7 @@ pub fn get_room_event_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_room_id>/upgrade", data = "") )] -pub fn upgrade_room_route( +pub async fn upgrade_room_route( db: State<'_, Database>, body: Ruma>, _room_id: String, @@ -379,7 +379,7 @@ pub fn upgrade_room_route( &body.room_id, &db.globals, &db.account_data, - )?; + ).await?; // Get the old room federations status let federate = serde_json::from_value::>( @@ -419,7 +419,7 @@ pub fn upgrade_room_route( &replacement_room, &db.globals, &db.account_data, - )?; + ).await?; // Join the new room db.rooms.build_and_append_pdu( @@ -441,7 +441,7 @@ pub fn upgrade_room_route( &replacement_room, &db.globals, &db.account_data, - )?; + ).await?; // Recommended transferable state events list from the specs let transferable_state_events = vec![ @@ -475,7 +475,7 @@ pub fn upgrade_room_route( &replacement_room, &db.globals, &db.account_data, - )?; + ).await?; } // Moves any local aliases to the new room @@ -505,7 +505,7 @@ pub fn upgrade_room_route( power_levels_event_content.invite = new_level; // Modify the power levels in the old room to prevent sending of events and inviting new users - db.rooms + let _ = db.rooms .build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, @@ -519,8 +519,7 @@ pub fn upgrade_room_route( &body.room_id, &db.globals, &db.account_data, - ) - .ok(); + ).await; // Return the replacement room id Ok(upgrade_room::Response { replacement_room }.into()) diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 1fe3cd6c..e9d20e2f 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -19,7 +19,7 @@ use rocket::{get, put}; feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/state/<_>/<_>", data = "") )] -pub fn send_state_event_for_key_route( +pub async fn send_state_event_for_key_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -41,7 +41,7 @@ pub fn send_state_event_for_key_route( content, &body.room_id, Some(body.state_key.to_owned()), - )?) + ).await?) .into(), ) } @@ -50,7 +50,7 @@ pub fn send_state_event_for_key_route( feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/state/<_>", data = "") )] -pub fn send_state_event_for_empty_key_route( +pub async fn send_state_event_for_empty_key_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -80,7 +80,7 @@ pub fn send_state_event_for_empty_key_route( json, &body.room_id, Some("".into()), - )?) + ).await?) .into(), ) } @@ -177,7 +177,7 @@ pub fn get_state_events_for_empty_key_route( .into()) } -pub fn send_state_event_for_key_helper( +pub async fn send_state_event_for_key_helper( db: &Database, sender: &UserId, content: &AnyStateEventContent, @@ -223,7 +223,7 @@ pub fn send_state_event_for_key_helper( &room_id, &db.globals, &db.account_data, - )?; + ).await?; Ok(event_id) } diff --git a/src/database.rs b/src/database.rs index 83f30c96..e1a356c7 100644 --- a/src/database.rs +++ b/src/database.rs @@ -109,6 +109,7 @@ impl Database { tokenids: db.open_tree("tokenids")?, + roomserverids: db.open_tree("roomserverids")?, userroomid_joined: db.open_tree("userroomid_joined")?, roomuserid_joined: db.open_tree("roomuserid_joined")?, roomuseroncejoinedids: db.open_tree("roomuseroncejoinedids")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index b538c854..ba54e7f2 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -2,11 +2,12 @@ mod edus; pub use edus::RoomEdus; -use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; +use crate::{pdu::PduBuilder, server_server, utils, Error, PduEvent, Result}; use log::error; use ring::digest; use ruma::{ api::client::error::ErrorKind, + api::federation, events::{ ignored_user_list, room::{ @@ -15,7 +16,7 @@ use ruma::{ }, EventType, }, - EventId, Raw, RoomAliasId, RoomId, UserId, + EventId, Raw, RoomAliasId, RoomId, ServerName, UserId, }; use sled::IVec; use state_res::{event_auth, Error as StateError, Requester, StateEvent, StateMap, StateStore}; @@ -25,6 +26,7 @@ use std::{ convert::{TryFrom, TryInto}, mem, sync::Arc, + time::SystemTime, }; /// The unique identifier of each state group. @@ -44,6 +46,8 @@ pub struct Rooms { pub(super) tokenids: sled::Tree, // TokenId = RoomId + Token + PduId + /// Participating servers in a room. + pub(super) roomserverids: sled::Tree, // RoomServerId = RoomId + ServerName pub(super) userroomid_joined: sled::Tree, pub(super) roomuserid_joined: sled::Tree, pub(super) roomuseroncejoinedids: sled::Tree, @@ -169,8 +173,7 @@ impl Rooms { Ok(events) } - // This fetches auth events from the current state using the - /// full `roomstateid_pdu` tree. + /// This fetches auth events from the current state. pub fn get_auth_events( &self, room_id: &RoomId, @@ -472,17 +475,10 @@ impl Rooms { pub fn append_pdu( &self, pdu: &PduEvent, + pdu_json: &serde_json::Value, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, ) -> Result> { - let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); - ruma::signatures::hash_and_sign_event( - globals.server_name().as_str(), - globals.keypair(), - &mut pdu_json, - ) - .expect("event is valid, we just created it"); - self.replace_pdu_leaves(&pdu.room_id, &pdu.event_id)?; // Increment the last index and use that @@ -610,7 +606,7 @@ impl Rooms { } /// Creates a new persisted data unit and adds it to a room. - pub fn build_and_append_pdu( + pub async fn build_and_append_pdu( &self, pdu_builder: PduBuilder, sender: &UserId, @@ -799,22 +795,59 @@ impl Rooms { signatures: BTreeMap::new(), }; + // Hash and sign + let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + ruma::signatures::hash_and_sign_event( + globals.server_name().as_str(), + globals.keypair(), + &mut pdu_json, + ) + .expect("event is valid, we just created it"); + // Generate event id pdu.event_id = EventId::try_from(&*format!( "${}", - ruma::signatures::reference_hash( - &serde_json::to_value(&pdu).expect("event is valid, we just created it") - ) - .expect("ruma can calculate reference hashes") + ruma::signatures::reference_hash(&pdu_json) + .expect("ruma can calculate reference hashes") )) .expect("ruma's reference hashes are valid event ids"); - let pdu_id = self.append_pdu(&pdu, globals, account_data)?; + pdu_json + .as_object_mut() + .expect("json is object") + .insert("event_id".to_owned(), pdu.event_id.to_string().into()); + + let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data)?; if pdu.state_key.is_some() { self.append_to_state(&pdu_id, &pdu)?; } + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + let response = server_server::send_request( + &globals, + "koesters.xyz".try_into().unwrap(), + federation::transactions::send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus: &[serde_json::from_value(pdu_json).expect("Raw::from_value always works")], + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &utils::random_string(16), + }, + ) + .await; + + let _ = dbg!(response); + Ok(pdu.event_id) } @@ -957,12 +990,17 @@ impl Rooms { &self, room_id: &RoomId, user_id: &UserId, - mut member_content: member::MemberEventContent, + member_content: member::MemberEventContent, sender: &UserId, account_data: &super::account_data::AccountData, globals: &super::globals::Globals, ) -> Result<()> { let membership = member_content.membership; + + let mut roomserver_id = room_id.as_bytes().to_vec(); + roomserver_id.push(0xff); + roomserver_id.extend_from_slice(user_id.server_name().as_bytes()); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -1056,6 +1094,7 @@ impl Rooms { } } + self.roomserverids.insert(&roomserver_id, &[])?; self.userroomid_joined.insert(&userroom_id, &[])?; self.roomuserid_joined.insert(&roomuser_id, &[])?; self.userroomid_invited.remove(&userroom_id)?; @@ -1075,25 +1114,10 @@ impl Rooms { }); if is_ignored { - member_content.membership = member::MembershipState::Leave; - - self.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - &user_id, - &room_id, - globals, - account_data, - )?; - return Ok(()); } + + self.roomserverids.insert(&roomserver_id, &[])?; self.userroomid_invited.insert(&userroom_id, &[])?; self.roomuserid_invited.insert(&roomuser_id, &[])?; self.userroomid_joined.remove(&userroom_id)?; @@ -1101,6 +1125,14 @@ impl Rooms { self.userroomid_left.remove(&userroom_id)?; } member::MembershipState::Leave | member::MembershipState::Ban => { + if self + .room_members(room_id) + .chain(self.room_members_invited(room_id)) + .filter_map(|r| r.ok()) + .all(|u| u.server_name() != user_id.server_name()) + { + self.roomserverids.remove(&roomserver_id)?; + } self.userroomid_left.insert(&userroom_id, &[])?; self.userroomid_joined.remove(&userroom_id)?; self.roomuserid_joined.remove(&roomuser_id)?; @@ -1294,10 +1326,34 @@ impl Rooms { }) } + /// Returns an iterator over all joined members of a room. + pub fn room_servers(&self, room_id: &RoomId) -> impl Iterator>> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + + self.roomserverids.scan_prefix(prefix).keys().map(|key| { + Ok(Box::::try_from( + utils::string_from_bytes( + &key? + .rsplit(|&b| b == 0xff) + .next() + .expect("rsplit always returns an element"), + ) + .map_err(|_| { + Error::bad_database("Server name in roomserverids is invalid unicode.") + })?, + ) + .map_err(|_| Error::bad_database("Server name in roomserverids is invalid."))?) + }) + } + /// Returns an iterator over all joined members of a room. pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + self.roomuserid_joined - .scan_prefix(room_id.as_bytes()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(UserId::try_from( @@ -1317,8 +1373,11 @@ impl Rooms { /// Returns an iterator over all User IDs who ever joined a room. pub fn room_useroncejoined(&self, room_id: &RoomId) -> impl Iterator> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + self.roomuseroncejoinedids - .scan_prefix(room_id.to_string()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(UserId::try_from( @@ -1338,8 +1397,11 @@ impl Rooms { /// Returns an iterator over all invited members of a room. pub fn room_members_invited(&self, room_id: &RoomId) -> impl Iterator> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + self.roomuserid_invited - .scan_prefix(room_id.as_bytes()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(UserId::try_from( @@ -1380,8 +1442,11 @@ impl Rooms { /// Returns an iterator over all rooms a user was invited to. pub fn rooms_invited(&self, user_id: &UserId) -> impl Iterator> { + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xff); + self.userroomid_invited - .scan_prefix(&user_id.as_bytes()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1401,23 +1466,23 @@ impl Rooms { /// Returns an iterator over all rooms a user left. pub fn rooms_left(&self, user_id: &UserId) -> impl Iterator> { - self.userroomid_left - .scan_prefix(&user_id.as_bytes()) - .keys() - .map(|key| { - Ok(RoomId::try_from( - utils::string_from_bytes( - &key? - .rsplit(|&b| b == 0xff) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| { - Error::bad_database("Room ID in userroomid_left is invalid unicode.") - })?, + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xff); + + self.userroomid_left.scan_prefix(prefix).keys().map(|key| { + Ok(RoomId::try_from( + utils::string_from_bytes( + &key? + .rsplit(|&b| b == 0xff) + .next() + .expect("rsplit always returns an element"), ) - .map_err(|_| Error::bad_database("Room ID in userroomid_left is invalid."))?) - }) + .map_err(|_| { + Error::bad_database("Room ID in userroomid_left is invalid unicode.") + })?, + ) + .map_err(|_| Error::bad_database("Room ID in userroomid_left is invalid."))?) + }) } pub fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { diff --git a/src/pdu.rs b/src/pdu.rs index 7f842e2d..c9042306 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -6,7 +6,7 @@ use ruma::{ AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, EventId, Raw, RoomId, ServerKeyId, ServerName, UserId, -}; +events::pdu::PduStub}; use serde::{Deserialize, Serialize}; use serde_json::json; use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::UNIX_EPOCH}; @@ -198,6 +198,31 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } + + pub fn to_outgoing_federation_event(&self) -> Raw { + let mut json = json!({ + "room_id": self.room_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "type": self.kind, + "content": self.content, + "prev_events": self.prev_events, + "depth": self.depth, + "auth_events": self.auth_events, + "unsigned": self.unsigned, + "hashes": self.hashes, + "signatures": self.signatures, + }); + + if let Some(state_key) = &self.state_key { + json["state_key"] = json!(state_key); + } + if let Some(redacts) = &self.redacts { + json["redacts"] = json!(redacts); + } + + serde_json::from_value(json).expect("Raw::from_value always works") + } } impl From<&state_res::StateEvent> for PduEvent { diff --git a/src/server_server.rs b/src/server_server.rs index 6c53aed2..9f4be132 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -24,9 +24,9 @@ use std::{ time::{Duration, SystemTime}, }; -pub async fn request_well_known(db: &crate::Database, destination: &str) -> Option { +pub async fn request_well_known(globals: &crate::database::globals::Globals, destination: &str) -> Option { let body: serde_json::Value = serde_json::from_str( - &db.globals + &globals .reqwest_client() .get(&format!( "https://{}/.well-known/matrix/server", @@ -44,7 +44,7 @@ pub async fn request_well_known(db: &crate::Database, destination: &str) -> Opti } pub async fn send_request( - db: &crate::Database, + globals: &crate::database::globals::Globals, destination: &ServerName, request: T, ) -> Result @@ -52,7 +52,7 @@ where T: Debug, { let actual_destination = "https://".to_owned() - + &request_well_known(db, &destination.as_str()) + + &request_well_known(globals, &destination.as_str()) .await .unwrap_or(destination.as_str().to_owned() + ":8448"); @@ -81,14 +81,14 @@ where ); request_map.insert( "origin".to_owned(), - db.globals.server_name().as_str().into(), + globals.server_name().as_str().into(), ); request_map.insert("destination".to_owned(), destination.as_str().into()); let mut request_json = request_map.into(); ruma::signatures::sign_json( - db.globals.server_name().as_str(), - db.globals.keypair(), + globals.server_name().as_str(), + globals.keypair(), &mut request_json, ) .unwrap(); @@ -110,7 +110,7 @@ where AUTHORIZATION, HeaderValue::from_str(&format!( "X-Matrix origin={},key=\"{}\",sig=\"{}\"", - db.globals.server_name(), + globals.server_name(), s.0, s.1 )) @@ -122,7 +122,7 @@ where let reqwest_request = reqwest::Request::try_from(http_request) .expect("all http requests are valid reqwest requests"); - let reqwest_response = db.globals.reqwest_client().execute(reqwest_request).await; + let reqwest_response = globals.reqwest_client().execute(reqwest_request).await; // Because reqwest::Response -> http::Response is complicated: match reqwest_response { @@ -317,9 +317,9 @@ pub fn send_transaction_message_route<'a>( .insert("event_id".to_owned(), event_id.to_string().into()); let pdu = - serde_json::from_value::(value).expect("all ruma pdus are conduit pdus"); + serde_json::from_value::(value.clone()).expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - db.rooms.append_pdu(&pdu, &db.globals, &db.account_data)?; + db.rooms.append_pdu(&pdu, &value, &db.globals, &db.account_data)?; } } Ok(send_transaction_message::v1::Response { From 71500b14b902321e91cab432d55dd3f3ae7aedfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 08:16:20 +0200 Subject: [PATCH 29/51] fix: send to all servers and fix media store --- src/client_server/alias.rs | 2 +- src/client_server/directory.rs | 2 +- src/client_server/media.rs | 38 ++++++++-------------------- src/client_server/membership.rs | 4 +-- src/database/rooms.rs | 44 ++++++++++++++++++++++----------- src/server_server.rs | 29 ++++++++++++---------- 6 files changed, 60 insertions(+), 59 deletions(-) diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index c5c514e0..c2c3eb9c 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -63,7 +63,7 @@ pub async fn get_alias_helper( if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( &db.globals, - room_alias.server_name(), + room_alias.server_name().to_owned(), federation::query::get_room_information::v1::Request { room_alias }, ) .await?; diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 372ce983..c82a15f0 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -122,7 +122,7 @@ pub async fn get_public_rooms_filtered_helper( { let response = server_server::send_request( &db.globals, - other_server, + other_server.to_owned(), federation::directory::get_public_rooms_filtered::v1::Request { limit, since: since.as_deref(), diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 8f7a9b96..8a93d492 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -48,22 +48,13 @@ pub fn create_content_route( #[cfg_attr( feature = "conduit_bin", - get( - "/_matrix/media/r0/download/<_server_name>/<_media_id>", - data = "" - ) + get("/_matrix/media/r0/download/<_>/<_>", data = "") )] pub async fn get_content_route( db: State<'_, Database>, body: Ruma>, - _server_name: String, - _media_id: String, ) -> ConduitResult { - let mxc = format!( - "mxc://{}/{}", - db.globals.server_name(), - utils::random_string(MXC_LENGTH) - ); + let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); if let Some(FileMeta { filename, @@ -77,10 +68,10 @@ pub async fn get_content_route( content_disposition: filename.unwrap_or_default(), // TODO: Spec says this should be optional } .into()) - } else if body.allow_remote { + } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_content_response = server_server::send_request( &db.globals, - body.server_name.as_ref(), + body.server_name.clone(), get_content::Request { allow_remote: false, server_name: &body.server_name, @@ -104,21 +95,18 @@ pub async fn get_content_route( #[cfg_attr( feature = "conduit_bin", - get( - "/_matrix/media/r0/thumbnail/<_server_name>/<_media_id>", - data = "" - ) + get("/_matrix/media/r0/thumbnail/<_>/<_>", data = "") )] pub async fn get_content_thumbnail_route( db: State<'_, Database>, body: Ruma>, - _server_name: String, - _media_id: String, ) -> ConduitResult { + let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); + if let Some(FileMeta { content_type, file, .. }) = db.media.get_thumbnail( - format!("mxc://{}/{}", body.server_name, body.media_id), + mxc.clone(), body.width .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, @@ -127,10 +115,10 @@ pub async fn get_content_thumbnail_route( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, )? { Ok(get_content_thumbnail::Response { file, content_type }.into()) - } else if body.allow_remote { + } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_thumbnail_response = server_server::send_request( &db.globals, - body.server_name.as_ref(), + body.server_name.clone(), get_content_thumbnail::Request { allow_remote: false, height: body.height, @@ -142,12 +130,6 @@ pub async fn get_content_thumbnail_route( ) .await?; - let mxc = format!( - "mxc://{}/{}", - db.globals.server_name(), - utils::random_string(MXC_LENGTH) - ); - db.media.upload_thumbnail( mxc, &None, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 18fb5a9e..f60601fd 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -440,7 +440,7 @@ async fn join_room_by_id_helper( for remote_server in servers { let make_join_response = server_server::send_request( &db.globals, - remote_server, + remote_server.clone(), federation::membership::create_join_event_template::v1::Request { room_id, user_id: sender_id, @@ -501,7 +501,7 @@ async fn join_room_by_id_helper( let send_join_response = server_server::send_request( &db.globals, - remote_server, + remote_server.clone(), federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index ba54e7f2..3c3a0b27 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1,9 +1,10 @@ mod edus; pub use edus::RoomEdus; +use rocket::futures; use crate::{pdu::PduBuilder, server_server, utils, Error, PduEvent, Result}; -use log::error; +use log::{error, warn}; use ring::digest; use ruma::{ api::client::error::ErrorKind, @@ -833,20 +834,35 @@ impl Rooms { .expect("json is object") .remove("event_id"); - let response = server_server::send_request( - &globals, - "koesters.xyz".try_into().unwrap(), - federation::transactions::send_transaction_message::v1::Request { - origin: globals.server_name(), - pdus: &[serde_json::from_value(pdu_json).expect("Raw::from_value always works")], - edus: &[], - origin_server_ts: SystemTime::now(), - transaction_id: &utils::random_string(16), - }, - ) - .await; + let raw_json = + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); + + let pdus = &[raw_json]; + let transaction_id = utils::random_string(16); - let _ = dbg!(response); + for result in futures::future::join_all( + self.room_servers(room_id) + .filter_map(|r| r.ok()) + .filter(|server| &**server != globals.server_name()) + .map(|server| { + server_server::send_request( + &globals, + server, + federation::transactions::send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus, + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &transaction_id, + }, + ) + }), + ) + .await { + if let Err(e) = result { + warn!("{}", e); + } + } Ok(pdu.event_id) } diff --git a/src/server_server.rs b/src/server_server.rs index 9f4be132..da5a6c1e 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,5 +1,6 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; +use log::warn; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ api::federation::directory::get_public_rooms_filtered, @@ -24,7 +25,10 @@ use std::{ time::{Duration, SystemTime}, }; -pub async fn request_well_known(globals: &crate::database::globals::Globals, destination: &str) -> Option { +pub async fn request_well_known( + globals: &crate::database::globals::Globals, + destination: &str, +) -> Option { let body: serde_json::Value = serde_json::from_str( &globals .reqwest_client() @@ -45,7 +49,7 @@ pub async fn request_well_known(globals: &crate::database::globals::Globals, des pub async fn send_request( globals: &crate::database::globals::Globals, - destination: &ServerName, + destination: Box, request: T, ) -> Result where @@ -79,10 +83,7 @@ where .to_string() .into(), ); - request_map.insert( - "origin".to_owned(), - globals.server_name().as_str().into(), - ); + request_map.insert("origin".to_owned(), globals.server_name().as_str().into()); request_map.insert("destination".to_owned(), destination.as_str().into()); let mut request_json = request_map.into(); @@ -144,10 +145,11 @@ where .into_iter() .collect(); - Ok( - T::IncomingResponse::try_from(http_response.body(body).unwrap()) - .expect("TODO: error handle other server errors"), - ) + let response = T::IncomingResponse::try_from(http_response.body(body).unwrap()); + response.map_err(|e| { + warn!("{}", e); + Error::BadServerResponse("Server returned bad response.") + }) } Err(e) => Err(e.into()), } @@ -316,10 +318,11 @@ pub fn send_transaction_message_route<'a>( .expect("ruma pdus are json objects") .insert("event_id".to_owned(), event_id.to_string().into()); - let pdu = - serde_json::from_value::(value.clone()).expect("all ruma pdus are conduit pdus"); + let pdu = serde_json::from_value::(value.clone()) + .expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - db.rooms.append_pdu(&pdu, &value, &db.globals, &db.account_data)?; + db.rooms + .append_pdu(&pdu, &value, &db.globals, &db.account_data)?; } } Ok(send_transaction_message::v1::Response { From 0b263208e39a735fcb6970d168494783ff9994a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 08:55:02 +0200 Subject: [PATCH 30/51] fix: don't panic on bad server names --- src/server_server.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index da5a6c1e..40ad654f 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -62,14 +62,18 @@ where let mut http_request = request .try_into_http_request(&actual_destination, Some("")) - .unwrap(); + .map_err(|e| { + warn!("{}: {}", actual_destination, e); + Error::BadServerResponse("Invalid destination") + })?; let mut request_map = serde_json::Map::new(); if !http_request.body().is_empty() { request_map.insert( "content".to_owned(), - serde_json::from_slice(http_request.body()).unwrap(), + serde_json::from_slice(http_request.body()) + .expect("body is valid json, we just created it"), ); }; @@ -92,7 +96,7 @@ where globals.keypair(), &mut request_json, ) - .unwrap(); + .expect("our request json is what ruma expects"); let signatures = request_json["signatures"] .as_object() @@ -145,7 +149,11 @@ where .into_iter() .collect(); - let response = T::IncomingResponse::try_from(http_response.body(body).unwrap()); + let response = T::IncomingResponse::try_from( + http_response + .body(body) + .expect("reqwest body is valid http body"), + ); response.map_err(|e| { warn!("{}", e); Error::BadServerResponse("Server returned bad response.") From b7ab57897bc96e468421cf82ecd7d49e75c3f7dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 16:13:54 +0200 Subject: [PATCH 31/51] fix: sending slowness --- Cargo.lock | 2 +- src/client_server/account.rs | 30 +- src/client_server/membership.rs | 6 + src/client_server/message.rs | 42 +-- src/client_server/profile.rs | 138 +++++---- src/client_server/redact.rs | 36 ++- src/client_server/room.rs | 512 ++++++++++++++++++-------------- src/client_server/state.rs | 48 +-- src/database.rs | 5 + src/database/globals.rs | 21 +- src/database/rooms.rs | 58 ++-- src/database/rooms/edus.rs | 1 + src/database/sending.rs | 83 ++++++ src/main.rs | 2 + src/pdu.rs | 3 +- 15 files changed, 572 insertions(+), 415 deletions(-) create mode 100644 src/database/sending.rs diff --git a/Cargo.lock b/Cargo.lock index e0de2a7b..30144ca5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1831,7 +1831,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#a9186476b748c901fbf4356414247a0b3ac01b5f" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#1d01b6e65b6afd50e65085fb40f1e7d2782f519e" dependencies = [ "itertools", "js_int", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 2ec9282f..7e0f942e 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -354,19 +354,23 @@ pub async fn deactivate_route( third_party_invite: None, }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // Remove devices and mark account as deactivated diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index f60601fd..c4eed95c 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -120,6 +120,7 @@ pub async fn leave_room_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -157,6 +158,7 @@ pub async fn invite_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -209,6 +211,7 @@ pub async fn kick_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -266,6 +269,7 @@ pub async fn ban_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -314,6 +318,7 @@ pub async fn unban_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -672,6 +677,7 @@ async fn join_room_by_id_helper( &sender_id, &room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 4ba0d9fd..3944d5bd 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -49,25 +49,29 @@ pub async fn send_message_event_route( let mut unsigned = serde_json::Map::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into()); - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: body.content.event_type().into(), - content: serde_json::from_str( - body.json_body - .as_ref() - .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? - .get(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, - unsigned: Some(unsigned), - state_key: None, - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - ).await?; + let event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: body.content.event_type().into(), + content: serde_json::from_str( + body.json_body + .as_ref() + .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? + .get(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, + unsigned: Some(unsigned), + state_key: None, + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; db.transaction_ids .add_txnid(sender_id, device_id, &body.txn_id, event_id.as_bytes())?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index be893e1b..53893c0f 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -31,40 +31,43 @@ pub async fn set_displayname_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - displayname: body.displayname.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + displayname: body.displayname.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( "Tried to send displayname update for user not in the room.", ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Presence update db.rooms.edus.update_presence( @@ -134,40 +137,43 @@ pub async fn set_avatar_url_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - avatar_url: body.avatar_url.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( - "Tried to send avatar url update for user not in the room.", - ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + avatar_url: body.avatar_url.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( + "Tried to send avatar url update for user not in the room.", + ) + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Presence update db.rooms.edus.update_presence( diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 701fc00d..24df8dd7 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -18,22 +18,26 @@ pub async fn redact_event_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomRedaction, - content: serde_json::to_value(redaction::RedactionEventContent { - reason: body.reason.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: Some(body.event_id.clone()), - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - ).await?; + let event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomRedaction, + content: serde_json::to_value(redaction::RedactionEventContent { + reason: body.reason.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: Some(body.event_id.clone()), + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; Ok(redact_event::Response { event_id }.into()) } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 0e5c5716..d21148bb 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -53,41 +53,47 @@ pub async fn create_room_route( content.room_version = RoomVersionId::Version6; // 1. The room create event - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 2. Let the room creator join - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 3. Power levels let mut users = BTreeMap::new(); @@ -117,19 +123,22 @@ pub async fn create_room_route( }) .expect("event is valid, we just created it") }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomPowerLevels, - content: power_levels_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: power_levels_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 4. Events set by preset @@ -140,104 +149,42 @@ pub async fn create_room_route( }); // 4.1 Join Rules - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomJoinRules, - content: match preset { - create_room::RoomPreset::PublicChat => serde_json::to_value( - join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), - ) - .expect("event is valid, we just created it"), - // according to spec "invite" is the default - _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( - join_rules::JoinRule::Invite, - )) - .expect("event is valid, we just created it"), - }, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; - - // 4.2 History Visibility - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomHistoryVisibility, - content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new( - history_visibility::HistoryVisibility::Shared, - )) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; - - // 4.3 Guest Access - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomGuestAccess, - content: match preset { - create_room::RoomPreset::PublicChat => { - serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::Forbidden, + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomJoinRules, + content: match preset { + create_room::RoomPreset::PublicChat => serde_json::to_value( + join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), + ) + .expect("event is valid, we just created it"), + // according to spec "invite" is the default + _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( + join_rules::JoinRule::Invite, )) - .expect("event is valid, we just created it") - } - _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::CanJoin, - )) - .expect("event is valid, we just created it"), + .expect("event is valid, we just created it"), + }, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, }, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; - - // 5. Events listed in initial_state - for event in &body.initial_state { - let pdu_builder = serde_json::from_str::( - &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"), - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; - - // Silently skip encryption events if they are not allowed - if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() { - continue; - } - - db.rooms.build_and_append_pdu( - pdu_builder, &sender_id, &room_id, &db.globals, + &db.sending, &db.account_data, - ).await?; - } + ) + .await?; - // 6. Events implied by name and topic - if let Some(name) = &body.name { - db.rooms.build_and_append_pdu( + // 4.2 History Visibility + db.rooms + .build_and_append_pdu( PduBuilder { - event_type: EventType::RoomName, + event_type: EventType::RoomHistoryVisibility, content: serde_json::to_value( - name::NameEventContent::new(name.clone()).map_err(|_| { - Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") - })?, + history_visibility::HistoryVisibilityEventContent::new( + history_visibility::HistoryVisibility::Shared, + ), ) .expect("event is valid, we just created it"), unsigned: None, @@ -247,18 +194,28 @@ pub async fn create_room_route( &sender_id, &room_id, &db.globals, + &db.sending, &db.account_data, - ).await?; - } + ) + .await?; - if let Some(topic) = &body.topic { - db.rooms.build_and_append_pdu( + // 4.3 Guest Access + db.rooms + .build_and_append_pdu( PduBuilder { - event_type: EventType::RoomTopic, - content: serde_json::to_value(topic::TopicEventContent { - topic: topic.clone(), - }) - .expect("event is valid, we just created it"), + event_type: EventType::RoomGuestAccess, + content: match preset { + create_room::RoomPreset::PublicChat => { + serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::Forbidden, + )) + .expect("event is valid, we just created it") + } + _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::CanJoin, + )) + .expect("event is valid, we just created it"), + }, unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -266,32 +223,107 @@ pub async fn create_room_route( &sender_id, &room_id, &db.globals, + &db.sending, &db.account_data, - ).await?; + ) + .await?; + + // 5. Events listed in initial_state + for event in &body.initial_state { + let pdu_builder = serde_json::from_str::( + &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"), + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; + + // Silently skip encryption events if they are not allowed + if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() { + continue; + } + + db.rooms + .build_and_append_pdu( + pdu_builder, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; + } + + // 6. Events implied by name and topic + if let Some(name) = &body.name { + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomName, + content: serde_json::to_value( + name::NameEventContent::new(name.clone()).map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") + })?, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; + } + + if let Some(topic) = &body.topic { + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTopic, + content: serde_json::to_value(topic::TopicEventContent { + topic: topic.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // 7. Events implied by invite (and TODO: invite_3pid) for user in &body.invite { - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user)?, - avatar_url: db.users.avatar_url(&user)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user)?, + avatar_url: db.users.avatar_url(&user)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // Homeserver specific stuff @@ -363,23 +395,29 @@ pub async fn upgrade_room_route( // Send a m.room.tombstone event to the old room to indicate that it is not intended to be used any further // Fail if the sender does not have the required permissions - let tombstone_event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTombstone, - content: serde_json::to_value(ruma::events::room::tombstone::TombstoneEventContent { - body: "This room has been replaced".to_string(), - replacement_room: replacement_room.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &body.room_id, - &db.globals, - &db.account_data, - ).await?; + let tombstone_event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTombstone, + content: serde_json::to_value( + ruma::events::room::tombstone::TombstoneEventContent { + body: "This room has been replaced".to_string(), + replacement_room: replacement_room.clone(), + }, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Get the old room federations status let federate = serde_json::from_value::>( @@ -406,42 +444,48 @@ pub async fn upgrade_room_route( create_event_content.room_version = new_version; create_event_content.predecessor = predecessor; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(create_event_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(create_event_content) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Join the new room - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Recommended transferable state events list from the specs let transferable_state_events = vec![ @@ -463,19 +507,22 @@ pub async fn upgrade_room_route( None => continue, // Skipping missing events. }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type, - content: event_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type, + content: event_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // Moves any local aliases to the new room @@ -505,7 +552,8 @@ pub async fn upgrade_room_route( power_levels_event_content.invite = new_level; // Modify the power levels in the old room to prevent sending of events and inviting new users - let _ = db.rooms + let _ = db + .rooms .build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, @@ -518,8 +566,10 @@ pub async fn upgrade_room_route( sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, - ).await; + ) + .await; // Return the replacement room id Ok(upgrade_room::Response { replacement_room }.into()) diff --git a/src/client_server/state.rs b/src/client_server/state.rs index e9d20e2f..46182a12 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -33,17 +33,18 @@ pub async fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - Ok( - send_state_event_for_key::Response::new(send_state_event_for_key_helper( + Ok(send_state_event_for_key::Response::new( + send_state_event_for_key_helper( &db, sender_id, &body.content, content, &body.room_id, Some(body.state_key.to_owned()), - ).await?) - .into(), + ) + .await?, ) + .into()) } #[cfg_attr( @@ -70,8 +71,8 @@ pub async fn send_state_event_for_empty_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - Ok( - send_state_event_for_empty_key::Response::new(send_state_event_for_key_helper( + Ok(send_state_event_for_empty_key::Response::new( + send_state_event_for_key_helper( &db, sender_id .as_ref() @@ -80,9 +81,10 @@ pub async fn send_state_event_for_empty_key_route( json, &body.room_id, Some("".into()), - ).await?) - .into(), + ) + .await?, ) + .into()) } #[cfg_attr( @@ -211,19 +213,23 @@ pub async fn send_state_event_for_key_helper( } } - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: content.event_type().into(), - content: json, - unsigned: None, - state_key, - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + let event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: content.event_type().into(), + content: json, + unsigned: None, + state_key, + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; Ok(event_id) } diff --git a/src/database.rs b/src/database.rs index e1a356c7..4b2cba10 100644 --- a/src/database.rs +++ b/src/database.rs @@ -3,6 +3,7 @@ pub mod globals; pub mod key_backups; pub mod media; pub mod rooms; +pub mod sending; pub mod transaction_ids; pub mod uiaa; pub mod users; @@ -25,6 +26,7 @@ pub struct Database { pub media: media::Media, pub key_backups: key_backups::KeyBackups, pub transaction_ids: transaction_ids::TransactionIds, + pub sending: sending::Sending, pub _db: sled::Db, } @@ -135,6 +137,9 @@ impl Database { transaction_ids: transaction_ids::TransactionIds { userdevicetxnid_response: db.open_tree("userdevicetxnid_response")?, }, + sending: sending::Sending { + serverpduids: db.open_tree("serverpduids")?, + }, _db: db, }) } diff --git a/src/database/globals.rs b/src/database/globals.rs index 5db28069..89514251 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -1,12 +1,13 @@ use crate::{utils, Error, Result}; use ruma::ServerName; -use std::convert::TryInto; +use std::{convert::TryInto, sync::Arc}; pub const COUNTER: &str = "c"; +#[derive(Clone)] pub struct Globals { pub(super) globals: sled::Tree, - keypair: ruma::signatures::Ed25519KeyPair, + keypair: Arc, reqwest_client: reqwest::Client, server_name: Box, max_request_size: u32, @@ -16,13 +17,15 @@ pub struct Globals { impl Globals { pub fn load(globals: sled::Tree, config: &rocket::Config) -> Result { - let keypair = ruma::signatures::Ed25519KeyPair::new( - &*globals - .update_and_fetch("keypair", utils::generate_keypair)? - .expect("utils::generate_keypair always returns Some"), - "key1".to_owned(), - ) - .map_err(|_| Error::bad_database("Private or public keys are invalid."))?; + let keypair = Arc::new( + ruma::signatures::Ed25519KeyPair::new( + &*globals + .update_and_fetch("keypair", utils::generate_keypair)? + .expect("utils::generate_keypair always returns Some"), + "key1".to_owned(), + ) + .map_err(|_| Error::bad_database("Private or public keys are invalid."))?, + ); Ok(Self { globals, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 3c3a0b27..2246a61f 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1,14 +1,12 @@ mod edus; pub use edus::RoomEdus; -use rocket::futures; -use crate::{pdu::PduBuilder, server_server, utils, Error, PduEvent, Result}; -use log::{error, warn}; +use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; +use log::error; use ring::digest; use ruma::{ api::client::error::ErrorKind, - api::federation, events::{ ignored_user_list, room::{ @@ -27,7 +25,6 @@ use std::{ convert::{TryFrom, TryInto}, mem, sync::Arc, - time::SystemTime, }; /// The unique identifier of each state group. @@ -36,6 +33,7 @@ use std::{ /// hashing the entire state. pub type StateHashId = Vec; +#[derive(Clone)] pub struct Rooms { pub edus: edus::RoomEdus, pub(super) pduid_pdu: sled::Tree, // PduId = RoomId + Count @@ -415,6 +413,16 @@ impl Rooms { }) } + /// Returns the pdu. + pub fn get_pdu_json_from_id(&self, pdu_id: &IVec) -> Result> { + self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| { + Ok(Some( + serde_json::from_slice(&pdu) + .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + )) + }) + } + /// Removes a pdu and creates a new one with the same id. fn replace_pdu(&self, pdu_id: &IVec, pdu: &PduEvent) -> Result<()> { if self.pduid_pdu.get(&pdu_id)?.is_some() { @@ -613,6 +621,7 @@ impl Rooms { sender: &UserId, room_id: &RoomId, globals: &super::globals::Globals, + sending: &super::sending::Sending, account_data: &super::account_data::AccountData, ) -> Result { let PduBuilder { @@ -829,39 +838,12 @@ impl Rooms { self.append_to_state(&pdu_id, &pdu)?; } - pdu_json - .as_object_mut() - .expect("json is object") - .remove("event_id"); - - let raw_json = - serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); - - let pdus = &[raw_json]; - let transaction_id = utils::random_string(16); - - for result in futures::future::join_all( - self.room_servers(room_id) - .filter_map(|r| r.ok()) - .filter(|server| &**server != globals.server_name()) - .map(|server| { - server_server::send_request( - &globals, - server, - federation::transactions::send_transaction_message::v1::Request { - origin: globals.server_name(), - pdus, - edus: &[], - origin_server_ts: SystemTime::now(), - transaction_id: &transaction_id, - }, - ) - }), - ) - .await { - if let Err(e) = result { - warn!("{}", e); - } + for server in self + .room_servers(room_id) + .filter_map(|r| r.ok()) + .filter(|server| &**server != globals.server_name()) + { + sending.send_pdu(server, &pdu_id)?; } Ok(pdu.event_id) diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index d60e1f16..a794c690 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -13,6 +13,7 @@ use std::{ convert::{TryFrom, TryInto}, }; +#[derive(Clone)] pub struct RoomEdus { pub(in super::super) readreceiptid_readreceipt: sled::Tree, // ReadReceiptId = RoomId + Count + UserId pub(in super::super) roomuserid_privateread: sled::Tree, // RoomUserId = Room + User, PrivateRead = Count diff --git a/src/database/sending.rs b/src/database/sending.rs new file mode 100644 index 00000000..187fd575 --- /dev/null +++ b/src/database/sending.rs @@ -0,0 +1,83 @@ +use std::{convert::TryFrom, time::SystemTime}; + +use crate::{server_server, utils, Error, Result}; +use rocket::futures::stream::{FuturesUnordered, StreamExt}; +use ruma::{api::federation, Raw, ServerName}; +use tokio::select; + +pub struct Sending { + /// The state for a given state hash. + pub(super) serverpduids: sled::Tree, // ServerPduId = ServerName + PduId +} + +impl Sending { + pub fn start_handler(&self, globals: &super::globals::Globals, rooms: &super::rooms::Rooms) { + let serverpduids = self.serverpduids.clone(); + let rooms = rooms.clone(); + let globals = globals.clone(); + + tokio::spawn(async move { + let mut futures = FuturesUnordered::new(); + let mut subscriber = serverpduids.watch_prefix(b""); + loop { + select! { + Some(_) = futures.next() => {}, + Some(event) = &mut subscriber => { + let serverpduid = if let sled::Event::Insert {key, ..} = event { + key + } else + { return Err::<(), Error>(Error::bad_database("")); }; + let mut parts = serverpduid.splitn(2, |&b| b == 0xff); + let server = Box::::try_from( + utils::string_from_bytes(parts.next().expect("splitn will always return 1 or more elements")) + .map_err(|_| Error::bad_database("ServerName in serverpduid bytes are invalid."))? + ).map_err(|_| Error::bad_database("ServerName in serverpduid is invalid."))?; + + let pdu_id = parts.next().ok_or_else(|| Error::bad_database("Invalid serverpduid in db."))?; + let mut pdu_json = rooms.get_pdu_json_from_id(&pdu_id.into())?.ok_or_else(|| Error::bad_database("Event in serverpduids not found in db."))?; + + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + let raw_json = + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); + + let globals = &globals; + + futures.push( + async move { + let pdus = vec![raw_json]; + let transaction_id = utils::random_string(16); + + server_server::send_request( + &globals, + server, + federation::transactions::send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus: &pdus, + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &transaction_id, + }, + ).await + } + ); + }, + } + } + }); + } + /* + */ + + pub fn send_pdu(&self, server: Box, pdu_id: &[u8]) -> Result<()> { + let mut key = server.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(pdu_id); + self.serverpduids.insert(key, b"")?; + + Ok(()) + } +} diff --git a/src/main.rs b/src/main.rs index eb060e3e..2817ab97 100644 --- a/src/main.rs +++ b/src/main.rs @@ -130,6 +130,8 @@ fn setup_rocket() -> rocket::Rocket { .attach(AdHoc::on_attach("Config", |mut rocket| async { let data = Database::load_or_create(rocket.config().await).expect("valid config"); + data.sending.start_handler(&data.globals, &data.rooms); + Ok(rocket.manage(data)) })) } diff --git a/src/pdu.rs b/src/pdu.rs index c9042306..6d78092f 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,12 +1,13 @@ use crate::{Error, Result}; use js_int::UInt; use ruma::{ + events::pdu::PduStub, events::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, EventId, Raw, RoomId, ServerKeyId, ServerName, UserId, -events::pdu::PduStub}; +}; use serde::{Deserialize, Serialize}; use serde_json::json; use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::UNIX_EPOCH}; From 1bf614b0f57a023cbc467f1e5bf03d1eae87b755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 17:02:20 +0200 Subject: [PATCH 32/51] fix: remove transaction_id from pdus over federation --- src/database/sending.rs | 6 ++++++ src/pdu.rs | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/database/sending.rs b/src/database/sending.rs index 187fd575..77998e79 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -36,6 +36,12 @@ impl Sending { let pdu_id = parts.next().ok_or_else(|| Error::bad_database("Invalid serverpduid in db."))?; let mut pdu_json = rooms.get_pdu_json_from_id(&pdu_id.into())?.ok_or_else(|| Error::bad_database("Event in serverpduids not found in db."))?; + if let Some(unsigned) = pdu_json + .as_object_mut() + .expect("json is object") + .get_mut("unsigned") { + unsigned.as_object_mut().expect("unsigned is object").remove("transaction_id"); + } pdu_json .as_object_mut() .expect("json is object") diff --git a/src/pdu.rs b/src/pdu.rs index 6d78092f..957d9e0a 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -201,6 +201,9 @@ impl PduEvent { } pub fn to_outgoing_federation_event(&self) -> Raw { + let mut unsigned = self.unsigned.clone(); + unsigned.remove("transaction_id"); + let mut json = json!({ "room_id": self.room_id, "sender": self.sender, @@ -210,7 +213,7 @@ impl PduEvent { "prev_events": self.prev_events, "depth": self.depth, "auth_events": self.auth_events, - "unsigned": self.unsigned, + "unsigned": unsigned, "hashes": self.hashes, "signatures": self.signatures, }); From 005e00e9b18459d569cca1993138a85d10dfc271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 17:16:55 +0200 Subject: [PATCH 33/51] fix: remove well-known --- src/main.rs | 1 - src/server_server.rs | 5 ----- 2 files changed, 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2817ab97..f81c7f46 100644 --- a/src/main.rs +++ b/src/main.rs @@ -119,7 +119,6 @@ fn setup_rocket() -> rocket::Rocket { client_server::get_pushers_route, client_server::set_pushers_route, client_server::upgrade_room_route, - server_server::well_known_server, server_server::get_server_version, server_server::get_server_keys, server_server::get_server_keys_deprecated, diff --git a/src/server_server.rs b/src/server_server.rs index 40ad654f..106f60eb 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -163,11 +163,6 @@ where } } -#[cfg_attr(feature = "conduit_bin", get("/.well-known/matrix/server"))] -pub fn well_known_server() -> Json { - rocket::response::content::Json(json!({ "m.server": "pc.koesters.xyz:59003"}).to_string()) -} - #[cfg_attr(feature = "conduit_bin", get("/_matrix/federation/v1/version"))] pub fn get_server_version() -> ConduitResult { Ok(get_server_version::Response { From dd749b8aee7c09ca8084059f91cd922e95fb6424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 21:46:10 +0200 Subject: [PATCH 34/51] fix: server keys and destination resolution when server name contains port --- src/database/globals.rs | 45 ++++++++++++++++++++++++++++++++--------- src/server_server.rs | 9 +++++++-- src/utils.rs | 9 +++++++-- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/database/globals.rs b/src/database/globals.rs index 89514251..8ce9c011 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -1,4 +1,5 @@ use crate::{utils, Error, Result}; +use log::error; use ruma::ServerName; use std::{convert::TryInto, sync::Arc}; @@ -17,19 +18,43 @@ pub struct Globals { impl Globals { pub fn load(globals: sled::Tree, config: &rocket::Config) -> Result { - let keypair = Arc::new( - ruma::signatures::Ed25519KeyPair::new( - &*globals - .update_and_fetch("keypair", utils::generate_keypair)? - .expect("utils::generate_keypair always returns Some"), - "key1".to_owned(), - ) - .map_err(|_| Error::bad_database("Private or public keys are invalid."))?, - ); + let bytes = &*globals + .update_and_fetch("keypair", utils::generate_keypair)? + .expect("utils::generate_keypair always returns Some"); + + let mut parts = bytes.splitn(2, |&b| b == 0xff); + + let keypair = utils::string_from_bytes( + // 1. version + parts + .next() + .expect("splitn always returns at least one element"), + ) + .map_err(|_| Error::bad_database("Invalid version bytes in keypair.")) + .and_then(|version| { + // 2. key + parts + .next() + .ok_or_else(|| Error::bad_database("Invalid keypair format in database.")) + .map(|key| (version, key)) + }) + .and_then(|(version, key)| { + ruma::signatures::Ed25519KeyPair::new(&key, version) + .map_err(|_| Error::bad_database("Private or public keys are invalid.")) + }); + + let keypair = match keypair { + Ok(k) => k, + Err(e) => { + error!("Keypair invalid. Deleting..."); + globals.remove("keypair")?; + return Err(e); + } + }; Ok(Self { globals, - keypair, + keypair: Arc::new(keypair), reqwest_client: reqwest::Client::new(), server_name: config .get_str("server_name") diff --git a/src/server_server.rs b/src/server_server.rs index 106f60eb..f334d6b5 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -17,7 +17,6 @@ use ruma::{ directory::{IncomingFilter, IncomingRoomNetwork}, EventId, ServerName, }; -use serde_json::json; use std::{ collections::BTreeMap, convert::TryFrom, @@ -58,7 +57,13 @@ where let actual_destination = "https://".to_owned() + &request_well_known(globals, &destination.as_str()) .await - .unwrap_or(destination.as_str().to_owned() + ":8448"); + .unwrap_or_else(|| { + let mut destination = destination.as_str().to_owned(); + if destination.find(':').is_none() { + destination += ":8448"; + } + destination + }); let mut http_request = request .try_into_http_request(&actual_destination, Some("")) diff --git a/src/utils.rs b/src/utils.rs index 8cf1b2ce..452b7c5a 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -29,8 +29,13 @@ pub fn increment(old: Option<&[u8]>) -> Option> { pub fn generate_keypair(old: Option<&[u8]>) -> Option> { Some(old.map(|s| s.to_vec()).unwrap_or_else(|| { - ruma::signatures::Ed25519KeyPair::generate() - .expect("Ed25519KeyPair generation always works (?)") + let mut value = random_string(8).as_bytes().to_vec(); + value.push(0xff); + value.extend_from_slice( + &ruma::signatures::Ed25519KeyPair::generate() + .expect("Ed25519KeyPair generation always works (?)"), + ); + value })) } From f4078a29eb2e4975bc5664aab718875ce67da6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 10:49:54 +0200 Subject: [PATCH 35/51] fix: synapse complains about missing origin --- src/database/rooms.rs | 6 ++++++ src/database/sending.rs | 1 + 2 files changed, 7 insertions(+) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 2246a61f..8e680330 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -812,6 +812,12 @@ impl Rooms { .expect("json is object") .remove("event_id"); + // Add origin because synapse likes that (and it's required in the spec) + pdu_json + .as_object_mut() + .expect("json is object") + .insert("origin".to_owned(), globals.server_name().as_str().into()); + ruma::signatures::hash_and_sign_event( globals.server_name().as_str(), globals.keypair(), diff --git a/src/database/sending.rs b/src/database/sending.rs index 77998e79..a3f15742 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -42,6 +42,7 @@ impl Sending { .get_mut("unsigned") { unsigned.as_object_mut().expect("unsigned is object").remove("transaction_id"); } + pdu_json .as_object_mut() .expect("json is object") From a567cd81d5e849285e6ef14b4d7ac41dd436c8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 15:08:51 +0200 Subject: [PATCH 36/51] improvement: better logs on deserialization errors --- src/server_server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server_server.rs b/src/server_server.rs index f334d6b5..aef3991e 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -160,7 +160,7 @@ where .expect("reqwest body is valid http body"), ); response.map_err(|e| { - warn!("{}", e); + warn!("Server returned bad response: {:?}", e); Error::BadServerResponse("Server returned bad response.") }) } From 4db6d7e4308b206e0f34e291f1d23e7a54ea254a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 18:10:17 +0200 Subject: [PATCH 37/51] fix: remove avatar url checks They are not in the spec and maubot relies on that --- src/client_server/profile.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 53893c0f..686d4c3e 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -119,18 +119,6 @@ pub async fn set_avatar_url_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - if let Some(avatar_url) = &body.avatar_url { - if !avatar_url.starts_with("mxc://") { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "avatar_url has to start with mxc://.", - )); - } - - // TODO in the future when we can handle media uploads make sure that this url is our own server - // TODO also make sure this is valid mxc:// format (not only starting with it) - } - db.users .set_avatar_url(&sender_id, body.avatar_url.clone())?; From 506c2a3146bb5314c95ad75ed069869a724ce628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 21:11:38 +0200 Subject: [PATCH 38/51] fix: can't find count from event in db --- src/client_server/sync.rs | 31 ++++++++++++++----------------- src/database/rooms.rs | 25 +++++++++++++------------ 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index eeeec005..6ece1809 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -93,7 +93,7 @@ pub async fn sync_events_route( let mut limited = false; let mut state_pdus = Vec::new(); - for pdu in non_timeline_pdus { + for (_, pdu) in non_timeline_pdus { if pdu.state_key.is_some() { state_pdus.push(pdu); } @@ -113,7 +113,7 @@ pub async fn sync_events_route( .rooms .pdus_since(&sender_id, &room_id, since)? .filter_map(|r| r.ok()) - .filter_map(|pdu| Some((pdu.state_key.clone()?, pdu))) + .filter_map(|(_, pdu)| Some((pdu.state_key.clone()?, pdu))) { if pdu.kind == EventType::RoomMember { send_member_count = true; @@ -188,8 +188,8 @@ pub async fn sync_events_route( .rooms .all_pdus(&sender_id, &room_id)? .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus - .filter(|pdu| pdu.kind == EventType::RoomMember) - .map(|pdu| { + .filter(|(_, pdu)| pdu.kind == EventType::RoomMember) + .map(|(_, pdu)| { let content = serde_json::from_value::< Raw, >(pdu.content.clone()) @@ -244,7 +244,7 @@ pub async fn sync_events_route( (db.rooms .pdus_since(&sender_id, &room_id, last_read)? .filter_map(|pdu| pdu.ok()) // Filter out buggy events - .filter(|pdu| { + .filter(|(_, pdu)| { matches!( pdu.kind.clone(), EventType::RoomMessage | EventType::RoomEncrypted @@ -260,18 +260,15 @@ pub async fn sync_events_route( None }; - let prev_batch = timeline_pdus.first().map_or(Ok::<_, Error>(None), |e| { - Ok(Some( - db.rooms - .get_pdu_count(&e.event_id)? - .ok_or_else(|| Error::bad_database("Can't find count from event in db."))? - .to_string(), - )) - })?; + let prev_batch = timeline_pdus + .first() + .map_or(Ok::<_, Error>(None), |(pdu_id, _)| { + Ok(Some(db.rooms.pdu_count(pdu_id)?.to_string())) + })?; let room_events = timeline_pdus .into_iter() - .map(|pdu| pdu.to_sync_room_event()) + .map(|(_, pdu)| pdu.to_sync_room_event()) .collect::>(); let mut edus = db @@ -380,7 +377,7 @@ pub async fn sync_events_route( let pdus = db.rooms.pdus_since(&sender_id, &room_id, since)?; let room_events = pdus .filter_map(|pdu| pdu.ok()) // Filter out buggy events - .map(|pdu| pdu.to_sync_room_event()) + .map(|(_, pdu)| pdu.to_sync_room_event()) .collect(); let left_room = sync_events::LeftRoom { @@ -395,7 +392,7 @@ pub async fn sync_events_route( let mut left_since_last_sync = false; for pdu in db.rooms.pdus_since(&sender_id, &room_id, since)? { - let pdu = pdu?; + let (_, pdu) = pdu?; if pdu.kind == EventType::RoomMember && pdu.state_key == Some(sender_id.to_string()) { let content = serde_json::from_value::< Raw, @@ -438,7 +435,7 @@ pub async fn sync_events_route( let room_id = room_id?; let mut invited_since_last_sync = false; for pdu in db.rooms.pdus_since(&sender_id, &room_id, since)? { - let pdu = pdu?; + let (_, pdu) = pdu?; if pdu.kind == EventType::RoomMember && pdu.state_key == Some(sender_id.to_string()) { let content = serde_json::from_value::< Raw, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 8e680330..263f51b9 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -355,18 +355,19 @@ impl Rooms { } } + /// Returns the `count` of this pdu's id. + pub fn pdu_count(&self, pdu_id: &[u8]) -> Result { + Ok( + utils::u64_from_bytes(&pdu_id[pdu_id.len() - mem::size_of::()..pdu_id.len()]) + .map_err(|_| Error::bad_database("PDU has invalid count bytes."))?, + ) + } + /// Returns the `count` of this pdu's id. pub fn get_pdu_count(&self, event_id: &EventId) -> Result> { self.eventid_pduid .get(event_id.as_bytes())? - .map_or(Ok(None), |pdu_id| { - Ok(Some( - utils::u64_from_bytes( - &pdu_id[pdu_id.len() - mem::size_of::()..pdu_id.len()], - ) - .map_err(|_| Error::bad_database("PDU has invalid count bytes."))?, - )) - }) + .map_or(Ok(None), |pdu_id| self.pdu_count(&pdu_id).map(Some)) } /// Returns the json of a pdu. @@ -860,7 +861,7 @@ impl Rooms { &self, user_id: &UserId, room_id: &RoomId, - ) -> Result>> { + ) -> Result>> { self.pdus_since(user_id, room_id, 0) } @@ -871,7 +872,7 @@ impl Rooms { user_id: &UserId, room_id: &RoomId, since: u64, - ) -> Result>> { + ) -> Result>> { let mut prefix = room_id.to_string().as_bytes().to_vec(); prefix.push(0xff); @@ -887,13 +888,13 @@ impl Rooms { .pduid_pdu .range(first_pdu_id..last_pdu_id) .filter_map(|r| r.ok()) - .map(move |(_, v)| { + .map(move |(pdu_id, v)| { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { pdu.unsigned.remove("transaction_id"); } - Ok(pdu) + Ok((pdu_id, pdu)) })) } From ea3aaa6b5c06e01bef52a66b64fe45d74d5f60c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 17 Sep 2020 14:44:47 +0200 Subject: [PATCH 39/51] improvement: more efficient /sync with gaps --- Cargo.lock | 48 +++---- src/client_server/context.rs | 10 +- src/client_server/membership.rs | 11 +- src/client_server/message.rs | 12 ++ src/client_server/sync.rs | 150 +++++++++++++------ src/database/rooms.rs | 245 ++++++++++++-------------------- src/main.rs | 1 + src/pdu.rs | 6 +- src/server_server.rs | 4 +- src/stateres.rs | 59 -------- 10 files changed, 249 insertions(+), 297 deletions(-) delete mode 100644 src/stateres.rs diff --git a/Cargo.lock b/Cargo.lock index 30144ca5..e142d725 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,9 +134,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cc" -version = "1.0.59" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381" +checksum = "ef611cc68ff783f18535d77ddd080185275713d852c4f5cbb6122c462a7a825c" [[package]] name = "cfg-if" @@ -213,7 +213,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ "percent-encoding", - "time 0.2.19", + "time 0.2.20", "version_check", ] @@ -342,9 +342,9 @@ checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" [[package]] name = "either" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" @@ -1370,7 +1370,7 @@ dependencies = [ "rocket_codegen", "rocket_http", "state", - "time 0.2.19", + "time 0.2.20", "tokio", "toml", "version_check", @@ -1405,7 +1405,7 @@ dependencies = [ "ref-cast", "smallvec", "state", - "time 0.2.19", + "time 0.2.20", "tokio", "tokio-rustls", "unicode-xid", @@ -1414,7 +1414,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1430,7 +1430,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "http", "percent-encoding", @@ -1445,7 +1445,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1456,7 +1456,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-common", @@ -1469,7 +1469,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "assign", "http", @@ -1488,7 +1488,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1502,7 +1502,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-common", @@ -1517,7 +1517,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1528,7 +1528,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1543,7 +1543,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1555,7 +1555,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro2", "quote", @@ -1566,7 +1566,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "serde", "strum", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "form_urlencoded", "itoa", @@ -1587,7 +1587,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "base64", "ring", @@ -1831,7 +1831,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#1d01b6e65b6afd50e65085fb40f1e7d2782f519e" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", @@ -1981,9 +1981,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c1a1fd93112fc50b11c43a1def21f926be3c18884fad676ea879572da070a1" +checksum = "0d4953c513c9bf1b97e9cdd83f11d60c4b0a83462880a360d80d96953a953fee" dependencies = [ "const_fn", "libc", diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 95937264..4c9be20f 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -49,7 +49,10 @@ pub fn get_context_route( .filter_map(|r| r.ok()) // Remove buggy events .collect::>(); - let start_token = events_before.last().map(|(count, _)| count.to_string()); + let start_token = events_before + .last() + .and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok()) + .map(|count| count.to_string()); let events_before = events_before .into_iter() @@ -68,7 +71,10 @@ pub fn get_context_route( .filter_map(|r| r.ok()) // Remove buggy events .collect::>(); - let end_token = events_after.last().map(|(count, _)| count.to_string()); + let end_token = events_after + .last() + .and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok()) + .map(|count| count.to_string()); let events_after = events_after .into_iter() diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index c4eed95c..628045de 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -601,8 +601,7 @@ async fn join_room_by_id_helper( .cloned() .collect::>(); - let power_level = - resolved_control_events.get(&(EventType::RoomPowerLevels, Some("".into()))); + let power_level = resolved_control_events.get(&(EventType::RoomPowerLevels, "".into())); // Sort the remaining non control events let sorted_event_ids = state_res::StateResolution::mainline_sort( room_id, @@ -644,13 +643,7 @@ async fn join_room_by_id_helper( )?; if state_events.contains(ev_id) { - state.insert( - ( - pdu.kind(), - pdu.state_key().expect("State events have a state key"), - ), - pdu_id, - ); + state.insert((pdu.kind(), pdu.state_key()), pdu_id); } } diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 3944d5bd..5a4488fe 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -117,6 +117,12 @@ pub fn get_message_events_route( .pdus_after(&sender_id, &body.room_id, from) .take(limit) .filter_map(|r| r.ok()) // Filter out buggy events + .filter_map(|(pdu_id, pdu)| { + db.rooms + .pdu_count(&pdu_id) + .map(|pdu_count| (pdu_count, pdu)) + .ok() + }) .take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to` .collect::>(); @@ -141,6 +147,12 @@ pub fn get_message_events_route( .pdus_until(&sender_id, &body.room_id, from) .take(limit) .filter_map(|r| r.ok()) // Filter out buggy events + .filter_map(|(pdu_id, pdu)| { + db.rooms + .pdu_count(&pdu_id) + .map(|pdu_count| (pdu_count, pdu)) + .ok() + }) .take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to` .collect::>(); diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 6ece1809..0e40bfbb 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -105,50 +105,92 @@ pub async fn sync_events_route( .room_state_get(&room_id, &EventType::RoomEncryption, "")? .is_some(); - // TODO: optimize this? - let mut send_member_count = false; - let mut joined_since_last_sync = false; - let mut new_encrypted_room = false; - for (state_key, pdu) in db + // Database queries: + let since_state_hash = db .rooms - .pdus_since(&sender_id, &room_id, since)? - .filter_map(|r| r.ok()) - .filter_map(|(_, pdu)| Some((pdu.state_key.clone()?, pdu))) - { - if pdu.kind == EventType::RoomMember { - send_member_count = true; + .pdus_until(sender_id, &room_id, since) + .next() + .and_then(|pdu| pdu.ok()) + .and_then(|pdu| db.rooms.pdu_state_hash(&pdu.0).ok()?); - let content = serde_json::from_value::< + let since_members = since_state_hash + .as_ref() + .and_then(|state_hash| db.rooms.state_type(state_hash, &EventType::RoomMember).ok()); + + let since_encryption = since_state_hash.as_ref().and_then(|state_hash| { + db.rooms + .state_get(&state_hash, &EventType::RoomEncryption, "") + .ok() + }); + + let current_members = db.rooms.room_state_type(&room_id, &EventType::RoomMember)?; + + // Calculations: + let new_encrypted_room = encrypted_room && since_encryption.is_none(); + + let send_member_count = since_members.as_ref().map_or(true, |since_members| { + current_members.len() != since_members.len() + }); + + let since_sender_member = since_members.as_ref().and_then(|members| { + members.get(sender_id.as_str()).and_then(|pdu| { + serde_json::from_value::>( + pdu.content.clone(), + ) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }) + }); + + if encrypted_room { + for (user_id, current_member) in current_members { + let current_membership = serde_json::from_value::< Raw, - >(pdu.content.clone()) + >(current_member.content.clone()) .expect("Raw::from_value always works") .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database."))?; + .map_err(|_| Error::bad_database("Invalid PDU in database."))? + .membership; + + let since_membership = since_members + .as_ref() + .and_then(|members| { + members.get(&user_id).and_then(|since_member| { + serde_json::from_value::< + Raw, + >(since_member.content.clone()) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }) + }) + .map_or(MembershipState::Leave, |member| member.membership); - if pdu.state_key == Some(sender_id.to_string()) - && content.membership == MembershipState::Join - { - joined_since_last_sync = true; - } else if encrypted_room && content.membership == MembershipState::Join { - // A new user joined an encrypted room - let user_id = UserId::try_from(state_key) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; - // Add encryption update if we didn't share an encrypted room already - if !share_encrypted_room(&db, &sender_id, &user_id, &room_id) { - device_list_updates.insert(user_id); + let user_id = UserId::try_from(user_id) + .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; + + match (since_membership, current_membership) { + (MembershipState::Leave, MembershipState::Join) => { + // A new user joined an encrypted room + if !share_encrypted_room(&db, &sender_id, &user_id, &room_id) { + device_list_updates.insert(user_id); + } + } + (MembershipState::Join, MembershipState::Leave) => { + // Write down users that have left encrypted rooms we are in + left_encrypted_users.insert(user_id); } - } else if encrypted_room && content.membership == MembershipState::Leave { - // Write down users that have left encrypted rooms we are in - left_encrypted_users.insert( - UserId::try_from(state_key) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?, - ); + _ => {} } - } else if pdu.kind == EventType::RoomEncryption { - new_encrypted_room = true; } } + let joined_since_last_sync = + since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); + if joined_since_last_sync && encrypted_room || new_encrypted_room { // If the user is in a new encrypted room, give them all joined users device_list_updates.extend( @@ -390,23 +432,37 @@ pub async fn sync_events_route( state: sync_events::State { events: Vec::new() }, }; - let mut left_since_last_sync = false; - for pdu in db.rooms.pdus_since(&sender_id, &room_id, since)? { - let (_, pdu) = pdu?; - if pdu.kind == EventType::RoomMember && pdu.state_key == Some(sender_id.to_string()) { - let content = serde_json::from_value::< - Raw, - >(pdu.content.clone()) + let since_member = db + .rooms + .pdus_until(sender_id, &room_id, since) + .next() + .and_then(|pdu| pdu.ok()) + .and_then(|pdu| { + db.rooms + .pdu_state_hash(&pdu.0) + .ok()? + .ok_or_else(|| Error::bad_database("Pdu in db doesn't have a state hash.")) + .ok() + }) + .and_then(|state_hash| { + db.rooms + .state_get(&state_hash, &EventType::RoomMember, sender_id.as_str()) + .ok()? + .ok_or_else(|| Error::bad_database("State hash in db doesn't have a state.")) + .ok() + }) + .and_then(|pdu| { + serde_json::from_value::>( + pdu.content.clone(), + ) .expect("Raw::from_value always works") .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database."))?; + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }); - if content.membership == MembershipState::Leave { - left_since_last_sync = true; - break; - } - } - } + let left_since_last_sync = + since_member.map_or(false, |member| member.membership == MembershipState::Join); if left_since_last_sync { device_list_left.extend( diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 263f51b9..59586262 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -31,7 +31,7 @@ use std::{ /// /// This is created when a state group is added to the database by /// hashing the entire state. -pub type StateHashId = Vec; +pub type StateHashId = IVec; #[derive(Clone)] pub struct Rooms { @@ -100,7 +100,7 @@ impl StateStore for Rooms { impl Rooms { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. - pub fn state_full(&self, state_hash: StateHashId) -> Result> { + pub fn state_full(&self, state_hash: &StateHashId) -> Result> { self.stateid_pduid .scan_prefix(&state_hash) .values() @@ -115,61 +115,87 @@ impl Rooms { }) .map(|pdu| { let pdu = pdu?; - Ok(((pdu.kind, pdu.state_key), pdu.event_id)) + Ok(( + ( + pdu.kind.clone(), + pdu.state_key + .as_ref() + .ok_or_else(|| Error::bad_database("State event has no state key."))? + .clone(), + ), + pdu, + )) }) .collect::>>() } - // TODO make this return Result - /// Fetches the previous StateHash ID to `current`. - pub fn prev_state_hash(&self, current: StateHashId) -> Option { - let mut found = false; - for pair in self.pduid_statehash.iter().rev() { - let prev = pair.ok()?.1; - if current == prev.as_ref() { - found = true; - } - if current != prev.as_ref() && found { - return Some(prev.to_vec()); - } + /// Returns all state entries for this type. + pub fn state_type( + &self, + state_hash: &StateHashId, + event_type: &EventType, + ) -> Result> { + let mut prefix = state_hash.to_vec(); + prefix.push(0xff); + prefix.extend_from_slice(&event_type.to_string().as_bytes()); + prefix.push(0xff); + + let mut hashmap = HashMap::new(); + for pdu in self + .stateid_pduid + .scan_prefix(&prefix) + .values() + .map(|pdu_id| { + Ok::<_, Error>( + serde_json::from_slice::(&self.pduid_pdu.get(pdu_id?)?.ok_or_else( + || Error::bad_database("PDU in state not found in database."), + )?) + .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?, + ) + }) + { + let pdu = pdu?; + let state_key = pdu.state_key.clone().ok_or_else(|| { + Error::bad_database("Room state contains event without state_key.") + })?; + hashmap.insert(state_key, pdu); } - None + Ok(hashmap) } - /// Returns the last state hash key added to the db. - pub fn current_state_hash(&self, room_id: &RoomId) -> Result> { - Ok(self - .roomid_statehash - .get(room_id.as_bytes())? - .map(|bytes| bytes.to_vec())) + /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`). + pub fn state_get( + &self, + state_hash: &StateHashId, + event_type: &EventType, + state_key: &str, + ) -> Result> { + let mut key = state_hash.to_vec(); + key.push(0xff); + key.extend_from_slice(&event_type.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(&state_key.as_bytes()); + + self.stateid_pduid.get(&key)?.map_or(Ok(None), |pdu_id| { + Ok::<_, Error>(Some( + serde_json::from_slice::( + &self.pduid_pdu.get(pdu_id)?.ok_or_else(|| { + Error::bad_database("PDU in state not found in database.") + })?, + ) + .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?, + )) + }) } - /// This fetches auth event_ids from the current state using the - /// full `roomstateid_pdu` tree. - pub fn get_auth_event_ids( - &self, - room_id: &RoomId, - kind: &EventType, - sender: &UserId, - state_key: Option<&str>, - content: serde_json::Value, - ) -> Result> { - let auth_events = state_res::auth_types_for_event( - kind.clone(), - sender, - state_key.map(|s| s.to_string()), - content, - ); + /// Returns the last state hash key added to the db. + pub fn pdu_state_hash(&self, pdu_id: &[u8]) -> Result> { + Ok(self.pduid_statehash.get(pdu_id)?) + } - let mut events = vec![]; - for (event_type, state_key) in auth_events { - if let Some(state_key) = state_key.as_ref() { - if let Some(id) = self.room_state_get(room_id, &event_type, state_key)? { - events.push(id.event_id); - } - } - } - Ok(events) + /// Returns the last state hash key added to the db. + pub fn current_state_hash(&self, room_id: &RoomId) -> Result> { + Ok(self.roomid_statehash.get(room_id.as_bytes())?) } /// This fetches auth events from the current state. @@ -190,10 +216,8 @@ impl Rooms { let mut events = StateMap::new(); for (event_type, state_key) in auth_events { - if let Some(s_key) = state_key.as_ref() { - if let Some(pdu) = self.room_state_get(room_id, &event_type, s_key)? { - events.insert((event_type, state_key), pdu); - } + if let Some(pdu) = self.room_state_get(room_id, &event_type, &state_key)? { + events.insert((event_type, state_key), pdu); } } Ok(events) @@ -206,7 +230,7 @@ impl Rooms { // We only hash the pdu's event ids, not the whole pdu let bytes = pdu_id_bytes.join(&0xff); let hash = digest::digest(&digest::SHA256, &bytes); - Ok(hash.as_ref().to_vec()) + Ok(hash.as_ref().into()) } /// Checks if a room exists. @@ -230,7 +254,7 @@ impl Rooms { ) -> Result<()> { let state_hash = self.calculate_hash(&state.values().map(|pdu_id| &**pdu_id).collect::>())?; - let mut prefix = state_hash.clone(); + let mut prefix = state_hash.to_vec(); prefix.push(0xff); for ((event_type, state_key), pdu_id) in state { @@ -248,41 +272,11 @@ impl Rooms { } /// Returns the full room state. - pub fn room_state_full( - &self, - room_id: &RoomId, - ) -> Result> { + pub fn room_state_full(&self, room_id: &RoomId) -> Result> { if let Some(current_state_hash) = self.current_state_hash(room_id)? { - let mut prefix = current_state_hash; - prefix.push(0xff); - - let mut hashmap = HashMap::new(); - for pdu in self - .stateid_pduid - .scan_prefix(prefix) - .values() - .map(|pdu_id| { - Ok::<_, Error>( - serde_json::from_slice::( - &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { - Error::bad_database("PDU in state not found in database.") - })?, - ) - .map_err(|_| { - Error::bad_database("Invalid PDU bytes in current room state.") - })?, - ) - }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert((pdu.kind.clone(), state_key), pdu); - } - Ok(hashmap) + self.state_full(¤t_state_hash) } else { - Ok(HashMap::new()) + Ok(BTreeMap::new()) } } @@ -293,36 +287,7 @@ impl Rooms { event_type: &EventType, ) -> Result> { if let Some(current_state_hash) = self.current_state_hash(room_id)? { - let mut prefix = current_state_hash; - prefix.push(0xff); - prefix.extend_from_slice(&event_type.to_string().as_bytes()); - prefix.push(0xff); - - let mut hashmap = HashMap::new(); - for pdu in self - .stateid_pduid - .scan_prefix(&prefix) - .values() - .map(|pdu_id| { - Ok::<_, Error>( - serde_json::from_slice::( - &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { - Error::bad_database("PDU in state not found in database.") - })?, - ) - .map_err(|_| { - Error::bad_database("Invalid PDU bytes in current room state.") - })?, - ) - }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert(state_key, pdu); - } - Ok(hashmap) + self.state_type(¤t_state_hash, event_type) } else { Ok(HashMap::new()) } @@ -336,20 +301,7 @@ impl Rooms { state_key: &str, ) -> Result> { if let Some(current_state_hash) = self.current_state_hash(room_id)? { - let mut key = current_state_hash; - key.push(0xff); - key.extend_from_slice(&event_type.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(&state_key.as_bytes()); - - self.stateid_pduid.get(&key)?.map_or(Ok(None), |pdu_id| { - Ok::<_, Error>(Some( - serde_json::from_slice::(&self.pduid_pdu.get(pdu_id)?.ok_or_else( - || Error::bad_database("PDU in state not found in database."), - )?) - .map_err(|_| Error::bad_database("Invalid PDU bytes in current room state."))?, - )) - }) + self.state_get(¤t_state_hash, event_type, state_key) } else { Ok(None) } @@ -562,14 +514,15 @@ impl Rooms { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. /// The incoming event is the `pdu_id` passed to this method. - fn append_to_state(&self, new_pdu_id: &[u8], new_pdu: &PduEvent) -> Result { + pub fn append_to_state(&self, new_pdu_id: &[u8], new_pdu: &PduEvent) -> Result { let old_state = if let Some(old_state_hash) = self.roomid_statehash.get(new_pdu.room_id.as_bytes())? { // Store state for event. The state does not include the event itself. // Instead it's the state before the pdu, so the room's old state. - self.pduid_statehash.insert(new_pdu_id, &old_state_hash)?; + self.pduid_statehash + .insert(dbg!(new_pdu_id), &old_state_hash)?; if new_pdu.state_key.is_none() { - return Ok(old_state_hash.to_vec()); + return Ok(old_state_hash); } let mut prefix = old_state_hash.to_vec(); @@ -841,9 +794,7 @@ impl Rooms { let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data)?; - if pdu.state_key.is_some() { - self.append_to_state(&pdu_id, &pdu)?; - } + self.append_to_state(&pdu_id, &pdu)?; for server in self .room_servers(room_id) @@ -905,7 +856,7 @@ impl Rooms { user_id: &UserId, room_id: &RoomId, until: u64, - ) -> impl Iterator> { + ) -> impl Iterator> { // Create the first part of the full pdu id let mut prefix = room_id.to_string().as_bytes().to_vec(); prefix.push(0xff); @@ -916,23 +867,18 @@ impl Rooms { let current: &[u8] = ¤t; let user_id = user_id.clone(); - let prefixlen = prefix.len(); self.pduid_pdu .range(..current) .rev() .filter_map(|r| r.ok()) .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(move |(k, v)| { + .map(move |(pdu_id, v)| { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { pdu.unsigned.remove("transaction_id"); } - Ok(( - utils::u64_from_bytes(&k[prefixlen..]) - .map_err(|_| Error::bad_database("Invalid pdu id in db."))?, - pdu, - )) + Ok((pdu_id, pdu)) }) } @@ -943,7 +889,7 @@ impl Rooms { user_id: &UserId, room_id: &RoomId, from: u64, - ) -> impl Iterator> { + ) -> impl Iterator> { // Create the first part of the full pdu id let mut prefix = room_id.to_string().as_bytes().to_vec(); prefix.push(0xff); @@ -954,22 +900,17 @@ impl Rooms { let current: &[u8] = ¤t; let user_id = user_id.clone(); - let prefixlen = prefix.len(); self.pduid_pdu .range(current..) .filter_map(|r| r.ok()) .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(move |(k, v)| { + .map(move |(pdu_id, v)| { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { pdu.unsigned.remove("transaction_id"); } - Ok(( - utils::u64_from_bytes(&k[prefixlen..]) - .map_err(|_| Error::bad_database("Invalid pdu id in db."))?, - pdu, - )) + Ok((pdu_id, pdu)) }) } diff --git a/src/main.rs b/src/main.rs index f81c7f46..06fda59a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -123,6 +123,7 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_server_keys, server_server::get_server_keys_deprecated, server_server::get_public_rooms_route, + server_server::get_public_rooms_filtered_route, server_server::send_transaction_message_route, ], ) diff --git a/src/pdu.rs b/src/pdu.rs index 957d9e0a..d5b5415c 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,4 +1,4 @@ -use crate::{Error, Result}; +use crate::Error; use js_int::UInt; use ruma::{ events::pdu::PduStub, @@ -35,7 +35,7 @@ pub struct PduEvent { } impl PduEvent { - pub fn redact(&mut self, reason: &PduEvent) -> Result<()> { + pub fn redact(&mut self, reason: &PduEvent) -> crate::Result<()> { self.unsigned.clear(); let allowed: &[&str] = match self.kind { @@ -244,7 +244,7 @@ impl From<&state_res::StateEvent> for PduEvent { .expect("time is valid"), kind: pdu.kind(), content: pdu.content().clone(), - state_key: pdu.state_key(), + state_key: Some(pdu.state_key()), prev_events: pdu.prev_event_ids(), depth: pdu.depth().clone(), auth_events: pdu.auth_events(), diff --git a/src/server_server.rs b/src/server_server.rs index aef3991e..6f2b1790 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -329,8 +329,10 @@ pub fn send_transaction_message_route<'a>( let pdu = serde_json::from_value::(value.clone()) .expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - db.rooms + let pdu_id = db + .rooms .append_pdu(&pdu, &value, &db.globals, &db.account_data)?; + db.rooms.append_to_state(&pdu_id, &pdu)?; } } Ok(send_transaction_message::v1::Response { diff --git a/src/stateres.rs b/src/stateres.rs deleted file mode 100644 index ee47099a..00000000 --- a/src/stateres.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::collections::HashMap; - -fn stateres(state_a: HashMap, state_b: HashMap) { - let mut unconflicted = todo!("state at fork event"); - - let mut conflicted: HashMap = state_a - .iter() - .filter(|(key_a, value_a)| match state_b.remove(key_a) { - Some(value_b) if value_a == value_b => unconflicted.insert(key_a, value_a), - _ => false, - }) - .collect(); - - // We removed unconflicted from state_b, now we can easily insert all events that are only in fork b - conflicted.extend(state_b); - - let partial_state = unconflicted.clone(); - - let full_conflicted = conflicted.clone(); // TODO: auth events - - let output_rev = Vec::new(); - let event_map = HashMap::new(); - let incoming_edges = HashMap::new(); - - for event in full_conflicted { - event_map.insert(event.event_id, event); - incoming_edges.insert(event.event_id, 0); - } - - for e in conflicted_control_events { - for a in e.auth_events { - incoming_edges[a.event_id] += 1; - } - } - - while incoming_edges.len() > 0 { - let mut count_0 = incoming_edges - .iter() - .filter(|(_, c)| c == 0) - .collect::>(); - - count_0.sort_by(|(x, _), (y, _)| { - x.power_level - .cmp(&a.power_level) - .then_with(|| x.origin_server.ts.cmp(&y.origin_server_ts)) - .then_with(|| x.event_id.cmp(&y.event_id)) - }); - - for (id, count) in count_0 { - output_rev.push(event_map[id]); - - for auth_event in event_map[id].auth_events { - incoming_edges[auth_event.event_id] -= 1; - } - - incoming_edges.remove(id); - } - } -} From 8bcfff276652e20fee14e13109d80a514e1a107d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 17 Sep 2020 19:58:19 +0200 Subject: [PATCH 40/51] fix: no notification counts for fast /syncs --- src/database/rooms.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 59586262..18881dd8 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -447,6 +447,11 @@ impl Rooms { // This is also the next_batch/since value let index = globals.next_count()?; + // Mark as read first so the sending client doesn't get a notification even if appending + // fails + self.edus + .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; + let mut pdu_id = pdu.room_id.as_bytes().to_vec(); pdu_id.push(0xff); pdu_id.extend_from_slice(&index.to_be_bytes()); @@ -503,9 +508,6 @@ impl Rooms { _ => {} } - self.edus - .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; - Ok(pdu_id) } @@ -520,7 +522,7 @@ impl Rooms { // Store state for event. The state does not include the event itself. // Instead it's the state before the pdu, so the room's old state. self.pduid_statehash - .insert(dbg!(new_pdu_id), &old_state_hash)?; + .insert(new_pdu_id, &old_state_hash)?; if new_pdu.state_key.is_none() { return Ok(old_state_hash); } From 267c7216162932c9f6e35d69be4c6303d5134952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 17 Sep 2020 22:41:43 +0200 Subject: [PATCH 41/51] fix: encryption and sync spam --- src/client_server/sync.rs | 106 +++++++++++++++++++++++--------------- src/database/rooms.rs | 3 +- src/database/users.rs | 2 +- 3 files changed, 67 insertions(+), 44 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 0e40bfbb..2f2c8eac 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -100,47 +100,61 @@ pub async fn sync_events_route( limited = true; } + // Database queries: let encrypted_room = db .rooms .room_state_get(&room_id, &EventType::RoomEncryption, "")? .is_some(); - // Database queries: + // These type is Option>. The outer Option is None when there is no event between + // since and the current room state, meaning there should be no updates. + // The inner Option is None when there is an event, but there is no state hash associated + // with it. This can happen for the RoomCreate event, so all updates should arrive. let since_state_hash = db .rooms - .pdus_until(sender_id, &room_id, since) + .pdus_after(sender_id, &room_id, since) // - 1 So we can get the event at since .next() - .and_then(|pdu| pdu.ok()) - .and_then(|pdu| db.rooms.pdu_state_hash(&pdu.0).ok()?); + .map(|pdu| db.rooms.pdu_state_hash(&pdu.ok()?.0).ok()?); - let since_members = since_state_hash - .as_ref() - .and_then(|state_hash| db.rooms.state_type(state_hash, &EventType::RoomMember).ok()); + let since_members = since_state_hash.as_ref().map(|state_hash| { + state_hash.as_ref().and_then(|state_hash| { + db.rooms + .state_type(&state_hash, &EventType::RoomMember) + .ok() + }) + }); - let since_encryption = since_state_hash.as_ref().and_then(|state_hash| { - db.rooms - .state_get(&state_hash, &EventType::RoomEncryption, "") - .ok() + let since_encryption = since_state_hash.as_ref().map(|state_hash| { + state_hash.as_ref().and_then(|state_hash| { + db.rooms + .state_get(&state_hash, &EventType::RoomEncryption, "") + .ok() + }) }); let current_members = db.rooms.room_state_type(&room_id, &EventType::RoomMember)?; // Calculations: - let new_encrypted_room = encrypted_room && since_encryption.is_none(); + let new_encrypted_room = + encrypted_room && since_encryption.map_or(false, |encryption| encryption.is_none()); - let send_member_count = since_members.as_ref().map_or(true, |since_members| { - current_members.len() != since_members.len() + let send_member_count = since_members.as_ref().map_or(false, |since_members| { + since_members.as_ref().map_or(true, |since_members| { + current_members.len() != since_members.len() + }) }); - let since_sender_member = since_members.as_ref().and_then(|members| { - members.get(sender_id.as_str()).and_then(|pdu| { - serde_json::from_value::>( - pdu.content.clone(), - ) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database.")) - .ok() + let since_sender_member = since_members.as_ref().map(|since_members| { + since_members.as_ref().and_then(|members| { + members.get(sender_id.as_str()).and_then(|pdu| { + serde_json::from_value::>( + pdu.content.clone(), + ) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }) }) }); @@ -154,20 +168,29 @@ pub async fn sync_events_route( .map_err(|_| Error::bad_database("Invalid PDU in database."))? .membership; - let since_membership = since_members - .as_ref() - .and_then(|members| { - members.get(&user_id).and_then(|since_member| { - serde_json::from_value::< - Raw, - >(since_member.content.clone()) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database.")) - .ok() - }) - }) - .map_or(MembershipState::Leave, |member| member.membership); + let since_membership = + since_members + .as_ref() + .map_or(MembershipState::Join, |members| { + members + .as_ref() + .and_then(|members| { + members.get(&user_id).and_then(|since_member| { + serde_json::from_value::< + Raw, + >( + since_member.content.clone() + ) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| { + Error::bad_database("Invalid PDU in database.") + }) + .ok() + }) + }) + .map_or(MembershipState::Leave, |member| member.membership) + }); let user_id = UserId::try_from(user_id) .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; @@ -188,8 +211,9 @@ pub async fn sync_events_route( } } - let joined_since_last_sync = - since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); + let joined_since_last_sync = since_sender_member.map_or(false, |member| { + member.map_or(true, |member| member.membership != MembershipState::Join) + }); if joined_since_last_sync && encrypted_room || new_encrypted_room { // If the user is in a new encrypted room, give them all joined users @@ -434,7 +458,7 @@ pub async fn sync_events_route( let since_member = db .rooms - .pdus_until(sender_id, &room_id, since) + .pdus_after(sender_id, &room_id, since) .next() .and_then(|pdu| pdu.ok()) .and_then(|pdu| { @@ -581,7 +605,7 @@ pub async fn sync_events_route( changed: device_list_updates.into_iter().collect(), left: device_list_left.into_iter().collect(), }, - device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since { + device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since || since == 0 { db.users.count_one_time_keys(sender_id, device_id)? } else { BTreeMap::new() diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 18881dd8..108edb5d 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -521,8 +521,7 @@ impl Rooms { if let Some(old_state_hash) = self.roomid_statehash.get(new_pdu.room_id.as_bytes())? { // Store state for event. The state does not include the event itself. // Instead it's the state before the pdu, so the room's old state. - self.pduid_statehash - .insert(new_pdu_id, &old_state_hash)?; + self.pduid_statehash.insert(new_pdu_id, &old_state_hash)?; if new_pdu.state_key.is_none() { return Ok(old_state_hash); } diff --git a/src/database/users.rs b/src/database/users.rs index 10e1ef30..2e26c1ea 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -603,7 +603,7 @@ impl Users { .room_state_get(&room_id, &EventType::RoomEncryption, "")? .is_none() { - return Ok(()); + continue; } let mut key = room_id.to_string().as_bytes().to_vec(); From 19207845bc2ec7db6d0ede2ff2b345162bc817ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 20 Sep 2020 13:49:13 +0200 Subject: [PATCH 42/51] Fix ruma dependency --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 98dbac9c..9ea58b46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1554,7 +1554,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "ruma-api", "ruma-client-api", @@ -1568,7 +1568,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "http", "percent-encoding", @@ -1583,7 +1583,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1594,7 +1594,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "assign", "http", @@ -1612,7 +1612,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "js_int", "ruma-identifiers", @@ -1625,7 +1625,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "js_int", "ruma-common", @@ -1640,7 +1640,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1651,7 +1651,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "js_int", "ruma-api", @@ -1666,7 +1666,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1678,7 +1678,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "proc-macro2", "quote", @@ -1689,7 +1689,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "ruma-serde", "serde", @@ -1700,7 +1700,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "form_urlencoded", "itoa", @@ -1712,7 +1712,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "base64", "ring", diff --git a/Cargo.toml b/Cargo.toml index 4945e3c8..db3c6852 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ edition = "2018" rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } #ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "987d48666cf166cf12100b5dbc61b5e3385c4014" } # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fixes" } # Used for matrix spec type definitions and helpers +ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-old-fixes" } # Used for matrix spec type definitions and helpers #ruma = { path = "../ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } tokio = "0.2.22" # Used for long polling sled = "0.32.0" # Used for storing data permanently From e08dfd982b87b632590bf125cdc48ee83bad9ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 23 Sep 2020 12:03:08 +0200 Subject: [PATCH 43/51] improvement: look at SRV record when sending requests --- Cargo.lock | 237 +++++++++++++++++++++++++++++++++----- Cargo.toml | 2 + src/client_server/sync.rs | 4 +- src/server_server.rs | 49 ++++++-- 4 files changed, 252 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e142d725..4cce6668 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,20 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "addr2line" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + [[package]] name = "adler32" version = "1.2.0" @@ -73,6 +88,20 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "backtrace" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide 0.4.2", + "object", + "rustc-demangle", +] + [[package]] name = "base-x" version = "0.2.6" @@ -192,6 +221,7 @@ dependencies = [ "state-res", "thiserror", "tokio", + "trust-dns-resolver", ] [[package]] @@ -213,7 +243,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ "percent-encoding", - "time 0.2.20", + "time 0.2.21", "version_check", ] @@ -355,6 +385,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-as-inner" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "fnv" version = "1.0.7" @@ -537,6 +579,12 @@ dependencies = [ "lzw", ] +[[package]] +name = "gimli" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" + [[package]] name = "glob" version = "0.3.0" @@ -586,6 +634,17 @@ dependencies = [ "libc", ] +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi 0.3.9", +] + [[package]] name = "http" version = "0.2.1" @@ -613,11 +672,17 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + [[package]] name = "hyper" -version = "0.13.7" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e68a8dd9716185d9e64ea473ea6ef63529252e3e27623295a0378a19665d5eb" +checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" dependencies = [ "bytes", "futures-channel", @@ -627,10 +692,10 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project", "socket2", - "time 0.1.44", "tokio", "tower-service", "tracing", @@ -695,9 +760,12 @@ checksum = "cb6ee2a7da03bfc3b66ca47c92c2e392fcc053ea040a85561749b026f7aad09a" [[package]] name = "instant" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" +checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" +dependencies = [ + "cfg-if", +] [[package]] name = "iovec" @@ -708,6 +776,18 @@ dependencies = [ "libc", ] +[[package]] +name = "ipconfig" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +dependencies = [ + "socket2", + "widestring", + "winapi 0.3.9", + "winreg 0.6.2", +] + [[package]] name = "ipnet" version = "2.3.0" @@ -778,6 +858,12 @@ version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" +[[package]] +name = "linked-hash-map" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" + [[package]] name = "lock_api" version = "0.4.1" @@ -796,6 +882,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lzw" version = "0.10.0" @@ -808,6 +903,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + [[package]] name = "matchers" version = "0.0.1" @@ -837,9 +938,9 @@ checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "memoffset" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" dependencies = [ "autocfg", ] @@ -869,6 +970,16 @@ dependencies = [ "adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.6.22" @@ -991,6 +1102,12 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" + [[package]] name = "once_cell" version = "1.4.1" @@ -1130,7 +1247,7 @@ dependencies = [ "bitflags", "crc32fast", "deflate", - "miniz_oxide", + "miniz_oxide 0.3.7", ] [[package]] @@ -1181,6 +1298,12 @@ dependencies = [ "yansi", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.7" @@ -1334,7 +1457,17 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg", + "winreg 0.7.0", +] + +[[package]] +name = "resolv-conf" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11834e137f3b14e309437a8276714eed3a80d1ef894869e510f2c0c0b98b9f4a" +dependencies = [ + "hostname", + "quick-error", ] [[package]] @@ -1370,7 +1503,7 @@ dependencies = [ "rocket_codegen", "rocket_http", "state", - "time 0.2.20", + "time 0.2.21", "tokio", "toml", "version_check", @@ -1405,7 +1538,7 @@ dependencies = [ "ref-cast", "smallvec", "state", - "time 0.2.20", + "time 0.2.21", "tokio", "tokio-rustls", "unicode-xid", @@ -1414,7 +1547,6 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1430,7 +1562,6 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "http", "percent-encoding", @@ -1445,7 +1576,6 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1456,7 +1586,6 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-common", @@ -1469,7 +1598,6 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "assign", "http", @@ -1488,7 +1616,6 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1502,7 +1629,6 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-common", @@ -1517,7 +1643,6 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1528,7 +1653,6 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1543,7 +1667,6 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1555,7 +1678,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro2", "quote", @@ -1566,7 +1688,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "serde", "strum", @@ -1575,7 +1696,6 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "form_urlencoded", "itoa", @@ -1587,7 +1707,6 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "base64", "ring", @@ -1607,6 +1726,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + [[package]] name = "rustc_version" version = "0.2.3" @@ -1831,7 +1956,6 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", @@ -1981,9 +2105,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4953c513c9bf1b97e9cdd83f11d60c4b0a83462880a360d80d96953a953fee" +checksum = "2c2e31fb28e2a9f01f5ed6901b066c1ba2333c04b64dc61254142bafcb3feb2c" dependencies = [ "const_fn", "libc", @@ -2182,6 +2306,46 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "trust-dns-proto" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd7061ba6f4d4d9721afedffbfd403f20f39a4301fee1b70d6fcd09cca69f28" +dependencies = [ + "async-trait", + "backtrace", + "enum-as-inner", + "futures", + "idna", + "lazy_static", + "log", + "rand", + "smallvec", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f23cdfdc3d8300b3c50c9e84302d3bd6d860fb9529af84ace6cf9665f181b77" +dependencies = [ + "backtrace", + "cfg-if", + "futures", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "trust-dns-proto", +] + [[package]] name = "try-lock" version = "0.2.3" @@ -2366,6 +2530,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "widestring" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a763e303c0e0f23b0da40888724762e802a8ffefbc22de4127ef42493c2ea68c" + [[package]] name = "winapi" version = "0.2.8" @@ -2400,6 +2570,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "winreg" version = "0.7.0" diff --git a/Cargo.toml b/Cargo.toml index 60296a2c..2126e427 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,8 @@ image = { version = "0.23.9", default-features = false, features = ["jpeg", "png base64 = "0.12.3" # Used when hashing the state ring = "0.16.15" +# Used when querying the SRV record of other servers +trust-dns-resolver = "0.19.5" [features] default = ["conduit_bin"] diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 2f2c8eac..aec03afa 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -605,7 +605,9 @@ pub async fn sync_events_route( changed: device_list_updates.into_iter().collect(), left: device_list_left.into_iter().collect(), }, - device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since || since == 0 { + device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since + || since == 0 + { db.users.count_one_time_keys(sender_id, device_id)? } else { BTreeMap::new() diff --git a/src/server_server.rs b/src/server_server.rs index 6f2b1790..3ebbeac4 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,5 +1,5 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; -use http::header::{HeaderValue, AUTHORIZATION}; +use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::warn; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ @@ -23,6 +23,7 @@ use std::{ fmt::Debug, time::{Duration, SystemTime}, }; +use trust_dns_resolver::AsyncResolver; pub async fn request_well_known( globals: &crate::database::globals::Globals, @@ -54,16 +55,36 @@ pub async fn send_request( where T: Debug, { + let resolver = AsyncResolver::tokio_from_system_conf() + .await + .map_err(|_| Error::BadConfig("Failed to set up trust dns resolver with system config."))?; + + let mut host = None; + let actual_destination = "https://".to_owned() - + &request_well_known(globals, &destination.as_str()) - .await - .unwrap_or_else(|| { - let mut destination = destination.as_str().to_owned(); - if destination.find(':').is_none() { - destination += ":8448"; + + &if let Some(mut delegated_hostname) = + request_well_known(globals, &destination.as_str()).await + { + if let Ok(Some(srv)) = resolver + .srv_lookup(format!("_matrix._tcp.{}", delegated_hostname)) + .await + .map(|srv| srv.iter().next().map(|result| result.target().to_string())) + { + host = Some(delegated_hostname); + srv.trim_end_matches('.').to_owned() + } else { + if delegated_hostname.find(':').is_none() { + delegated_hostname += ":8448"; } - destination - }); + delegated_hostname + } + } else { + let mut destination = destination.as_str().to_owned(); + if destination.find(':').is_none() { + destination += ":8448"; + } + destination + }; let mut http_request = request .try_into_http_request(&actual_destination, Some("")) @@ -129,9 +150,17 @@ where } } - let reqwest_request = reqwest::Request::try_from(http_request) + if let Some(host) = host { + http_request + .headers_mut() + .insert(HOST, HeaderValue::from_str(&host).unwrap()); + } + + let mut reqwest_request = reqwest::Request::try_from(http_request) .expect("all http requests are valid reqwest requests"); + *reqwest_request.timeout_mut() = Some(Duration::from_secs(30)); + let reqwest_response = globals.reqwest_client().execute(reqwest_request).await; // Because reqwest::Response -> http::Response is complicated: From 26e200e290133e0e2424da0d1e354eaa764c6d4c Mon Sep 17 00:00:00 2001 From: miruka Date: Fri, 25 Sep 2020 14:18:36 -0400 Subject: [PATCH 44/51] Reduce media ID length from 256 to 32 Most common filesystems limit paths to 255 bytes. This change brings down the media ID length to be similar to Synapse servers (25), and makes it possible for clients to download media with the ID included in the filename. --- src/client_server/media.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index efcb3a6d..394cf740 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -9,7 +9,7 @@ use ruma::api::client::{ use rocket::{get, post}; use std::convert::TryInto; -const MXC_LENGTH: usize = 256; +const MXC_LENGTH: usize = 32; #[cfg_attr(feature = "conduit_bin", get("/_matrix/media/r0/config"))] pub fn get_media_config_route( From ab332363ce3af716694cde7bf35cf421e91707d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 23 Sep 2020 15:23:29 +0200 Subject: [PATCH 45/51] fix: don't send new requests to servers if we are already waiting --- Cargo.lock | 15 ++++ src/database/sending.rs | 147 ++++++++++++++++++++++++++-------------- src/server_server.rs | 6 +- 3 files changed, 116 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4cce6668..5f6b21c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1547,6 +1547,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1562,6 +1563,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "http", "percent-encoding", @@ -1576,6 +1578,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1586,6 +1589,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "ruma-api", "ruma-common", @@ -1598,6 +1602,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "assign", "http", @@ -1616,6 +1621,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "js_int", "ruma-api", @@ -1629,6 +1635,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "js_int", "ruma-common", @@ -1643,6 +1650,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1653,6 +1661,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "js_int", "ruma-api", @@ -1667,6 +1676,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1678,6 +1688,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "proc-macro2", "quote", @@ -1688,6 +1699,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "serde", "strum", @@ -1696,6 +1708,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "form_urlencoded", "itoa", @@ -1707,6 +1720,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "base64", "ring", @@ -1956,6 +1970,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", diff --git a/src/database/sending.rs b/src/database/sending.rs index a3f15742..d3c7fc6a 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -1,8 +1,11 @@ -use std::{convert::TryFrom, time::SystemTime}; +use std::{collections::HashSet, convert::TryFrom, time::SystemTime}; use crate::{server_server, utils, Error, Result}; +use federation::transactions::send_transaction_message; +use log::warn; use rocket::futures::stream::{FuturesUnordered, StreamExt}; use ruma::{api::federation, Raw, ServerName}; +use sled::IVec; use tokio::select; pub struct Sending { @@ -18,66 +21,49 @@ impl Sending { tokio::spawn(async move { let mut futures = FuturesUnordered::new(); + let mut waiting_servers = HashSet::new(); + let mut subscriber = serverpduids.watch_prefix(b""); loop { select! { - Some(_) = futures.next() => {}, + Some(server) = futures.next() => { + warn!("response: {:?}", &server); + match server { + Ok((server, _response)) => { + waiting_servers.remove(&server) + } + Err((server, _e)) => { + waiting_servers.remove(&server) + } + }; + }, Some(event) = &mut subscriber => { - let serverpduid = if let sled::Event::Insert {key, ..} = event { - key - } else - { return Err::<(), Error>(Error::bad_database("")); }; - let mut parts = serverpduid.splitn(2, |&b| b == 0xff); - let server = Box::::try_from( - utils::string_from_bytes(parts.next().expect("splitn will always return 1 or more elements")) - .map_err(|_| Error::bad_database("ServerName in serverpduid bytes are invalid."))? - ).map_err(|_| Error::bad_database("ServerName in serverpduid is invalid."))?; - - let pdu_id = parts.next().ok_or_else(|| Error::bad_database("Invalid serverpduid in db."))?; - let mut pdu_json = rooms.get_pdu_json_from_id(&pdu_id.into())?.ok_or_else(|| Error::bad_database("Event in serverpduids not found in db."))?; - - if let Some(unsigned) = pdu_json - .as_object_mut() - .expect("json is object") - .get_mut("unsigned") { - unsigned.as_object_mut().expect("unsigned is object").remove("transaction_id"); - } - - pdu_json - .as_object_mut() - .expect("json is object") - .remove("event_id"); - - let raw_json = - serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); - - let globals = &globals; - - futures.push( - async move { - let pdus = vec![raw_json]; - let transaction_id = utils::random_string(16); + if let sled::Event::Insert { key, .. } = event { + let serverpduid = key.clone(); + let mut parts = serverpduid.splitn(2, |&b| b == 0xff); - server_server::send_request( - &globals, - server, - federation::transactions::send_transaction_message::v1::Request { - origin: globals.server_name(), - pdus: &pdus, - edus: &[], - origin_server_ts: SystemTime::now(), - transaction_id: &transaction_id, - }, - ).await + if let Some((server, pdu_id)) = utils::string_from_bytes( + parts + .next() + .expect("splitn will always return 1 or more elements"), + ) + .map_err(|_| Error::bad_database("ServerName in serverpduid bytes are invalid.")) + .and_then(|server_str|Box::::try_from(server_str) + .map_err(|_| Error::bad_database("ServerName in serverpduid is invalid."))) + .ok() + .filter(|server| waiting_servers.insert(server.clone())) + .and_then(|server| parts + .next() + .ok_or_else(|| Error::bad_database("Invalid serverpduid in db.")).ok().map(|pdu_id| (server, pdu_id))) + { + futures.push(Self::handle_event(server, pdu_id.into(), &globals, &rooms)); } - ); - }, + } + } } } }); } - /* - */ pub fn send_pdu(&self, server: Box, pdu_id: &[u8]) -> Result<()> { let mut key = server.as_bytes().to_vec(); @@ -87,4 +73,63 @@ impl Sending { Ok(()) } + + async fn handle_event( + server: Box, + pdu_id: IVec, + globals: &super::globals::Globals, + rooms: &super::rooms::Rooms, + ) -> std::result::Result< + (Box, send_transaction_message::v1::Response), + (Box, Error), + > { + let mut pdu_json = rooms + .get_pdu_json_from_id(&pdu_id) + .map_err(|e| (server.clone(), e))? + .ok_or_else(|| { + ( + server.clone(), + Error::bad_database("Event in serverpduids not found in db."), + ) + })?; + + if let Some(unsigned) = pdu_json + .as_object_mut() + .expect("json is object") + .get_mut("unsigned") + { + unsigned + .as_object_mut() + .expect("unsigned is object") + .remove("transaction_id"); + } + + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + let raw_json = + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); + + let globals = &globals; + + let pdus = vec![raw_json]; + let transaction_id = utils::random_string(16); + + server_server::send_request( + &globals, + server.clone(), + send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus: &pdus, + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &transaction_id, + }, + ) + .await + .map(|response| (server.clone(), response)) + .map_err(|e| (server, e)) + } } diff --git a/src/server_server.rs b/src/server_server.rs index 3ebbeac4..d67b0b60 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -161,6 +161,7 @@ where *reqwest_request.timeout_mut() = Some(Duration::from_secs(30)); + let url = reqwest_request.url().clone(); let reqwest_response = globals.reqwest_client().execute(reqwest_request).await; // Because reqwest::Response -> http::Response is complicated: @@ -189,7 +190,10 @@ where .expect("reqwest body is valid http body"), ); response.map_err(|e| { - warn!("Server returned bad response: {:?}", e); + warn!( + "Server returned bad response {} ({}): {:?}", + destination, url, e + ); Error::BadServerResponse("Server returned bad response.") }) } From 0d6159c2dafc87685dc2e6fd79b034e4b3f21c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 25 Sep 2020 12:26:29 +0200 Subject: [PATCH 46/51] improvement: get_missing_events route and cleanup --- Cargo.lock | 32 ++++++++++++------------- src/database/sending.rs | 52 ++++++++++++----------------------------- src/main.rs | 1 + src/pdu.rs | 43 ++++++++++++++-------------------- src/server_server.rs | 47 +++++++++++++++++++++++++++++++++++-- 5 files changed, 95 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f6b21c5..ed20ca37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -627,9 +627,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" +checksum = "4c30f6d0bc6b00693347368a67d41b58f2fb851215ff1da49e90fe2c5c667151" dependencies = [ "libc", ] @@ -1547,7 +1547,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1563,7 +1563,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "http", "percent-encoding", @@ -1578,7 +1578,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1589,7 +1589,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-common", @@ -1602,7 +1602,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "assign", "http", @@ -1621,7 +1621,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1635,7 +1635,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-common", @@ -1650,7 +1650,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1661,7 +1661,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1688,7 +1688,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro2", "quote", @@ -1699,7 +1699,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "serde", "strum", @@ -1708,7 +1708,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "form_urlencoded", "itoa", @@ -1720,7 +1720,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "base64", "ring", diff --git a/src/database/sending.rs b/src/database/sending.rs index d3c7fc6a..1ed94cc2 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -1,10 +1,10 @@ use std::{collections::HashSet, convert::TryFrom, time::SystemTime}; -use crate::{server_server, utils, Error, Result}; +use crate::{server_server, utils, Error, PduEvent, Result}; use federation::transactions::send_transaction_message; use log::warn; use rocket::futures::stream::{FuturesUnordered, StreamExt}; -use ruma::{api::federation, Raw, ServerName}; +use ruma::{api::federation, ServerName}; use sled::IVec; use tokio::select; @@ -83,49 +83,27 @@ impl Sending { (Box, send_transaction_message::v1::Response), (Box, Error), > { - let mut pdu_json = rooms - .get_pdu_json_from_id(&pdu_id) - .map_err(|e| (server.clone(), e))? - .ok_or_else(|| { - ( - server.clone(), - Error::bad_database("Event in serverpduids not found in db."), - ) - })?; - - if let Some(unsigned) = pdu_json - .as_object_mut() - .expect("json is object") - .get_mut("unsigned") - { - unsigned - .as_object_mut() - .expect("unsigned is object") - .remove("transaction_id"); - } - - pdu_json - .as_object_mut() - .expect("json is object") - .remove("event_id"); - - let raw_json = - serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); - - let globals = &globals; - - let pdus = vec![raw_json]; - let transaction_id = utils::random_string(16); + let pdu_json = PduEvent::to_outgoing_federation_event( + rooms + .get_pdu_json_from_id(&pdu_id) + .map_err(|e| (server.clone(), e))? + .ok_or_else(|| { + ( + server.clone(), + Error::bad_database("Event in serverpduids not found in db."), + ) + })?, + ); server_server::send_request( &globals, server.clone(), send_transaction_message::v1::Request { origin: globals.server_name(), - pdus: &pdus, + pdus: &[pdu_json], edus: &[], origin_server_ts: SystemTime::now(), - transaction_id: &transaction_id, + transaction_id: &utils::random_string(16), }, ) .await diff --git a/src/main.rs b/src/main.rs index 06fda59a..fa1cc5ca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -125,6 +125,7 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_public_rooms_route, server_server::get_public_rooms_filtered_route, server_server::send_transaction_message_route, + server_server::get_missing_events_route, ], ) .attach(AdHoc::on_attach("Config", |mut rocket| async { diff --git a/src/pdu.rs b/src/pdu.rs index d5b5415c..4b1df4b8 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,7 +1,6 @@ use crate::Error; use js_int::UInt; use ruma::{ - events::pdu::PduStub, events::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, @@ -200,32 +199,26 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - pub fn to_outgoing_federation_event(&self) -> Raw { - let mut unsigned = self.unsigned.clone(); - unsigned.remove("transaction_id"); - - let mut json = json!({ - "room_id": self.room_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "type": self.kind, - "content": self.content, - "prev_events": self.prev_events, - "depth": self.depth, - "auth_events": self.auth_events, - "unsigned": unsigned, - "hashes": self.hashes, - "signatures": self.signatures, - }); - - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &self.redacts { - json["redacts"] = json!(redacts); + pub fn to_outgoing_federation_event( + mut pdu_json: serde_json::Value, + ) -> Raw { + if let Some(unsigned) = pdu_json + .as_object_mut() + .expect("json is object") + .get_mut("unsigned") + { + unsigned + .as_object_mut() + .expect("unsigned is object") + .remove("transaction_id"); } - serde_json::from_value(json).expect("Raw::from_value always works") + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works") } } diff --git a/src/server_server.rs b/src/server_server.rs index d67b0b60..2d52c4ac 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -3,13 +3,13 @@ use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::warn; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ - api::federation::directory::get_public_rooms_filtered, api::{ federation::{ - directory::get_public_rooms, + directory::{get_public_rooms, get_public_rooms_filtered}, discovery::{ get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, }, + event::get_missing_events, transactions::send_transaction_message, }, OutgoingRequest, @@ -373,3 +373,46 @@ pub fn send_transaction_message_route<'a>( } .into()) } + +#[cfg_attr( + feature = "conduit_bin", + post("/_matrix/federation/v1/get_missing_events/<_>", data = "") +)] +pub fn get_missing_events_route<'a>( + db: State<'a, Database>, + body: Ruma>, +) -> ConduitResult { + let mut queued_events = body.latest_events.clone(); + let mut events = Vec::new(); + + let mut i = 0; + while i < queued_events.len() && events.len() < u64::from(body.limit) as usize { + if let Some(pdu) = db.rooms.get_pdu_json(&queued_events[i])? { + if body.earliest_events.contains( + &serde_json::from_value( + pdu.get("event_id") + .cloned() + .ok_or_else(|| Error::bad_database("Event in db has no event_id field."))?, + ) + .map_err(|_| Error::bad_database("Invalid event_id field in pdu in db."))?, + ) { + i += 1; + continue; + } + queued_events.extend_from_slice( + &serde_json::from_value::>( + pdu.get("prev_events").cloned().ok_or_else(|| { + Error::bad_database("Invalid prev_events field of pdu in db.") + })?, + ) + .map_err(|_| Error::bad_database("Invalid prev_events content in pdu in db."))?, + ); + events.push(PduEvent::to_outgoing_federation_event(pdu)); + } + i += 1; + } + + dbg!(&events); + + Ok(get_missing_events::v1::Response { events }.into()) +} From bcd1fe18561cbda0f26f53464325fccd177e42ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 5 Oct 2020 22:19:22 +0200 Subject: [PATCH 47/51] feat: admin room --- Cargo.lock | 15 - src/client_server/account.rs | 301 ++++++++++++++++-- src/client_server/membership.rs | 196 ++++++------ src/client_server/message.rs | 43 ++- src/client_server/profile.rs | 140 ++++---- src/client_server/redact.rs | 37 +-- src/client_server/room.rs | 546 +++++++++++++++----------------- src/client_server/state.rs | 31 +- src/database/rooms.rs | 49 ++- src/database/users.rs | 5 + src/main.rs | 1 + src/server_server.rs | 64 +++- 12 files changed, 859 insertions(+), 569 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed20ca37..6571f7cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1547,7 +1547,6 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1563,7 +1562,6 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "http", "percent-encoding", @@ -1578,7 +1576,6 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1589,7 +1586,6 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-common", @@ -1602,7 +1598,6 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "assign", "http", @@ -1621,7 +1616,6 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1635,7 +1629,6 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-common", @@ -1650,7 +1643,6 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1661,7 +1653,6 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1676,7 +1667,6 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1688,7 +1678,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro2", "quote", @@ -1699,7 +1688,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "serde", "strum", @@ -1708,7 +1696,6 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "form_urlencoded", "itoa", @@ -1720,7 +1707,6 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "base64", "ring", @@ -1970,7 +1956,6 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 7e0f942e..66b4a62a 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -1,3 +1,5 @@ +use std::{collections::BTreeMap, convert::TryInto}; + use super::{State, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; use crate::{pdu::PduBuilder, utils, ConduitResult, Database, Error, Ruma}; use ruma::{ @@ -11,8 +13,11 @@ use ruma::{ uiaa::{AuthFlow, UiaaInfo}, }, }, - events::{room::member, EventType}, - UserId, + events::{ + room::canonical_alias, room::guest_access, room::history_visibility, room::join_rules, + room::member, room::name, room::topic, EventType, + }, + RoomAliasId, RoomId, RoomVersionId, UserId, }; use register::RegistrationKind; @@ -73,7 +78,7 @@ pub fn get_register_available_route( feature = "conduit_bin", post("/_matrix/client/r0/register", data = "") )] -pub fn register_route( +pub async fn register_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -202,6 +207,265 @@ pub fn register_route( body.initial_device_display_name.clone(), )?; + // If this is the first user on this server, create the admins room + if db.users.count() == 1 { + // Create a user for the server + let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name()) + .expect("@conduit:server_name is valid"); + + db.users.create(&conduit_user, "")?; + + let room_id = RoomId::new(db.globals.server_name()); + + let mut content = ruma::events::room::create::CreateEventContent::new(conduit_user.clone()); + content.federate = true; + content.predecessor = None; + content.room_version = RoomVersionId::Version6; + + // 1. The room create event + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 2. Make conduit bot join + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(conduit_user.to_string()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 3. Power levels + let mut users = BTreeMap::new(); + users.insert(conduit_user.clone(), 100.into()); + users.insert(user_id.clone(), 100.into()); + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: serde_json::to_value( + ruma::events::room::power_levels::PowerLevelsEventContent { + ban: 50.into(), + events: BTreeMap::new(), + events_default: 0.into(), + invite: 50.into(), + kick: 50.into(), + redact: 50.into(), + state_default: 50.into(), + users, + users_default: 0.into(), + notifications: ruma::events::room::power_levels::NotificationPowerLevels { + room: 50.into(), + }, + }, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.1 Join Rules + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomJoinRules, + content: serde_json::to_value(join_rules::JoinRulesEventContent::new( + join_rules::JoinRule::Invite, + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.2 History Visibility + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomHistoryVisibility, + content: serde_json::to_value( + history_visibility::HistoryVisibilityEventContent::new( + history_visibility::HistoryVisibility::Shared, + ), + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.3 Guest Access + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomGuestAccess, + content: serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::Forbidden, + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 6. Events implied by name and topic + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomName, + content: serde_json::to_value( + name::NameEventContent::new("Admin Room".to_owned()).map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") + })?, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTopic, + content: serde_json::to_value(topic::TopicEventContent { + topic: format!("Manage {}", db.globals.server_name()), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // Room alias + let alias: RoomAliasId = format!("#admins:{}", db.globals.server_name()) + .try_into() + .expect("#admins:server_name is a valid alias name"); + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCanonicalAlias, + content: serde_json::to_value(canonical_alias::CanonicalAliasEventContent { + alias: Some(alias.clone()), + alt_aliases: Vec::new(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?; + + // Invite and join the real user + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &user_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + } + Ok(register::Response { access_token: Some(token), user_id, @@ -354,23 +618,20 @@ pub async fn deactivate_route( third_party_invite: None, }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } // Remove devices and mark account as deactivated diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 628045de..526e82f1 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -108,22 +108,20 @@ pub async fn leave_room_route( event.membership = member::MembershipState::Leave; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(leave_room::Response::new().into()) } @@ -139,29 +137,27 @@ pub async fn invite_user_route( let sender_id = body.sender_id.as_ref().expect("user is authenticated"); if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user_id)?, - avatar_url: db.users.avatar_url(&user_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user_id)?, + avatar_url: db.users.avatar_url(&user_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(invite_user::Response.into()) } else { @@ -199,22 +195,20 @@ pub async fn kick_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; // TODO: reason - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(kick_user::Response::new().into()) } @@ -257,22 +251,20 @@ pub async fn ban_user_route( }, )?; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(ban_user::Response::new().into()) } @@ -306,22 +298,20 @@ pub async fn unban_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(unban_user::Response::new().into()) } @@ -640,6 +630,7 @@ async fn join_room_by_id_helper( &serde_json::to_value(&**pdu).expect("PDU is valid value"), &db.globals, &db.account_data, + &db.sending, )?; if state_events.contains(ev_id) { @@ -657,23 +648,20 @@ async fn join_room_by_id_helper( third_party_invite: None, }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } Ok(join_room_by_id::Response::new(room_id.clone()).into()) diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 5a4488fe..c32bd687 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -49,29 +49,26 @@ pub async fn send_message_event_route( let mut unsigned = serde_json::Map::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into()); - let event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: body.content.event_type().into(), - content: serde_json::from_str( - body.json_body - .as_ref() - .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? - .get(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, - unsigned: Some(unsigned), - state_key: None, - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: body.content.event_type().into(), + content: serde_json::from_str( + body.json_body + .as_ref() + .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? + .get(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, + unsigned: Some(unsigned), + state_key: None, + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; db.transaction_ids .add_txnid(sender_id, device_id, &body.txn_id, event_id.as_bytes())?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 686d4c3e..9c6bd512 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -31,43 +31,41 @@ pub async fn set_displayname_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - displayname: body.displayname.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + displayname: body.displayname.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( "Tried to send displayname update for user not in the room.", ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Presence update db.rooms.edus.update_presence( @@ -125,43 +123,41 @@ pub async fn set_avatar_url_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - avatar_url: body.avatar_url.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( - "Tried to send avatar url update for user not in the room.", - ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + avatar_url: body.avatar_url.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( + "Tried to send avatar url update for user not in the room.", + ) + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Presence update db.rooms.edus.update_presence( diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 24df8dd7..b13cd802 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -18,26 +18,23 @@ pub async fn redact_event_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - let event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomRedaction, - content: serde_json::to_value(redaction::RedactionEventContent { - reason: body.reason.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: Some(body.event_id.clone()), - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomRedaction, + content: serde_json::to_value(redaction::RedactionEventContent { + reason: body.reason.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: Some(body.event_id.clone()), + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(redact_event::Response { event_id }.into()) } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index d21148bb..28d30e29 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -53,47 +53,43 @@ pub async fn create_room_route( content.room_version = RoomVersionId::Version6; // 1. The room create event - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 2. Let the room creator join - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 3. Power levels let mut users = BTreeMap::new(); @@ -123,22 +119,20 @@ pub async fn create_room_route( }) .expect("event is valid, we just created it") }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomPowerLevels, - content: power_levels_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: power_levels_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 4. Events set by preset @@ -149,42 +143,108 @@ pub async fn create_room_route( }); // 4.1 Join Rules - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomJoinRules, - content: match preset { - create_room::RoomPreset::PublicChat => serde_json::to_value( - join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), - ) - .expect("event is valid, we just created it"), - // according to spec "invite" is the default - _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( - join_rules::JoinRule::Invite, + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomJoinRules, + content: match preset { + create_room::RoomPreset::PublicChat => serde_json::to_value( + join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), + ) + .expect("event is valid, we just created it"), + // according to spec "invite" is the default + _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( + join_rules::JoinRule::Invite, + )) + .expect("event is valid, we just created it"), + }, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.2 History Visibility + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomHistoryVisibility, + content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new( + history_visibility::HistoryVisibility::Shared, + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.3 Guest Access + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomGuestAccess, + content: match preset { + create_room::RoomPreset::PublicChat => { + serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::Forbidden, )) - .expect("event is valid, we just created it"), - }, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, + .expect("event is valid, we just created it") + } + _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::CanJoin, + )) + .expect("event is valid, we just created it"), }, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 5. Events listed in initial_state + for event in &body.initial_state { + let pdu_builder = serde_json::from_str::( + &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"), + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; + + // Silently skip encryption events if they are not allowed + if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() { + continue; + } + + db.rooms.build_and_append_pdu( + pdu_builder, &sender_id, &room_id, &db.globals, &db.sending, &db.account_data, - ) - .await?; + )?; + } - // 4.2 History Visibility - db.rooms - .build_and_append_pdu( + // 6. Events implied by name and topic + if let Some(name) = &body.name { + db.rooms.build_and_append_pdu( PduBuilder { - event_type: EventType::RoomHistoryVisibility, + event_type: EventType::RoomName, content: serde_json::to_value( - history_visibility::HistoryVisibilityEventContent::new( - history_visibility::HistoryVisibility::Shared, - ), + name::NameEventContent::new(name.clone()).map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") + })?, ) .expect("event is valid, we just created it"), unsigned: None, @@ -196,26 +256,17 @@ pub async fn create_room_route( &db.globals, &db.sending, &db.account_data, - ) - .await?; + )?; + } - // 4.3 Guest Access - db.rooms - .build_and_append_pdu( + if let Some(topic) = &body.topic { + db.rooms.build_and_append_pdu( PduBuilder { - event_type: EventType::RoomGuestAccess, - content: match preset { - create_room::RoomPreset::PublicChat => { - serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::Forbidden, - )) - .expect("event is valid, we just created it") - } - _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::CanJoin, - )) - .expect("event is valid, we just created it"), - }, + event_type: EventType::RoomTopic, + content: serde_json::to_value(topic::TopicEventContent { + topic: topic.clone(), + }) + .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -225,105 +276,32 @@ pub async fn create_room_route( &db.globals, &db.sending, &db.account_data, - ) - .await?; - - // 5. Events listed in initial_state - for event in &body.initial_state { - let pdu_builder = serde_json::from_str::( - &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"), - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; - - // Silently skip encryption events if they are not allowed - if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() { - continue; - } - - db.rooms - .build_and_append_pdu( - pdu_builder, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; - } - - // 6. Events implied by name and topic - if let Some(name) = &body.name { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomName, - content: serde_json::to_value( - name::NameEventContent::new(name.clone()).map_err(|_| { - Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") - })?, - ) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; - } - - if let Some(topic) = &body.topic { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTopic, - content: serde_json::to_value(topic::TopicEventContent { - topic: topic.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + )?; } // 7. Events implied by invite (and TODO: invite_3pid) for user in &body.invite { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user)?, - avatar_url: db.users.avatar_url(&user)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user)?, + avatar_url: db.users.avatar_url(&user)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } // Homeserver specific stuff @@ -395,29 +373,24 @@ pub async fn upgrade_room_route( // Send a m.room.tombstone event to the old room to indicate that it is not intended to be used any further // Fail if the sender does not have the required permissions - let tombstone_event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTombstone, - content: serde_json::to_value( - ruma::events::room::tombstone::TombstoneEventContent { - body: "This room has been replaced".to_string(), - replacement_room: replacement_room.clone(), - }, - ) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let tombstone_event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTombstone, + content: serde_json::to_value(ruma::events::room::tombstone::TombstoneEventContent { + body: "This room has been replaced".to_string(), + replacement_room: replacement_room.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Get the old room federations status let federate = serde_json::from_value::>( @@ -444,48 +417,44 @@ pub async fn upgrade_room_route( create_event_content.room_version = new_version; create_event_content.predecessor = predecessor; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(create_event_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(create_event_content) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + )?; // Join the new room - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + )?; // Recommended transferable state events list from the specs let transferable_state_events = vec![ @@ -507,22 +476,20 @@ pub async fn upgrade_room_route( None => continue, // Skipping missing events. }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type, - content: event_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type, + content: event_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + )?; } // Moves any local aliases to the new room @@ -552,24 +519,21 @@ pub async fn upgrade_room_route( power_levels_event_content.invite = new_level; // Modify the power levels in the old room to prevent sending of events and inviting new users - let _ = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomPowerLevels, - content: serde_json::to_value(power_levels_event_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await; + let _ = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: serde_json::to_value(power_levels_event_content) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Return the replacement room id Ok(upgrade_room::Response { replacement_room }.into()) diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 46182a12..1e13b42f 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -213,23 +213,20 @@ pub async fn send_state_event_for_key_helper( } } - let event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: content.event_type().into(), - content: json, - unsigned: None, - state_key, - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: content.event_type().into(), + content: json, + unsigned: None, + state_key, + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(event_id) } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 108edb5d..ab05b390 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -10,7 +10,7 @@ use ruma::{ events::{ ignored_user_list, room::{ - member, + member, message, power_levels::{self, PowerLevelsEventContent}, }, EventType, @@ -440,6 +440,7 @@ impl Rooms { pdu_json: &serde_json::Value, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, + sending: &super::sending::Sending, ) -> Result> { self.replace_pdu_leaves(&pdu.room_id, &pdu.event_id)?; @@ -452,7 +453,8 @@ impl Rooms { self.edus .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; - let mut pdu_id = pdu.room_id.as_bytes().to_vec(); + let room_id = pdu.room_id.clone(); + let mut pdu_id = room_id.as_bytes().to_vec(); pdu_id.push(0xff); pdu_id.extend_from_slice(&index.to_be_bytes()); @@ -503,6 +505,45 @@ impl Rooms { key.extend_from_slice(&pdu_id); self.tokenids.insert(key, &[])?; } + + if body.starts_with(&format!("@conduit:{}: ", globals.server_name())) + && self + .id_from_alias( + &format!("#admins:{}", globals.server_name()) + .try_into() + .expect("#admins:server_name is a valid room alias"), + )? + .as_ref() + == Some(&pdu.room_id) + { + let mut parts = body.split_whitespace().skip(1); + if let Some(command) = parts.next() { + let args = parts.collect::>(); + + self.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMessage, + content: serde_json::to_value( + message::TextMessageEventContent { + body: format!("Command: {}, Args: {:?}", command, args), + formatted: None, + relates_to: None, + }, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: None, + }, + &UserId::try_from(format!("@conduit:{}", globals.server_name())) + .expect("@conduit:server_name is valid"), + &room_id, + &globals, + &sending, + &account_data, + )?; + } + } } } _ => {} @@ -570,7 +611,7 @@ impl Rooms { } /// Creates a new persisted data unit and adds it to a room. - pub async fn build_and_append_pdu( + pub fn build_and_append_pdu( &self, pdu_builder: PduBuilder, sender: &UserId, @@ -793,7 +834,7 @@ impl Rooms { .expect("json is object") .insert("event_id".to_owned(), pdu.event_id.to_string().into()); - let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data)?; + let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data, sending)?; self.append_to_state(&pdu_id, &pdu)?; diff --git a/src/database/users.rs b/src/database/users.rs index 2e26c1ea..0d35e362 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -57,6 +57,11 @@ impl Users { Ok(()) } + /// Returns the number of users registered on this server. + pub fn count(&self) -> usize { + self.userid_password.iter().count() + } + /// Find out which user an access token belongs to. pub fn find_from_token(&self, token: &str) -> Result> { self.token_userdeviceid diff --git a/src/main.rs b/src/main.rs index fa1cc5ca..8fb5fda9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -126,6 +126,7 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_public_rooms_filtered_route, server_server::send_transaction_message_route, server_server::get_missing_events_route, + server_server::get_profile_information_route, ], ) .attach(AdHoc::on_attach("Config", |mut rocket| async { diff --git a/src/server_server.rs b/src/server_server.rs index 2d52c4ac..0c175bfb 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,4 +1,5 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; +use get_profile_information::v1::ProfileField; use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::warn; use rocket::{get, post, put, response::content::Json, State}; @@ -10,6 +11,7 @@ use ruma::{ get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, }, event::get_missing_events, + query::get_profile_information, transactions::send_transaction_message, }, OutgoingRequest, @@ -362,9 +364,9 @@ pub fn send_transaction_message_route<'a>( let pdu = serde_json::from_value::(value.clone()) .expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - let pdu_id = db - .rooms - .append_pdu(&pdu, &value, &db.globals, &db.account_data)?; + let pdu_id = + db.rooms + .append_pdu(&pdu, &value, &db.globals, &db.account_data, &db.sending)?; db.rooms.append_to_state(&pdu_id, &pdu)?; } } @@ -416,3 +418,59 @@ pub fn get_missing_events_route<'a>( Ok(get_missing_events::v1::Response { events }.into()) } + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/federation/v1/query/profile", data = "") +)] +pub fn get_profile_information_route<'a>( + db: State<'a, Database>, + body: Ruma>, +) -> ConduitResult { + let mut displayname = None; + let mut avatar_url = None; + + match body.field { + Some(ProfileField::DisplayName) => displayname = db.users.displayname(&body.user_id)?, + Some(ProfileField::AvatarUrl) => avatar_url = db.users.avatar_url(&body.user_id)?, + None => { + displayname = db.users.displayname(&body.user_id)?; + avatar_url = db.users.avatar_url(&body.user_id)?; + } + } + + Ok(get_profile_information::v1::Response { + displayname, + avatar_url, + } + .into()) +} + +/* +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/federation/v2/invite/<_>/<_>", data = "") +)] +pub fn get_user_devices_route<'a>( + db: State<'a, Database>, + body: Ruma>, +) -> ConduitResult { + let mut displayname = None; + let mut avatar_url = None; + + match body.field { + Some(ProfileField::DisplayName) => displayname = db.users.displayname(&body.user_id)?, + Some(ProfileField::AvatarUrl) => avatar_url = db.users.avatar_url(&body.user_id)?, + None => { + displayname = db.users.displayname(&body.user_id)?; + avatar_url = db.users.avatar_url(&body.user_id)?; + } + } + + Ok(get_profile_information::v1::Response { + displayname, + avatar_url, + } + .into()) +} +*/ From c15ae3c126020c3826f15e5f50ca70a669fc1402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 6 Oct 2020 20:43:35 +0200 Subject: [PATCH 48/51] fix: invalid typing bytes because of 0xff in numbers --- src/database/rooms/edus.rs | 9 ++++++--- src/database/sending.rs | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index a794c690..29f5407b 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -11,6 +11,7 @@ use ruma::{ use std::{ collections::HashMap, convert::{TryFrom, TryInto}, + mem, }; #[derive(Clone)] @@ -228,9 +229,11 @@ impl RoomEdus { let key = key?; Ok::<_, Error>(( key.clone(), - utils::u64_from_bytes(key.split(|&b| b == 0xff).nth(1).ok_or_else(|| { - Error::bad_database("RoomTyping has invalid timestamp or delimiters.") - })?) + utils::u64_from_bytes( + &key.splitn(2, |&b| b == 0xff).nth(1).ok_or_else(|| { + Error::bad_database("RoomTyping has invalid timestamp or delimiters.") + })?[0..mem::size_of::()], + ) .map_err(|_| Error::bad_database("RoomTyping has invalid timestamp bytes."))?, )) }) diff --git a/src/database/sending.rs b/src/database/sending.rs index 1ed94cc2..c818cbfc 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -28,6 +28,7 @@ impl Sending { select! { Some(server) = futures.next() => { warn!("response: {:?}", &server); + warn!("futures left: {}", &futures.len()); match server { Ok((server, _response)) => { waiting_servers.remove(&server) From 6afc4c9b3e066f2d071e8420c9e4111d0dc65d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 6 Oct 2020 21:04:51 +0200 Subject: [PATCH 49/51] feat: federation disabled by default It can be enable in the Rocket.toml config or using ROCKET_FEDERATION_ENABLED=true --- DEPLOY_FROM_SOURCE.md | 3 +++ Rocket-example.toml | 2 ++ docker-compose.yml | 1 + src/database/globals.rs | 6 ++++++ src/server_server.rs | 39 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/DEPLOY_FROM_SOURCE.md b/DEPLOY_FROM_SOURCE.md index 4d685f6e..456fe6ea 100644 --- a/DEPLOY_FROM_SOURCE.md +++ b/DEPLOY_FROM_SOURCE.md @@ -27,7 +27,10 @@ Environment="ROCKET_SERVER_NAME=YOURSERVERNAME.HERE" # EDIT THIS Environment="ROCKET_PORT=14004" # Reverse proxy port +#Environment="ROCKET_MAX_REQUEST_SIZE=20000000" # in bytes #Environment="ROCKET_REGISTRATION_DISABLED=true" +#Environment="ROCKET_ENCRYPTION_DISABLED=true" +#Environment="ROCKET_FEDERATION_ENABLED=true" #Environment="ROCKET_LOG=normal" # Detailed logging Environment="ROCKET_ENV=production" diff --git a/Rocket-example.toml b/Rocket-example.toml index 41b36d3a..8eb48e95 100644 --- a/Rocket-example.toml +++ b/Rocket-example.toml @@ -16,6 +16,8 @@ port = 14004 # Note: existing rooms will continue to work #encryption_disabled = true +#federation_enabled = true + # Default path is in this user's data #database_path = "/home/timo/MyConduitServer" diff --git a/docker-compose.yml b/docker-compose.yml index f06eaca9..7d197622 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ services: # ROCKET_PORT: 8000 # ROCKET_REGISTRATION_DISABLED: 'true' # ROCKET_ENCRYPTION_DISABLED: 'true' + # ROCKET_FEDERATION_ENABLED: 'true' # ROCKET_DATABASE_PATH: /srv/conduit/.local/share/conduit # ROCKET_WORKERS: 10 # ROCKET_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB diff --git a/src/database/globals.rs b/src/database/globals.rs index 8ce9c011..37f10eec 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -14,6 +14,7 @@ pub struct Globals { max_request_size: u32, registration_disabled: bool, encryption_disabled: bool, + federation_enabled: bool, } impl Globals { @@ -69,6 +70,7 @@ impl Globals { .map_err(|_| Error::BadConfig("Invalid max_request_size."))?, registration_disabled: config.get_bool("registration_disabled").unwrap_or(false), encryption_disabled: config.get_bool("encryption_disabled").unwrap_or(false), + federation_enabled: config.get_bool("federation_enabled").unwrap_or(false), }) } @@ -114,4 +116,8 @@ impl Globals { pub fn encryption_disabled(&self) -> bool { self.encryption_disabled } + + pub fn federation_enabled(&self) -> bool { + self.federation_enabled + } } diff --git a/src/server_server.rs b/src/server_server.rs index 0c175bfb..79976c09 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -57,6 +57,10 @@ pub async fn send_request( where T: Debug, { + if !globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let resolver = AsyncResolver::tokio_from_system_conf() .await .map_err(|_| Error::BadConfig("Failed to set up trust dns resolver with system config."))?; @@ -204,7 +208,11 @@ where } #[cfg_attr(feature = "conduit_bin", get("/_matrix/federation/v1/version"))] -pub fn get_server_version() -> ConduitResult { +pub fn get_server_version(db: State<'_, Database>) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + Ok(get_server_version::Response { server: Some(get_server_version::Server { name: Some("Conduit".to_owned()), @@ -216,6 +224,11 @@ pub fn get_server_version() -> ConduitResult { #[cfg_attr(feature = "conduit_bin", get("/_matrix/key/v2/server"))] pub fn get_server_keys(db: State<'_, Database>) -> Json { + if !db.globals.federation_enabled() { + // TODO: Use proper types + return Json("Federation is disabled.".to_owned()); + } + let mut verify_keys = BTreeMap::new(); verify_keys.insert( format!("ed25519:{}", db.globals.keypair().version()), @@ -259,6 +272,10 @@ pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let response = client_server::get_public_rooms_filtered_helper( &db, None, @@ -302,6 +319,10 @@ pub async fn get_public_rooms_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let response = client_server::get_public_rooms_filtered_helper( &db, None, @@ -345,6 +366,10 @@ pub fn send_transaction_message_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + //dbg!(&*body); for pdu in &body.pdus { let mut value = serde_json::from_str(pdu.json().get()) @@ -384,6 +409,10 @@ pub fn get_missing_events_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let mut queued_events = body.latest_events.clone(); let mut events = Vec::new(); @@ -427,6 +456,10 @@ pub fn get_profile_information_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let mut displayname = None; let mut avatar_url = None; @@ -455,6 +488,10 @@ pub fn get_user_devices_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let mut displayname = None; let mut avatar_url = None; From 304c53c4f53397a251e567a891c54ece7835173f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 7 Oct 2020 12:29:19 +0200 Subject: [PATCH 50/51] style: make clippy happier --- src/client_server/room.rs | 8 ++------ src/client_server/sync.rs | 2 +- src/database/sending.rs | 2 +- src/pdu.rs | 4 ++-- src/server_server.rs | 2 +- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 28d30e29..744d9490 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -354,12 +354,8 @@ pub async fn upgrade_room_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - // Validate the room version requested - let new_version = - RoomVersionId::try_from(body.new_version.clone()).expect("invalid room version id"); - if !matches!( - new_version, + body.new_version, RoomVersionId::Version5 | RoomVersionId::Version6 ) { return Err(Error::BadRequest( @@ -414,7 +410,7 @@ pub async fn upgrade_room_route( let mut create_event_content = ruma::events::room::create::CreateEventContent::new(sender_id.clone()); create_event_content.federate = federate; - create_event_content.room_version = new_version; + create_event_content.room_version = body.new_version.clone(); create_event_content.predecessor = predecessor; db.rooms.build_and_append_pdu( diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index aec03afa..6f411602 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -477,7 +477,7 @@ pub async fn sync_events_route( }) .and_then(|pdu| { serde_json::from_value::>( - pdu.content.clone(), + pdu.content, ) .expect("Raw::from_value always works") .deserialize() diff --git a/src/database/sending.rs b/src/database/sending.rs index c818cbfc..24a783b6 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -84,7 +84,7 @@ impl Sending { (Box, send_transaction_message::v1::Response), (Box, Error), > { - let pdu_json = PduEvent::to_outgoing_federation_event( + let pdu_json = PduEvent::convert_to_outgoing_federation_event( rooms .get_pdu_json_from_id(&pdu_id) .map_err(|e| (server.clone(), e))? diff --git a/src/pdu.rs b/src/pdu.rs index 4b1df4b8..7118bfc8 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -199,7 +199,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - pub fn to_outgoing_federation_event( + pub fn convert_to_outgoing_federation_event( mut pdu_json: serde_json::Value, ) -> Raw { if let Some(unsigned) = pdu_json @@ -239,7 +239,7 @@ impl From<&state_res::StateEvent> for PduEvent { content: pdu.content().clone(), state_key: Some(pdu.state_key()), prev_events: pdu.prev_event_ids(), - depth: pdu.depth().clone(), + depth: *pdu.depth(), auth_events: pdu.auth_events(), redacts: pdu.redacts().cloned(), unsigned: pdu.unsigned().clone().into_iter().collect(), diff --git a/src/server_server.rs b/src/server_server.rs index 79976c09..462d636f 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -438,7 +438,7 @@ pub fn get_missing_events_route<'a>( ) .map_err(|_| Error::bad_database("Invalid prev_events content in pdu in db."))?, ); - events.push(PduEvent::to_outgoing_federation_event(pdu)); + events.push(PduEvent::convert_to_outgoing_federation_event(pdu)); } i += 1; } From 9d1387954f0fcb59ddf805b972a7cbef489a7367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 16 Oct 2020 14:04:29 +0200 Subject: [PATCH 51/51] Update dependencies, remove dbgs --- Cargo.lock | 119 +++++++++++++++++++++++-------------------- rust-toolchain | 2 +- src/server_server.rs | 2 - 3 files changed, 64 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 857635db..a658ee2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,14 +90,14 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec1931848a574faa8f7c71a12ea00453ff5effbb5f51afe7f77d7a48cace6ac1" +checksum = "707b586e0e2f247cbde68cdd2c3ce69ea7b7be43e1c5b426e37c9319c4b9838e" dependencies = [ "addr2line", - "cfg-if", + "cfg-if 1.0.0", "libc", - "miniz_oxide 0.4.2", + "miniz_oxide 0.4.3", "object", "rustc-demangle", ] @@ -163,9 +163,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cc" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef611cc68ff783f18535d77ddd080185275713d852c4f5cbb6122c462a7a825c" +checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" [[package]] name = "cfg-if" @@ -173,6 +173,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "chrono" version = "0.4.19" @@ -197,9 +203,9 @@ dependencies = [ [[package]] name = "color_quant" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "conduit" @@ -271,7 +277,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -281,7 +287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 0.1.10", "crossbeam-utils", "lazy_static", "maybe-uninit", @@ -296,7 +302,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 0.1.10", "lazy_static", ] @@ -384,7 +390,7 @@ version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -566,7 +572,7 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -766,7 +772,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -881,7 +887,7 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -968,9 +974,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" dependencies = [ "adler", "autocfg", @@ -982,7 +988,7 @@ version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "fuchsia-zircon", "fuchsia-zircon-sys", "iovec", @@ -1042,7 +1048,7 @@ version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "winapi 0.3.9", ] @@ -1100,9 +1106,9 @@ dependencies = [ [[package]] name = "object" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" +checksum = "37fd5004feb2ce328a52b0b3d01dbf4ffff72583493900ed15f22d4111c51693" [[package]] name = "once_cell" @@ -1117,7 +1123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 0.1.10", "foreign-types", "lazy_static", "libc", @@ -1160,7 +1166,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "cloudabi", "instant", "libc", @@ -1198,18 +1204,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13fbdfd6bdee3dc9be46452f86af4a4072975899cf8592466668620bebfbcc17" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c82fb1329f632c3552cf352d14427d57a511b1cf41db93b3a7d77906a82dcc8e" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ "proc-macro2", "quote", @@ -1230,9 +1236,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" [[package]] name = "png" @@ -1389,9 +1395,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.9" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" dependencies = [ "regex-syntax", ] @@ -1408,9 +1414,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" +checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" [[package]] name = "remove_dir_all" @@ -1738,9 +1744,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" +checksum = "b2610b7f643d18c87dff3b489950269617e6601a51f1f05aa5daefee36f64f0b" [[package]] name = "rustc_version" @@ -1836,18 +1842,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96fe57af81d28386a513cbc6858332abc6117cfdb5999647c6444b8f43a370a5" +checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f630a6370fd8e457873b4bd2ffdae75408bc291ba72be773772a4c2a065d9ae8" +checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" dependencies = [ "proc-macro2", "quote", @@ -1856,9 +1862,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a230ea9107ca2220eea9d46de97eddcb04cd00e92d13dda78e478dd33fa82bd4" +checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" dependencies = [ "itoa", "ryu", @@ -1936,7 +1942,7 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "redox_syscall", "winapi 0.3.9", @@ -1950,9 +1956,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33a71ea1ea5f8747d1af1979bfb7e65c3a025a70609f04ceb78425bc5adad8e6" +checksum = "f4e0831040d2cf2bdfd51b844be71885783d489898a192f254ae25d57cce725c" dependencies = [ "version_check", ] @@ -1966,7 +1972,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#a7d76935f12757aecfee305838069c9bcbe7d34a" dependencies = [ "itertools", "js_int", @@ -2051,9 +2057,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.42" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c51d92969d209b54a98397e1b91c8ae82d8c87a7bb87df0b29aa2ad81454228" +checksum = "e03e57e4fcbfe7749842d53e24ccb9aa12b7252dbe5e91d2acad31834c8b8fdd" dependencies = [ "proc-macro2", "quote", @@ -2066,7 +2072,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "rand", "redox_syscall", @@ -2230,9 +2236,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" +checksum = "75cf45bb0bef80604d001caaec0d09da99611b3c0fd39d3080468875cdb65645" dependencies = [ "serde", ] @@ -2249,7 +2255,7 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "log", "pin-project-lite", "tracing-attributes", @@ -2299,9 +2305,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82bb5079aa76438620837198db8a5c529fb9878c730bc2b28179b0241cf04c10" +checksum = "4ef0a5e15477aa303afbfac3a44cba9b6430fdaad52423b1e6c0dbbe28c3eedd" dependencies = [ "ansi_term", "chrono", @@ -2313,6 +2319,7 @@ dependencies = [ "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", "tracing-serde", @@ -2345,7 +2352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f23cdfdc3d8300b3c50c9e84302d3bd6d860fb9529af84ace6cf9665f181b77" dependencies = [ "backtrace", - "cfg-if", + "cfg-if 0.1.10", "futures", "ipconfig", "lazy_static", @@ -2460,7 +2467,7 @@ version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "serde", "serde_json", "wasm-bindgen-macro", @@ -2487,7 +2494,7 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "js-sys", "wasm-bindgen", "web-sys", diff --git a/rust-toolchain b/rust-toolchain index 50aceaa7..21998d3c 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.45.0 +1.47.0 diff --git a/src/server_server.rs b/src/server_server.rs index 462d636f..b8b575eb 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -443,8 +443,6 @@ pub fn get_missing_events_route<'a>( i += 1; } - dbg!(&events); - Ok(get_missing_events::v1::Response { events }.into()) }