From e17bbdd42d3245f2fb3730753f1feb51cd452207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 28 Jan 2022 17:26:43 +0100 Subject: [PATCH 1/5] tests --- Cargo.toml | 2 +- src/database.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 78a4c8ff..0089e7f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ crossbeam = { version = "0.8.1", optional = true } num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } -rocksdb = { version = "0.17.0", default-features = false, features = ["multi-threaded-cf", "zstd"], optional = true } +rocksdb = { version = "0.17.0", default-features = true, features = ["multi-threaded-cf", "zstd"], optional = true } thread_local = "1.1.3" # used for TURN server authentication diff --git a/src/database.rs b/src/database.rs index 4f230f32..79b82088 100644 --- a/src/database.rs +++ b/src/database.rs @@ -130,7 +130,7 @@ fn default_db_cache_capacity_mb() -> f64 { } fn default_rocksdb_max_open_files() -> i32 { - 512 + 20 } fn default_pdu_cache_capacity() -> u32 { @@ -361,15 +361,15 @@ impl Database { .try_into() .expect("pdu cache capacity fits into usize"), )), - auth_chain_cache: Mutex::new(LruCache::new(1_000_000)), - shorteventid_cache: Mutex::new(LruCache::new(1_000_000)), - eventidshort_cache: Mutex::new(LruCache::new(1_000_000)), - shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)), - statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)), + auth_chain_cache: Mutex::new(LruCache::new(100_000)), + shorteventid_cache: Mutex::new(LruCache::new(100_000)), + eventidshort_cache: Mutex::new(LruCache::new(100_000)), + shortstatekey_cache: Mutex::new(LruCache::new(100_000)), + statekeyshort_cache: Mutex::new(LruCache::new(100_000)), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), lazy_load_waiting: Mutex::new(HashMap::new()), - stateinfo_cache: Mutex::new(LruCache::new(1000)), + stateinfo_cache: Mutex::new(LruCache::new(100)), }, account_data: account_data::AccountData { roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, From 23aecb78c7c5ba5872a058f806cd722787eefc10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 31 Jan 2022 15:39:46 +0100 Subject: [PATCH 2/5] fix: use to_lowercase on /register/available username --- src/client_server/account.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client_server/account.rs b/src/client_server/account.rs index c4e118c9..80c6f702 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -60,7 +60,7 @@ pub async fn get_register_available_route( body: Ruma>, ) -> ConduitResult { // Validate user id - let user_id = UserId::parse_with_server_name(body.username.clone(), db.globals.server_name()) + let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name()) .ok() .filter(|user_id| { !user_id.is_historical() && user_id.server_name() == db.globals.server_name() From caf9834e50b540fc48bf8cf50dc439c57c503de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 1 Feb 2022 14:42:13 +0100 Subject: [PATCH 3/5] feat: cache capacity modifier --- src/database.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/database.rs b/src/database.rs index 79b82088..449d71bd 100644 --- a/src/database.rs +++ b/src/database.rs @@ -49,6 +49,8 @@ pub struct Config { database_path: String, #[serde(default = "default_db_cache_capacity_mb")] db_cache_capacity_mb: f64, + #[serde(default = "default_conduit_cache_capacity_modifier")] + conduit_cache_capacity_modifier: f64, #[serde(default = "default_rocksdb_max_open_files")] rocksdb_max_open_files: i32, #[serde(default = "default_pdu_cache_capacity")] @@ -129,6 +131,10 @@ fn default_db_cache_capacity_mb() -> f64 { 10.0 } +fn default_conduit_cache_capacity_modifier() -> f64 { + 1.0 +} + fn default_rocksdb_max_open_files() -> i32 { 20 } @@ -361,15 +367,15 @@ impl Database { .try_into() .expect("pdu cache capacity fits into usize"), )), - auth_chain_cache: Mutex::new(LruCache::new(100_000)), - shorteventid_cache: Mutex::new(LruCache::new(100_000)), - eventidshort_cache: Mutex::new(LruCache::new(100_000)), - shortstatekey_cache: Mutex::new(LruCache::new(100_000)), - statekeyshort_cache: Mutex::new(LruCache::new(100_000)), + auth_chain_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + shorteventid_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + eventidshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + shortstatekey_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + statekeyshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), lazy_load_waiting: Mutex::new(HashMap::new()), - stateinfo_cache: Mutex::new(LruCache::new(100)), + stateinfo_cache: Mutex::new(LruCache::new((100.0 * config.conduit_cache_capacity_modifier) as usize)), }, account_data: account_data::AccountData { roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, From a5f004d7e9c783caf280884d7fd332c7bafa67ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 2 Feb 2022 12:36:55 +0100 Subject: [PATCH 4/5] fix: signature mismatch on odd send_join servers --- Cargo.toml | 2 +- src/client_server/account.rs | 19 ++++++++++--------- src/client_server/membership.rs | 17 +++++++++-------- src/database.rs | 24 ++++++++++++++++++------ 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0089e7f8..78a4c8ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ crossbeam = { version = "0.8.1", optional = true } num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } -rocksdb = { version = "0.17.0", default-features = true, features = ["multi-threaded-cf", "zstd"], optional = true } +rocksdb = { version = "0.17.0", default-features = false, features = ["multi-threaded-cf", "zstd"], optional = true } thread_local = "1.1.3" # used for TURN server authentication diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 80c6f702..ff348545 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -60,15 +60,16 @@ pub async fn get_register_available_route( body: Ruma>, ) -> ConduitResult { // Validate user id - let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name()) - .ok() - .filter(|user_id| { - !user_id.is_historical() && user_id.server_name() == db.globals.server_name() - }) - .ok_or(Error::BadRequest( - ErrorKind::InvalidUsername, - "Username is invalid.", - ))?; + let user_id = + UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name()) + .ok() + .filter(|user_id| { + !user_id.is_historical() && user_id.server_name() == db.globals.server_name() + }) + .ok_or(Error::BadRequest( + ErrorKind::InvalidUsername, + "Username is invalid.", + ))?; // Check if username is creative enough if db.users.exists(&user_id)? { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 70352784..216c4c0a 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -655,7 +655,7 @@ async fn join_room_by_id_helper( db.rooms.get_or_create_shortroomid(room_id, &db.globals)?; - let pdu = PduEvent::from_id_val(event_id, join_event.clone()) + let parsed_pdu = PduEvent::from_id_val(event_id, join_event.clone()) .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?; let mut state = HashMap::new(); @@ -695,14 +695,15 @@ async fn join_room_by_id_helper( } let incoming_shortstatekey = db.rooms.get_or_create_shortstatekey( - &pdu.kind, - pdu.state_key + &parsed_pdu.kind, + parsed_pdu + .state_key .as_ref() .expect("Pdu is a membership state event"), &db.globals, )?; - state.insert(incoming_shortstatekey, pdu.event_id.clone()); + state.insert(incoming_shortstatekey, parsed_pdu.event_id.clone()); let create_shortstatekey = db .rooms @@ -738,12 +739,12 @@ async fn join_room_by_id_helper( // We append to state before appending the pdu, so we don't have a moment in time with the // pdu without it's state. This is okay because append_pdu can't fail. - let statehashid = db.rooms.append_to_state(&pdu, &db.globals)?; + let statehashid = db.rooms.append_to_state(&parsed_pdu, &db.globals)?; db.rooms.append_pdu( - &pdu, - utils::to_canonical_object(&pdu).expect("Pdu is valid canonical object"), - iter::once(&*pdu.event_id), + &parsed_pdu, + join_event, + iter::once(&*parsed_pdu.event_id), db, )?; diff --git a/src/database.rs b/src/database.rs index 449d71bd..8d245b7f 100644 --- a/src/database.rs +++ b/src/database.rs @@ -367,15 +367,27 @@ impl Database { .try_into() .expect("pdu cache capacity fits into usize"), )), - auth_chain_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - shorteventid_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - eventidshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - shortstatekey_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - statekeyshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + auth_chain_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + shorteventid_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + eventidshort_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + shortstatekey_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + statekeyshort_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), lazy_load_waiting: Mutex::new(HashMap::new()), - stateinfo_cache: Mutex::new(LruCache::new((100.0 * config.conduit_cache_capacity_modifier) as usize)), + stateinfo_cache: Mutex::new(LruCache::new( + (100.0 * config.conduit_cache_capacity_modifier) as usize, + )), }, account_data: account_data::AccountData { roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, From 9ef3abacd43571300a7fbd7d35ba05d040816d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 2 Feb 2022 18:03:50 +0100 Subject: [PATCH 5/5] fix: initial state deserialize->serialize error --- src/client_server/room.rs | 7 +++++-- src/pdu.rs | 19 ++----------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 52d25425..a2339639 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -344,10 +344,13 @@ pub async fn create_room_route( // 6. Events listed in initial_state for event in &body.initial_state { - let pdu_builder = PduBuilder::from(event.deserialize().map_err(|e| { + let mut pdu_builder = event.deserialize_as::().map_err(|e| { warn!("Invalid initial state event: {:?}", e); Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event.") - })?); + })?; + + // Implicit state key defaults to "" + pdu_builder.state_key.get_or_insert_with(|| "".to_owned()); // Silently skip encryption events if they are not allowed if pdu_builder.event_type == EventType::RoomEncryption && !db.globals.allow_encryption() { diff --git a/src/pdu.rs b/src/pdu.rs index db9375e4..fe004609 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,9 +1,8 @@ use crate::Error; use ruma::{ events::{ - room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyInitialStateEvent, - AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, - EventType, StateEvent, + room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyRoomEvent, AnyStateEvent, + AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, serde::{CanonicalJsonObject, CanonicalJsonValue, Raw}, state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, UInt, UserId, @@ -361,17 +360,3 @@ pub struct PduBuilder { pub state_key: Option, pub redacts: Option>, } - -/// Direct conversion prevents loss of the empty `state_key` that ruma requires. -impl From for PduBuilder { - fn from(event: AnyInitialStateEvent) -> Self { - Self { - event_type: EventType::from(event.event_type()), - content: to_raw_value(&event.content()) - .expect("AnyStateEventContent came from JSON and can thus turn back into JSON."), - unsigned: None, - state_key: Some(event.state_key().to_owned()), - redacts: None, - } - } -}