From 0fcefa4125d924395063672cab9a238c8e6ff589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 15 Jul 2021 19:54:04 +0200 Subject: [PATCH 1/7] fix: ruma --- Cargo.lock | 23 ++--------- Cargo.toml | 8 ++-- src/client_server/account.rs | 17 ++++---- src/client_server/directory.rs | 16 ++++---- src/client_server/push.rs | 14 +++---- src/client_server/room.rs | 14 ++++--- src/database/rooms.rs | 73 +++++++++++++++++++++++++++++++++- src/server_server.rs | 16 ++------ 8 files changed, 115 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index befd3e2f..6c50f881 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2015,7 +2015,6 @@ dependencies = [ [[package]] name = "ruma" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "assign", "js_int", @@ -2036,7 +2035,6 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.1" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "bytes", "http", @@ -2052,7 +2050,6 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.1" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2063,7 +2060,6 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "ruma-api", "ruma-common", @@ -2077,7 +2073,6 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.11.0" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "assign", "bytes", @@ -2097,7 +2092,6 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.5.4" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "indexmap", "js_int", @@ -2111,8 +2105,7 @@ dependencies = [ [[package]] name = "ruma-events" -version = "0.23.1" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" +version = "0.23.2" dependencies = [ "indoc", "js_int", @@ -2122,12 +2115,12 @@ dependencies = [ "ruma-serde", "serde", "serde_json", + "thiserror", ] [[package]] name = "ruma-events-macros" -version = "0.23.1" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" +version = "0.23.2" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2138,7 +2131,6 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "js_int", "ruma-api", @@ -2153,7 +2145,6 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.19.4" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "paste", "rand 0.8.4", @@ -2167,7 +2158,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.19.4" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2177,12 +2167,10 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" [[package]] name = "ruma-identity-service-api" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "js_int", "ruma-api", @@ -2195,7 +2183,6 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "js_int", "ruma-api", @@ -2210,7 +2197,6 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "bytes", "form_urlencoded", @@ -2224,7 +2210,6 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2235,7 +2220,6 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.8.0" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2252,7 +2236,6 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=174555857ef90d49e4b9a672be9e2fe0acdc2687#174555857ef90d49e4b9a672be9e2fe0acdc2687" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 0f40ab78..2a0f6972 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ edition = "2018" rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/ruma/ruma", rev = "174555857ef90d49e4b9a672be9e2fe0acdc2687", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -#ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +#ruma = { git = "https://github.com/ruma/ruma", rev = "174555857ef90d49e4b9a672be9e2fe0acdc2687", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } # Used for long polling and federation sender, should be the same as rocket::tokio tokio = "1.2.0" @@ -119,5 +119,5 @@ maintainer-scripts = "debian/" systemd-units = { unit-name = "matrix-conduit" } # For flamegraphs: -#[profile.release] -#debug = true +[profile.release] +debug = true diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 9e16d90d..740a2dcc 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -1,4 +1,8 @@ -use std::{collections::BTreeMap, convert::TryInto, sync::Arc}; +use std::{ + collections::BTreeMap, + convert::{TryFrom, TryInto}, + sync::Arc, +}; use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; use crate::{database::DatabaseGuard, pdu::PduBuilder, utils, ConduitResult, Error, Ruma}; @@ -16,7 +20,8 @@ use ruma::{ }, events::{ room::{ - canonical_alias, guest_access, history_visibility, join_rules, member, message, name, + canonical_alias, guest_access, history_visibility, join_rules, member, message, + name::{self, RoomName}, topic, }, EventType, @@ -375,11 +380,9 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomName, - content: serde_json::to_value( - name::NameEventContent::new("Admin Room".to_owned()).map_err(|_| { - Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") - })?, - ) + content: serde_json::to_value(name::NameEventContent::new(Some( + RoomName::try_from("Admin Room".to_owned()).expect("Room name is valid"), + ))) .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 4a440fd9..36b817f5 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -87,17 +87,17 @@ pub async fn set_room_visibility_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); match &body.visibility { - room::Visibility::_Custom(_s) => { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Room visibility type is not supported.", - )); - } room::Visibility::Public => { db.rooms.set_public(&body.room_id, true)?; info!("{} made {} public", sender_user, body.room_id); } room::Visibility::Private => db.rooms.set_public(&body.room_id, false)?, + _ => { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Room visibility type is not supported.", + )); + } } db.flush().await?; @@ -231,8 +231,8 @@ pub async fn get_public_rooms_filtered_helper( .map_err(|_| { Error::bad_database("Invalid room name event in database.") })? - .name() - .map(|n| n.to_owned())) + .name + .map(|n| n.to_owned().into())) })?, num_joined_members: (db.rooms.room_members(&room_id).count() as u32).into(), topic: db diff --git a/src/client_server/push.rs b/src/client_server/push.rs index 794cbce4..33e62d9a 100644 --- a/src/client_server/push.rs +++ b/src/client_server/push.rs @@ -81,7 +81,7 @@ pub async fn get_pushrule_route( .content .get(body.rule_id.as_str()) .map(|rule| rule.clone().into()), - RuleKind::_Custom(_) => None, + _ => None, }; if let Some(rule) = rule { @@ -181,7 +181,7 @@ pub async fn set_pushrule_route( .into(), ); } - RuleKind::_Custom(_) => {} + _ => {} } db.account_data.update( @@ -245,7 +245,7 @@ pub async fn get_pushrule_actions_route( .content .get(body.rule_id.as_str()) .map(|rule| rule.actions.clone()), - RuleKind::_Custom(_) => None, + _ => None, }; db.flush().await?; @@ -314,7 +314,7 @@ pub async fn set_pushrule_actions_route( global.content.replace(rule); } } - RuleKind::_Custom(_) => {} + _ => {} }; db.account_data.update( @@ -383,7 +383,7 @@ pub async fn get_pushrule_enabled_route( .iter() .find(|rule| rule.rule_id == body.rule_id) .map_or(false, |rule| rule.enabled), - RuleKind::_Custom(_) => false, + _ => false, }; db.flush().await?; @@ -454,7 +454,7 @@ pub async fn set_pushrule_enabled_route( global.content.insert(rule); } } - RuleKind::_Custom(_) => {} + _ => {} } db.account_data.update( @@ -523,7 +523,7 @@ pub async fn delete_pushrule_route( global.content.remove(&rule); } } - RuleKind::_Custom(_) => {} + _ => {} } db.account_data.update( diff --git a/src/client_server/room.rs b/src/client_server/room.rs index f48c5e93..9f381a02 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -9,7 +9,11 @@ use ruma::{ r0::room::{self, create_room, get_room_event, upgrade_room}, }, events::{ - room::{guest_access, history_visibility, join_rules, member, name, topic}, + room::{ + guest_access, history_visibility, join_rules, member, + name::{self, RoomName}, + topic, + }, EventType, }, serde::Raw, @@ -113,7 +117,7 @@ pub async fn create_room_route( .unwrap_or_else(|| match &body.visibility { room::Visibility::Private => create_room::RoomPreset::PrivateChat, room::Visibility::Public => create_room::RoomPreset::PublicChat, - room::Visibility::_Custom(_) => create_room::RoomPreset::PrivateChat, // Room visibility should not be custom + _ => create_room::RoomPreset::PrivateChat, // Room visibility should not be custom }); let mut users = BTreeMap::new(); @@ -251,11 +255,11 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomName, - content: serde_json::to_value( - name::NameEventContent::new(name.clone()).map_err(|_| { + content: serde_json::to_value(name::NameEventContent::new(Some( + RoomName::try_from(name.clone()).map_err(|_| { Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") })?, - ) + ))) .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 1542db85..f6f50215 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -490,6 +490,27 @@ impl Rooms { .transpose() } + /// Returns the json of a pdu. + pub fn get_non_outlier_pdu_json( + &self, + event_id: &EventId, + ) -> Result> { + self.eventid_pduid + .get(event_id.as_bytes())? + .map_or_else::, _, _>( + || Ok(None), + |pduid| { + Ok(Some(self.pduid_pdu.get(&pduid)?.ok_or_else(|| { + Error::bad_database("Invalid pduid in eventid_pduid.") + })?)) + }, + )? + .map(|pdu| { + serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db.")) + }) + .transpose() + } + /// Returns the pdu's id. pub fn get_pdu_id(&self, event_id: &EventId) -> Result>> { self.eventid_pduid @@ -903,11 +924,59 @@ impl Rooms { "list_appservices" => { db.admin.send(AdminCommand::ListAppservices); } + "get_pdu" => { + if args.len() == 1 { + if let Ok(event_id) = EventId::try_from(args[0]) { + let mut outlier = false; + let mut pdu_json = + db.rooms.get_non_outlier_pdu_json(&event_id)?; + if pdu_json.is_none() { + outlier = true; + pdu_json = db.rooms.get_pdu_json(&event_id)?; + } + match pdu_json { + Some(json) => { + db.admin.send(AdminCommand::SendMessage( + message::MessageEventContent::text_html( + format!("{}\n```json\n{:#?}\n```", + if outlier { + "PDU is outlier" + } else { "PDU was accepted"}, json), + format!("

{}

\n
{}\n
\n", + if outlier { + "PDU is outlier" + } else { "PDU was accepted"}, serde_json::to_string_pretty(&json).expect("canonical json is valid json")) + ), + )); + } + None => { + db.admin.send(AdminCommand::SendMessage( + message::MessageEventContent::text_plain( + "PDU not found.", + ), + )); + } + } + } else { + db.admin.send(AdminCommand::SendMessage( + message::MessageEventContent::text_plain( + "Event ID could not be parsed.", + ), + )); + } + } else { + db.admin.send(AdminCommand::SendMessage( + message::MessageEventContent::text_plain( + "Usage: get_pdu ", + ), + )); + } + } _ => { db.admin.send(AdminCommand::SendMessage( message::MessageEventContent::text_plain(format!( - "Command: {}, Args: {:?}", - command, args + "Unrecognized command: {}", + command )), )); } diff --git a/src/server_server.rs b/src/server_server.rs index fb49d0ca..bc854079 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1254,15 +1254,6 @@ pub fn handle_incoming_pdu<'a>( // We do need to force an update to this room's state update_state = true; - let mut auth_events = vec![]; - for map in &fork_states { - let state_auth = map - .values() - .flat_map(|pdu| pdu.auth_events.clone()) - .collect(); - auth_events.push(state_auth); - } - match state_res::StateResolution::resolve( &room_id, room_version_id, @@ -1274,8 +1265,7 @@ pub fn handle_incoming_pdu<'a>( .collect::>() }) .collect::>(), - auth_events, - &|id| { + |id| { let res = db.rooms.get_pdu(id); if let Err(e) = &res { error!("LOOK AT ME Failed to fetch event: {}", e); @@ -2432,10 +2422,10 @@ pub fn get_profile_information_route( let mut avatar_url = None; match &body.field { - // TODO: what to do with custom - Some(ProfileField::_Custom(_s)) => {} Some(ProfileField::DisplayName) => displayname = db.users.displayname(&body.user_id)?, Some(ProfileField::AvatarUrl) => avatar_url = db.users.avatar_url(&body.user_id)?, + // TODO: what to do with custom + Some(_) => {} None => { displayname = db.users.displayname(&body.user_id)?; avatar_url = db.users.avatar_url(&body.user_id)?; From f5273f7eb11c00e84e3ccd30cc4d5485d7ff5e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 15 Jul 2021 23:17:58 +0200 Subject: [PATCH 2/7] improvement: upgrade ruma and implement blurhashes --- Cargo.lock | 18 ++++++++++++++++++ Cargo.toml | 4 ++-- src/client_server/account.rs | 6 +++++- src/client_server/backup.rs | 4 ++-- src/client_server/config.rs | 4 ++-- src/client_server/device.rs | 6 +++--- src/client_server/directory.rs | 2 +- src/client_server/keys.rs | 4 ++-- src/client_server/membership.rs | 7 ++++++- src/client_server/mod.rs | 2 +- src/client_server/presence.rs | 2 +- src/client_server/profile.rs | 8 ++++++-- src/client_server/push.rs | 8 ++++---- src/client_server/read_marker.rs | 4 ++-- src/client_server/room.rs | 2 ++ src/client_server/sync.rs | 1 + src/client_server/tag.rs | 4 ++-- src/client_server/to_device.rs | 2 +- src/client_server/typing.rs | 2 +- src/database.rs | 1 + src/database/users.rs | 28 +++++++++++++++++++++++++++- src/server_server.rs | 9 ++++++++- 22 files changed, 98 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c50f881..828d7cc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2015,6 +2015,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.2.0" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "assign", "js_int", @@ -2035,6 +2036,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.1" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "bytes", "http", @@ -2050,6 +2052,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.1" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2060,6 +2063,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.3.0" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "ruma-api", "ruma-common", @@ -2073,6 +2077,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.11.0" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "assign", "bytes", @@ -2092,6 +2097,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.5.4" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "indexmap", "js_int", @@ -2106,6 +2112,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.23.2" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "indoc", "js_int", @@ -2121,6 +2128,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.23.2" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2131,6 +2139,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.2.0" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "js_int", "ruma-api", @@ -2145,6 +2154,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.19.4" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "paste", "rand 0.8.4", @@ -2158,6 +2168,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.19.4" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2167,10 +2178,12 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.4.0" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" [[package]] name = "ruma-identity-service-api" version = "0.2.0" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "js_int", "ruma-api", @@ -2183,6 +2196,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.2.0" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "js_int", "ruma-api", @@ -2197,6 +2211,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.4.1" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "bytes", "form_urlencoded", @@ -2210,6 +2225,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.4.1" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2220,6 +2236,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.8.0" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2236,6 +2253,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.2.0" +source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 2a0f6972..fd72d0e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ edition = "2018" rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests # Used for matrix spec type definitions and helpers -#ruma = { git = "https://github.com/ruma/ruma", rev = "174555857ef90d49e4b9a672be9e2fe0acdc2687", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "c29c2b16ec114fa655e2b70bdd53c82e35859005", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +#ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } # Used for long polling and federation sender, should be the same as rocket::tokio tokio = "1.2.0" diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 740a2dcc..ebaf7b17 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -283,6 +283,7 @@ pub async fn register_route( avatar_url: None, is_direct: None, third_party_invite: None, + blurhash: None, }) .expect("event is valid, we just created it"), unsigned: None, @@ -446,6 +447,7 @@ pub async fn register_route( avatar_url: None, is_direct: None, third_party_invite: None, + blurhash: None, }) .expect("event is valid, we just created it"), unsigned: None, @@ -466,6 +468,7 @@ pub async fn register_route( avatar_url: None, is_direct: None, third_party_invite: None, + blurhash: None, }) .expect("event is valid, we just created it"), unsigned: None, @@ -578,7 +581,7 @@ pub async fn change_password_route( db.flush().await?; - Ok(change_password::Response.into()) + Ok(change_password::Response {}.into()) } /// # `GET _matrix/client/r0/account/whoami` @@ -664,6 +667,7 @@ pub async fn deactivate_route( avatar_url: None, is_direct: None, third_party_invite: None, + blurhash: None, }; let mutex = Arc::clone( diff --git a/src/client_server/backup.rs b/src/client_server/backup.rs index ccb17faa..6d540cb1 100644 --- a/src/client_server/backup.rs +++ b/src/client_server/backup.rs @@ -46,7 +46,7 @@ pub async fn update_backup_route( db.flush().await?; - Ok(update_backup::Response.into()) + Ok(update_backup::Response {}.into()) } #[cfg_attr( @@ -119,7 +119,7 @@ pub async fn delete_backup_route( db.flush().await?; - Ok(delete_backup::Response.into()) + Ok(delete_backup::Response {}.into()) } /// Add the received backup keys to the database. diff --git a/src/client_server/config.rs b/src/client_server/config.rs index 4f33689a..b9826bfb 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -45,7 +45,7 @@ pub async fn set_global_account_data_route( db.flush().await?; - Ok(set_global_account_data::Response.into()) + Ok(set_global_account_data::Response {}.into()) } #[cfg_attr( @@ -80,7 +80,7 @@ pub async fn set_room_account_data_route( db.flush().await?; - Ok(set_room_account_data::Response.into()) + Ok(set_room_account_data::Response {}.into()) } #[cfg_attr( diff --git a/src/client_server/device.rs b/src/client_server/device.rs index 44b9c32a..085d034f 100644 --- a/src/client_server/device.rs +++ b/src/client_server/device.rs @@ -73,7 +73,7 @@ pub async fn update_device_route( db.flush().await?; - Ok(update_device::Response.into()) + Ok(update_device::Response {}.into()) } #[cfg_attr( @@ -125,7 +125,7 @@ pub async fn delete_device_route( db.flush().await?; - Ok(delete_device::Response.into()) + Ok(delete_device::Response {}.into()) } #[cfg_attr( @@ -179,5 +179,5 @@ pub async fn delete_devices_route( db.flush().await?; - Ok(delete_devices::Response.into()) + Ok(delete_devices::Response {}.into()) } diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 36b817f5..64375f00 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -102,7 +102,7 @@ pub async fn set_room_visibility_route( db.flush().await?; - Ok(set_room_visibility::Response.into()) + Ok(set_room_visibility::Response {}.into()) } #[cfg_attr( diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index 8eee4080..8f1afba9 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -163,7 +163,7 @@ pub async fn upload_signing_keys_route( db.flush().await?; - Ok(upload_signing_keys::Response.into()) + Ok(upload_signing_keys::Response {}.into()) } #[cfg_attr( @@ -224,7 +224,7 @@ pub async fn upload_signatures_route( db.flush().await?; - Ok(upload_signatures::Response.into()) + Ok(upload_signatures::Response {}.into()) } #[cfg_attr( diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index a74950b6..ef141f04 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -165,7 +165,7 @@ pub async fn invite_user_route( if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { invite_helper(sender_user, user_id, &body.room_id, &db, false).await?; db.flush().await?; - Ok(invite_user::Response.into()) + Ok(invite_user::Response {}.into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "User not found.")) } @@ -261,6 +261,7 @@ pub async fn ban_user_route( avatar_url: db.users.avatar_url(&body.user_id)?, is_direct: None, third_party_invite: None, + blurhash: db.users.blurhash(&body.user_id)?, }), |event| { let mut event = serde_json::from_value::>( @@ -556,6 +557,7 @@ async fn join_room_by_id_helper( avatar_url: db.users.avatar_url(&sender_user)?, is_direct: None, third_party_invite: None, + blurhash: db.users.blurhash(&sender_user)?, }) .expect("event is valid, we just created it"), ); @@ -685,6 +687,7 @@ async fn join_room_by_id_helper( avatar_url: db.users.avatar_url(&sender_user)?, is_direct: None, third_party_invite: None, + blurhash: db.users.blurhash(&sender_user)?, }; db.rooms.build_and_append_pdu( @@ -833,6 +836,7 @@ pub async fn invite_helper<'a>( is_direct: Some(is_direct), membership: MembershipState::Invite, third_party_invite: None, + blurhash: db.users.blurhash(&sender_user)?, }) .expect("member event is valid value"); @@ -1008,6 +1012,7 @@ pub async fn invite_helper<'a>( avatar_url: db.users.avatar_url(&user_id)?, is_direct: Some(is_direct), third_party_invite: None, + blurhash: db.users.blurhash(&sender_user)?, }) .expect("event is valid, we just created it"), unsigned: None, diff --git a/src/client_server/mod.rs b/src/client_server/mod.rs index f211a571..040015dc 100644 --- a/src/client_server/mod.rs +++ b/src/client_server/mod.rs @@ -75,5 +75,5 @@ pub const SESSION_ID_LENGTH: usize = 256; #[options("/<_..>")] #[tracing::instrument] pub async fn options_route() -> ConduitResult { - Ok(send_event_to_device::Response.into()) + Ok(send_event_to_device::Response {}.into()) } diff --git a/src/client_server/presence.rs b/src/client_server/presence.rs index bfe638fb..ca78a88b 100644 --- a/src/client_server/presence.rs +++ b/src/client_server/presence.rs @@ -43,7 +43,7 @@ pub async fn set_presence_route( db.flush().await?; - Ok(set_presence::Response.into()) + Ok(set_presence::Response {}.into()) } #[cfg_attr( diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index d947bbe1..1938c87c 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -108,7 +108,7 @@ pub async fn set_displayname_route( db.flush().await?; - Ok(set_display_name::Response.into()) + Ok(set_display_name::Response {}.into()) } #[cfg_attr( @@ -140,6 +140,8 @@ pub async fn set_avatar_url_route( db.users .set_avatar_url(&sender_user, body.avatar_url.clone())?; + db.users.set_blurhash(&sender_user, body.blurhash.clone())?; + // Send a new membership event and presence update into all joined rooms for (pdu_builder, room_id) in db .rooms @@ -220,7 +222,7 @@ pub async fn set_avatar_url_route( db.flush().await?; - Ok(set_avatar_url::Response.into()) + Ok(set_avatar_url::Response {}.into()) } #[cfg_attr( @@ -234,6 +236,7 @@ pub async fn get_avatar_url_route( ) -> ConduitResult { Ok(get_avatar_url::Response { avatar_url: db.users.avatar_url(&body.user_id)?, + blurhash: db.users.blurhash(&body.user_id)?, } .into()) } @@ -257,6 +260,7 @@ pub async fn get_profile_route( Ok(get_profile::Response { avatar_url: db.users.avatar_url(&body.user_id)?, + blurhash: db.users.blurhash(&body.user_id)?, displayname: db.users.displayname(&body.user_id)?, } .into()) diff --git a/src/client_server/push.rs b/src/client_server/push.rs index 33e62d9a..867b4525 100644 --- a/src/client_server/push.rs +++ b/src/client_server/push.rs @@ -194,7 +194,7 @@ pub async fn set_pushrule_route( db.flush().await?; - Ok(set_pushrule::Response.into()) + Ok(set_pushrule::Response {}.into()) } #[cfg_attr( @@ -327,7 +327,7 @@ pub async fn set_pushrule_actions_route( db.flush().await?; - Ok(set_pushrule_actions::Response.into()) + Ok(set_pushrule_actions::Response {}.into()) } #[cfg_attr( @@ -467,7 +467,7 @@ pub async fn set_pushrule_enabled_route( db.flush().await?; - Ok(set_pushrule_enabled::Response.into()) + Ok(set_pushrule_enabled::Response {}.into()) } #[cfg_attr( @@ -536,7 +536,7 @@ pub async fn delete_pushrule_route( db.flush().await?; - Ok(delete_pushrule::Response.into()) + Ok(delete_pushrule::Response {}.into()) } #[cfg_attr( diff --git a/src/client_server/read_marker.rs b/src/client_server/read_marker.rs index fe49af9d..f5e2924e 100644 --- a/src/client_server/read_marker.rs +++ b/src/client_server/read_marker.rs @@ -77,7 +77,7 @@ pub async fn set_read_marker_route( db.flush().await?; - Ok(set_read_marker::Response.into()) + Ok(set_read_marker::Response {}.into()) } #[cfg_attr( @@ -130,5 +130,5 @@ pub async fn create_receipt_route( db.flush().await?; - Ok(create_receipt::Response.into()) + Ok(create_receipt::Response {}.into()) } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 9f381a02..c9b93c35 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -96,6 +96,7 @@ pub async fn create_room_route( avatar_url: db.users.avatar_url(&sender_user)?, is_direct: Some(body.is_direct), third_party_invite: None, + blurhash: db.users.blurhash(&sender_user)?, }) .expect("event is valid, we just created it"), unsigned: None, @@ -444,6 +445,7 @@ pub async fn upgrade_room_route( avatar_url: db.users.avatar_url(&sender_user)?, is_direct: None, third_party_invite: None, + blurhash: db.users.blurhash(&sender_user)?, }) .expect("event is valid, we just created it"), unsigned: None, diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index fe113048..3beddad6 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -753,6 +753,7 @@ async fn sync_helper( leave: left_rooms, join: joined_rooms, invite: invited_rooms, + knock: BTreeMap::new(), // TODO }, presence: sync_events::Presence { events: presence_updates diff --git a/src/client_server/tag.rs b/src/client_server/tag.rs index 17df2c2e..223d122c 100644 --- a/src/client_server/tag.rs +++ b/src/client_server/tag.rs @@ -42,7 +42,7 @@ pub async fn update_tag_route( db.flush().await?; - Ok(create_tag::Response.into()) + Ok(create_tag::Response {}.into()) } #[cfg_attr( @@ -76,7 +76,7 @@ pub async fn delete_tag_route( db.flush().await?; - Ok(delete_tag::Response.into()) + Ok(delete_tag::Response {}.into()) } #[cfg_attr( diff --git a/src/client_server/to_device.rs b/src/client_server/to_device.rs index 9faa2555..7896af93 100644 --- a/src/client_server/to_device.rs +++ b/src/client_server/to_device.rs @@ -69,5 +69,5 @@ pub async fn send_event_to_device_route( db.flush().await?; - Ok(send_event_to_device::Response.into()) + Ok(send_event_to_device::Response {}.into()) } diff --git a/src/client_server/typing.rs b/src/client_server/typing.rs index 7a590af9..50082ee4 100644 --- a/src/client_server/typing.rs +++ b/src/client_server/typing.rs @@ -29,5 +29,5 @@ pub fn create_typing_event_route( .typing_remove(&sender_user, &body.room_id, &db.globals)?; } - Ok(create_typing_event::Response.into()) + Ok(create_typing_event::Response {}.into()) } diff --git a/src/database.rs b/src/database.rs index 9452e638..27b9eb60 100644 --- a/src/database.rs +++ b/src/database.rs @@ -229,6 +229,7 @@ impl Database { userid_password: builder.open_tree("userid_password")?, userid_displayname: builder.open_tree("userid_displayname")?, userid_avatarurl: builder.open_tree("userid_avatarurl")?, + userid_blurhash: builder.open_tree("userid_blurhash")?, userdeviceid_token: builder.open_tree("userdeviceid_token")?, userdeviceid_metadata: builder.open_tree("userdeviceid_metadata")?, userid_devicelistversion: builder.open_tree("userid_devicelistversion")?, diff --git a/src/database/users.rs b/src/database/users.rs index 1480d3fa..cd46c458 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -15,6 +15,7 @@ pub struct Users { pub(super) userid_password: Arc, pub(super) userid_displayname: Arc, pub(super) userid_avatarurl: Arc, + pub(super) userid_blurhash: Arc, pub(super) userdeviceid_token: Arc, pub(super) userdeviceid_metadata: Arc, // This is also used to check if a device exists pub(super) userid_devicelistversion: Arc, // DevicelistVersion = u64 @@ -150,7 +151,7 @@ impl Users { Ok(()) } - /// Get a the avatar_url of a user. + /// Get the avatar_url of a user. pub fn avatar_url(&self, user_id: &UserId) -> Result> { self.userid_avatarurl .get(user_id.as_bytes())? @@ -174,6 +175,31 @@ impl Users { Ok(()) } + /// Get the blurhash of a user. + pub fn blurhash(&self, user_id: &UserId) -> Result> { + self.userid_blurhash + .get(user_id.as_bytes())? + .map(|bytes| { + let s = utils::string_from_bytes(&bytes) + .map_err(|_| Error::bad_database("Avatar URL in db is invalid."))?; + + Ok(s) + }) + .transpose() + } + + /// Sets a new avatar_url or removes it if avatar_url is None. + pub fn set_blurhash(&self, user_id: &UserId, blurhash: Option) -> Result<()> { + if let Some(blurhash) = blurhash { + self.userid_blurhash + .insert(user_id.as_bytes(), blurhash.as_bytes())?; + } else { + self.userid_blurhash.remove(user_id.as_bytes())?; + } + + Ok(()) + } + /// Adds a new device to a user. pub fn create_device( &self, diff --git a/src/server_server.rs b/src/server_server.rs index bc854079..bfb3e727 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -2001,6 +2001,7 @@ pub fn create_join_event_template_route( let content = serde_json::to_value(MemberEventContent { avatar_url: None, + blurhash: None, displayname: None, is_direct: None, membership: MembershipState::Join, @@ -2420,19 +2421,25 @@ pub fn get_profile_information_route( let mut displayname = None; let mut avatar_url = None; + let mut blurhash = None; match &body.field { Some(ProfileField::DisplayName) => displayname = db.users.displayname(&body.user_id)?, - Some(ProfileField::AvatarUrl) => avatar_url = db.users.avatar_url(&body.user_id)?, + Some(ProfileField::AvatarUrl) => { + avatar_url = db.users.avatar_url(&body.user_id)?; + blurhash = db.users.blurhash(&body.user_id)? + } // TODO: what to do with custom Some(_) => {} None => { displayname = db.users.displayname(&body.user_id)?; avatar_url = db.users.avatar_url(&body.user_id)?; + blurhash = db.users.blurhash(&body.user_id)?; } } Ok(get_profile_information::v1::Response { + blurhash, displayname, avatar_url, } From cfaa900e8306b35c2d9b719b96bfce29ac37f58f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 18 Jul 2021 20:43:39 +0200 Subject: [PATCH 3/7] improvement: auth chain cache --- Cargo.lock | 36 ++-- Cargo.toml | 7 +- src/client_server/membership.rs | 4 +- src/client_server/sync.rs | 6 +- src/database.rs | 5 +- src/database/abstraction/rocksdb.rs | 6 +- src/database/abstraction/sqlite.rs | 6 +- src/database/globals.rs | 24 +-- src/database/rooms.rs | 23 ++- src/server_server.rs | 260 +++++++++++++++------------- 10 files changed, 201 insertions(+), 176 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 828d7cc9..8bfba0e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2015,7 +2015,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "assign", "js_int", @@ -2036,7 +2036,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.1" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "bytes", "http", @@ -2052,7 +2052,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.1" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "ruma-api", "ruma-common", @@ -2077,7 +2077,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.11.0" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "assign", "bytes", @@ -2097,7 +2097,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.5.4" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "indexmap", "js_int", @@ -2112,7 +2112,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.23.2" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "indoc", "js_int", @@ -2128,7 +2128,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.23.2" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2139,7 +2139,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "js_int", "ruma-api", @@ -2154,7 +2154,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.19.4" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "paste", "rand 0.8.4", @@ -2168,7 +2168,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.19.4" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2178,12 +2178,12 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" [[package]] name = "ruma-identity-service-api" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "js_int", "ruma-api", @@ -2196,7 +2196,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "js_int", "ruma-api", @@ -2211,7 +2211,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "bytes", "form_urlencoded", @@ -2225,7 +2225,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2236,7 +2236,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.8.0" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2253,7 +2253,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=c29c2b16ec114fa655e2b70bdd53c82e35859005#c29c2b16ec114fa655e2b70bdd53c82e35859005" +source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index fd72d0e6..64d67a14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,8 @@ edition = "2018" rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/ruma/ruma", rev = "c29c2b16ec114fa655e2b70bdd53c82e35859005", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +#ruma = { git = "https://github.com/ruma/ruma", rev = "c29c2b16ec114fa655e2b70bdd53c82e35859005", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/timokoesters/ruma", rev = "a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } # Used for long polling and federation sender, should be the same as rocket::tokio @@ -119,5 +120,5 @@ maintainer-scripts = "debian/" systemd-units = { unit-name = "matrix-conduit" } # For flamegraphs: -[profile.release] -debug = true +#[profile.release] +#debug = true diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index ef141f04..d8c27816 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -29,7 +29,7 @@ use ruma::{ uint, EventId, RoomId, RoomVersionId, ServerName, UserId, }; use std::{ - collections::{btree_map::Entry, BTreeMap, HashSet}, + collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, convert::{TryFrom, TryInto}, sync::{Arc, RwLock}, time::{Duration, Instant}, @@ -607,7 +607,7 @@ async fn join_room_by_id_helper( let pdu = PduEvent::from_id_val(&event_id, join_event.clone()) .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?; - let mut state = BTreeMap::new(); + let mut state = HashMap::new(); let pub_key_map = RwLock::new(BTreeMap::new()); for result in futures::future::join_all( diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 3beddad6..65922bec 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -7,7 +7,7 @@ use ruma::{ DeviceId, RoomId, UserId, }; use std::{ - collections::{btree_map::Entry, hash_map, BTreeMap, HashMap, HashSet}, + collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, convert::{TryFrom, TryInto}, sync::Arc, time::Duration, @@ -622,10 +622,10 @@ async fn sync_helper( .presence_since(&room_id, since, &db.rooms, &db.globals)? { match presence_updates.entry(user_id) { - hash_map::Entry::Vacant(v) => { + Entry::Vacant(v) => { v.insert(presence); } - hash_map::Entry::Occupied(mut o) => { + Entry::Occupied(mut o) => { let p = o.get_mut(); // Update existing presence event with more info diff --git a/src/database.rs b/src/database.rs index 27b9eb60..e359a5f1 100644 --- a/src/database.rs +++ b/src/database.rs @@ -33,7 +33,7 @@ use std::{ io::Write, ops::Deref, path::Path, - sync::{Arc, RwLock}, + sync::{Arc, Mutex, RwLock}, }; use tokio::sync::{OwnedRwLockReadGuard, RwLock as TokioRwLock, Semaphore}; @@ -292,7 +292,8 @@ impl Database { eventid_outlierpdu: builder.open_tree("eventid_outlierpdu")?, prevevent_parent: builder.open_tree("prevevent_parent")?, - pdu_cache: RwLock::new(LruCache::new(10_000)), + pdu_cache: Mutex::new(LruCache::new(100_000)), + auth_chain_cache: Mutex::new(LruCache::new(100_000)), }, account_data: account_data::AccountData { roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index b9961302..4699b2d5 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -5,14 +5,14 @@ use std::{future::Future, pin::Pin, sync::Arc}; use super::{DatabaseEngine, Tree}; -use std::{collections::BTreeMap, sync::RwLock}; +use std::{collections::HashMap, sync::RwLock}; pub struct Engine(rocksdb::DBWithThreadMode); pub struct RocksDbEngineTree<'a> { db: Arc, name: &'a str, - watchers: RwLock, Vec>>>, + watchers: RwLock, Vec>>>, } impl DatabaseEngine for Engine { @@ -58,7 +58,7 @@ impl DatabaseEngine for Engine { Ok(Arc::new(RocksDbEngineTree { name, db: Arc::clone(self), - watchers: RwLock::new(BTreeMap::new()), + watchers: RwLock::new(HashMap::new()), })) } } diff --git a/src/database/abstraction/sqlite.rs b/src/database/abstraction/sqlite.rs index 8100ed91..8cc6a8de 100644 --- a/src/database/abstraction/sqlite.rs +++ b/src/database/abstraction/sqlite.rs @@ -7,7 +7,7 @@ use log::debug; use parking_lot::{Mutex, MutexGuard, RwLock}; use rusqlite::{params, Connection, DatabaseName::Main, OptionalExtension}; use std::{ - collections::BTreeMap, + collections::HashMap, future::Future, ops::Deref, path::{Path, PathBuf}, @@ -206,7 +206,7 @@ impl DatabaseEngine for Engine { Ok(Arc::new(SqliteTable { engine: Arc::clone(self), name: name.to_owned(), - watchers: RwLock::new(BTreeMap::new()), + watchers: RwLock::new(HashMap::new()), })) } @@ -266,7 +266,7 @@ impl Engine { pub struct SqliteTable { engine: Arc, name: String, - watchers: RwLock, Vec>>>, + watchers: RwLock, Vec>>>, } type TupleOfBytes = (Vec, Vec); diff --git a/src/database/globals.rs b/src/database/globals.rs index 0e722973..fbd41a38 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -41,12 +41,12 @@ pub struct Globals { dns_resolver: TokioAsyncResolver, jwt_decoding_key: Option>, pub(super) server_signingkeys: Arc, - pub bad_event_ratelimiter: Arc>>, - pub bad_signature_ratelimiter: Arc, RateLimitState>>>, - pub servername_ratelimiter: Arc, Arc>>>, - pub sync_receivers: RwLock), SyncHandle>>, - pub roomid_mutex: RwLock>>>, - pub roomid_mutex_federation: RwLock>>>, // this lock will be held longer + pub bad_event_ratelimiter: Arc>>, + pub bad_signature_ratelimiter: Arc, RateLimitState>>>, + pub servername_ratelimiter: Arc, Arc>>>, + pub sync_receivers: RwLock), SyncHandle>>, + pub roomid_mutex: RwLock>>>, + pub roomid_mutex_federation: RwLock>>>, // this lock will be held longer pub rotate: RotationHandler, } @@ -196,12 +196,12 @@ impl Globals { tls_name_override, server_signingkeys, jwt_decoding_key, - bad_event_ratelimiter: Arc::new(RwLock::new(BTreeMap::new())), - bad_signature_ratelimiter: Arc::new(RwLock::new(BTreeMap::new())), - servername_ratelimiter: Arc::new(RwLock::new(BTreeMap::new())), - roomid_mutex: RwLock::new(BTreeMap::new()), - roomid_mutex_federation: RwLock::new(BTreeMap::new()), - sync_receivers: RwLock::new(BTreeMap::new()), + bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), + bad_signature_ratelimiter: Arc::new(RwLock::new(HashMap::new())), + servername_ratelimiter: Arc::new(RwLock::new(HashMap::new())), + roomid_mutex: RwLock::new(HashMap::new()), + roomid_mutex_federation: RwLock::new(HashMap::new()), + sync_receivers: RwLock::new(HashMap::new()), rotate: RotationHandler::new(), }; diff --git a/src/database/rooms.rs b/src/database/rooms.rs index f6f50215..fa121bda 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -25,7 +25,7 @@ use std::{ collections::{BTreeMap, BTreeSet, HashMap, HashSet}, convert::{TryFrom, TryInto}, mem, - sync::{Arc, RwLock}, + sync::{Arc, Mutex}, }; use super::{abstraction::Tree, admin::AdminCommand, pusher}; @@ -84,7 +84,8 @@ pub struct Rooms { /// RoomId + EventId -> Parent PDU EventId. pub(super) prevevent_parent: Arc, - pub(super) pdu_cache: RwLock>>, + pub(super) pdu_cache: Mutex>>, + pub(super) auth_chain_cache: Mutex>>, } impl Rooms { @@ -109,7 +110,7 @@ impl Rooms { pub fn state_full( &self, shortstatehash: u64, - ) -> Result>> { + ) -> Result>> { let state = self .stateid_shorteventid .scan_prefix(shortstatehash.to_be_bytes().to_vec()) @@ -282,7 +283,7 @@ impl Rooms { pub fn force_state( &self, room_id: &RoomId, - state: BTreeMap<(EventType, String), EventId>, + state: HashMap<(EventType, String), EventId>, db: &Database, ) -> Result<()> { let state_hash = self.calculate_hash( @@ -402,11 +403,11 @@ impl Rooms { pub fn room_state_full( &self, room_id: &RoomId, - ) -> Result>> { + ) -> Result>> { if let Some(current_shortstatehash) = self.current_shortstatehash(room_id)? { self.state_full(current_shortstatehash) } else { - Ok(BTreeMap::new()) + Ok(HashMap::new()) } } @@ -542,7 +543,7 @@ impl Rooms { /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. pub fn get_pdu(&self, event_id: &EventId) -> Result>> { - if let Some(p) = self.pdu_cache.write().unwrap().get_mut(&event_id) { + if let Some(p) = self.pdu_cache.lock().unwrap().get_mut(&event_id) { return Ok(Some(Arc::clone(p))); } @@ -568,7 +569,7 @@ impl Rooms { .transpose()? { self.pdu_cache - .write() + .lock() .unwrap() .insert(event_id.clone(), Arc::clone(&pdu)); Ok(Some(pdu)) @@ -2520,4 +2521,10 @@ impl Rooms { Ok(self.userroomid_leftstate.get(&userroom_id)?.is_some()) } + + pub fn auth_chain_cache( + &self, + ) -> std::sync::MutexGuard<'_, LruCache>> { + self.auth_chain_cache.lock().unwrap() + } } diff --git a/src/server_server.rs b/src/server_server.rs index bfb3e727..39a18473 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -6,6 +6,7 @@ use crate::{ use get_profile_information::v1::ProfileField; use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::{debug, error, info, trace, warn}; +use lru_cache::LruCache; use regex::Regex; use rocket::response::content::Json; use ruma::{ @@ -52,7 +53,7 @@ use ruma::{ ServerSigningKeyId, UserId, }; use std::{ - collections::{btree_map::Entry, BTreeMap, BTreeSet, HashSet}, + collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, convert::{TryFrom, TryInto}, fmt::Debug, future::Future, @@ -931,7 +932,7 @@ pub fn handle_incoming_pdu<'a>( ); // Build map of auth events - let mut auth_events = BTreeMap::new(); + let mut auth_events = HashMap::new(); for id in &incoming_pdu.auth_events { let auth_event = db .rooms @@ -1097,7 +1098,7 @@ pub fn handle_incoming_pdu<'a>( Err(_) => return Err("Failed to fetch state events.".to_owned()), }; - let mut state = BTreeMap::new(); + let mut state = HashMap::new(); for pdu in state_vec { match state.entry((pdu.kind.clone(), pdu.state_key.clone().ok_or_else(|| "Found non-state pdu in state events.".to_owned())?)) { Entry::Vacant(v) => { @@ -1173,7 +1174,8 @@ pub fn handle_incoming_pdu<'a>( } } - let mut fork_states = BTreeSet::new(); + let mut extremity_statehashes = Vec::new(); + for id in &extremities { match db .rooms @@ -1181,30 +1183,19 @@ pub fn handle_incoming_pdu<'a>( .map_err(|_| "Failed to ask db for pdu.".to_owned())? { Some(leaf_pdu) => { - let pdu_shortstatehash = db - .rooms - .pdu_shortstatehash(&leaf_pdu.event_id) - .map_err(|_| "Failed to ask db for pdu state hash.".to_owned())? - .ok_or_else(|| { - error!( - "Found extremity pdu with no statehash in db: {:?}", - leaf_pdu - ); - "Found pdu with no statehash in db.".to_owned() - })?; - - let mut leaf_state = db - .rooms - .state_full(pdu_shortstatehash) - .map_err(|_| "Failed to ask db for room state.".to_owned())?; - - if let Some(state_key) = &leaf_pdu.state_key { - // Now it's the state after - let key = (leaf_pdu.kind.clone(), state_key.clone()); - leaf_state.insert(key, leaf_pdu); - } - - fork_states.insert(leaf_state); + extremity_statehashes.push(( + db.rooms + .pdu_shortstatehash(&leaf_pdu.event_id) + .map_err(|_| "Failed to ask db for pdu state hash.".to_owned())? + .ok_or_else(|| { + error!( + "Found extremity pdu with no statehash in db: {:?}", + leaf_pdu + ); + "Found pdu with no statehash in db.".to_owned() + })?, + Some(leaf_pdu), + )); } _ => { error!("Missing state snapshot for {:?}", id); @@ -1218,12 +1209,36 @@ pub fn handle_incoming_pdu<'a>( // don't just trust a set of state we got from a remote). // We do this by adding the current state to the list of fork states + let current_statehash = db + .rooms + .current_shortstatehash(&room_id) + .map_err(|_| "Failed to load current state hash.".to_owned())? + .expect("every room has state"); + let current_state = db .rooms - .room_state_full(&room_id) - .map_err(|_| "Failed to load room state.".to_owned())?; + .state_full(current_statehash) + .map_err(|_| "Failed to load room state.")?; - fork_states.insert(current_state.clone()); + extremity_statehashes.push((current_statehash.clone(), None)); + + let mut fork_states = Vec::new(); + for (statehash, leaf_pdu) in extremity_statehashes { + let mut leaf_state = db + .rooms + .state_full(statehash) + .map_err(|_| "Failed to ask db for room state.".to_owned())?; + + if let Some(leaf_pdu) = leaf_pdu { + if let Some(state_key) = &leaf_pdu.state_key { + // Now it's the state after + let key = (leaf_pdu.kind.clone(), state_key.clone()); + leaf_state.insert(key, leaf_pdu); + } + } + + fork_states.push(leaf_state); + } // We also add state after incoming event to the fork states extremities.insert(incoming_pdu.event_id.clone()); @@ -1234,9 +1249,7 @@ pub fn handle_incoming_pdu<'a>( incoming_pdu.clone(), ); } - fork_states.insert(state_after.clone()); - - let fork_states = fork_states.into_iter().collect::>(); + fork_states.push(state_after.clone()); let mut update_state = false; // 14. Use state resolution to find new room state @@ -1254,17 +1267,31 @@ pub fn handle_incoming_pdu<'a>( // We do need to force an update to this room's state update_state = true; - match state_res::StateResolution::resolve( + let fork_states = &fork_states + .into_iter() + .map(|map| { + map.into_iter() + .map(|(k, v)| (k, v.event_id.clone())) + .collect::>() + }) + .collect::>(); + + let auth_chain_t = Instant::now(); + let mut auth_chain_sets = Vec::new(); + for state in fork_states { + auth_chain_sets.push( + get_auth_chain(state.iter().map(|(_, id)| id.clone()).collect(), db) + .map_err(|_| "Failed to load auth chain.".to_owned())?, + ); + } + dbg!(auth_chain_t.elapsed()); + + let state_res_t = Instant::now(); + let state = match state_res::StateResolution::resolve( &room_id, room_version_id, - &fork_states - .into_iter() - .map(|map| { - map.into_iter() - .map(|(k, v)| (k, v.event_id.clone())) - .collect::>() - }) - .collect::>(), + fork_states, + auth_chain_sets, |id| { let res = db.rooms.get_pdu(id); if let Err(e) = &res { @@ -1277,7 +1304,9 @@ pub fn handle_incoming_pdu<'a>( Err(_) => { return Err("State resolution failed, either an event could not be found or deserialization".into()); } - } + }; + dbg!(state_res_t.elapsed()); + state }; // 13. Check if the event passes auth based on the "current state" of the room, if not "soft fail" it @@ -1696,6 +1725,42 @@ async fn append_incoming_pdu( Ok(pdu_id) } +fn get_auth_chain(starting_events: Vec, db: &Database) -> Result> { + let mut auth_chain_cache = db.rooms.auth_chain_cache(); + + let mut auth_chain = HashSet::new(); + + for event in starting_events { + auth_chain.extend(get_auth_chain_recursive(&event, &mut auth_chain_cache, db)?); + } + + Ok(auth_chain) +} + +fn get_auth_chain_recursive( + event_id: &EventId, + auth_chain_cache: &mut std::sync::MutexGuard<'_, LruCache>>, + db: &Database, +) -> Result> { + if let Some(cached) = auth_chain_cache.get_mut(event_id) { + return Ok(cached.clone()); + } + + let mut auth_chain = HashSet::new(); + + if let Some(pdu) = db.rooms.get_pdu(&event_id)? { + for auth_event in &pdu.auth_events { + auth_chain.extend(get_auth_chain_recursive(&auth_event, auth_chain_cache, db)?); + } + } else { + warn!("Could not find pdu mentioned in auth events."); + } + + auth_chain_cache.insert(event_id.clone(), auth_chain.clone()); + + Ok(auth_chain) +} + #[cfg_attr( feature = "conduit_bin", get("/_matrix/federation/v1/event/<_>", data = "") @@ -1783,35 +1848,20 @@ pub fn get_event_authorization_route( return Err(Error::bad_config("Federation is disabled.")); } - let mut auth_chain = Vec::new(); - let mut auth_chain_ids = BTreeSet::::new(); - let mut todo = BTreeSet::new(); - todo.insert(body.event_id.clone()); - - while let Some(event_id) = todo.iter().next().cloned() { - if let Some(pdu) = db.rooms.get_pdu(&event_id)? { - todo.extend( - pdu.auth_events - .clone() - .into_iter() - .collect::>() - .difference(&auth_chain_ids) - .cloned(), - ); - auth_chain_ids.extend(pdu.auth_events.clone().into_iter()); + let auth_chain_ids = get_auth_chain(vec![body.event_id.clone()], &db)?; - let pdu_json = PduEvent::convert_to_outgoing_federation_event( - db.rooms.get_pdu_json(&event_id)?.unwrap(), - ); - auth_chain.push(pdu_json); - } else { - warn!("Could not find pdu mentioned in auth events."); - } - - todo.remove(&event_id); + Ok(get_event_authorization::v1::Response { + auth_chain: auth_chain_ids + .into_iter() + .map(|id| { + Ok::<_, Error>(PduEvent::convert_to_outgoing_federation_event( + db.rooms.get_pdu_json(&id)?.unwrap(), + )) + }) + .filter_map(|r| r.ok()) + .collect(), } - - Ok(get_event_authorization::v1::Response { auth_chain }.into()) + .into()) } #[cfg_attr( @@ -1846,35 +1896,21 @@ pub fn get_room_state_route( }) .collect(); - let mut auth_chain = Vec::new(); - let mut auth_chain_ids = BTreeSet::::new(); - let mut todo = BTreeSet::new(); - todo.insert(body.event_id.clone()); - - while let Some(event_id) = todo.iter().next().cloned() { - if let Some(pdu) = db.rooms.get_pdu(&event_id)? { - todo.extend( - pdu.auth_events - .clone() - .into_iter() - .collect::>() - .difference(&auth_chain_ids) - .cloned(), - ); - auth_chain_ids.extend(pdu.auth_events.clone().into_iter()); - - let pdu_json = PduEvent::convert_to_outgoing_federation_event( - db.rooms.get_pdu_json(&event_id)?.unwrap(), - ); - auth_chain.push(pdu_json); - } else { - warn!("Could not find pdu mentioned in auth events."); - } + let auth_chain_ids = get_auth_chain(vec![body.event_id.clone()], &db)?; - todo.remove(&event_id); + Ok(get_room_state::v1::Response { + auth_chain: auth_chain_ids + .into_iter() + .map(|id| { + Ok::<_, Error>(PduEvent::convert_to_outgoing_federation_event( + db.rooms.get_pdu_json(&id)?.unwrap(), + )) + }) + .filter_map(|r| r.ok()) + .collect(), + pdus, } - - Ok(get_room_state::v1::Response { auth_chain, pdus }.into()) + .into()) } #[cfg_attr( @@ -1904,27 +1940,7 @@ pub fn get_room_state_ids_route( .into_iter() .collect(); - let mut auth_chain_ids = BTreeSet::::new(); - let mut todo = BTreeSet::new(); - todo.insert(body.event_id.clone()); - - while let Some(event_id) = todo.iter().next().cloned() { - if let Some(pdu) = db.rooms.get_pdu(&event_id)? { - todo.extend( - pdu.auth_events - .clone() - .into_iter() - .collect::>() - .difference(&auth_chain_ids) - .cloned(), - ); - auth_chain_ids.extend(pdu.auth_events.clone().into_iter()); - } else { - warn!("Could not find pdu mentioned in auth events."); - } - - todo.remove(&event_id); - } + let auth_chain_ids = get_auth_chain(vec![body.event_id.clone()], &db)?; Ok(get_room_state_ids::v1::Response { auth_chain_ids: auth_chain_ids.into_iter().collect(), @@ -2182,8 +2198,8 @@ pub async fn create_join_event_route( let state_ids = db.rooms.state_full_ids(shortstatehash)?; - let mut auth_chain_ids = BTreeSet::::new(); - let mut todo = state_ids.iter().cloned().collect::>(); + let mut auth_chain_ids = HashSet::::new(); + let mut todo = state_ids.iter().cloned().collect::>(); while let Some(event_id) = todo.iter().next().cloned() { if let Some(pdu) = db.rooms.get_pdu(&event_id)? { @@ -2191,7 +2207,7 @@ pub async fn create_join_event_route( pdu.auth_events .clone() .into_iter() - .collect::>() + .collect::>() .difference(&auth_chain_ids) .cloned(), ); From 0f8ecdc625cd5fa66e726d2b47a9fe7f419b5541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 20 Jul 2021 12:41:35 +0200 Subject: [PATCH 4/7] fix: state res fixes --- Cargo.lock | 36 +++++++++++++++++----------------- Cargo.toml | 2 +- src/database/rooms.rs | 6 +++--- src/server_server.rs | 45 ++++++++++++++++++++++--------------------- 4 files changed, 45 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8bfba0e8..a6b87c9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2015,7 +2015,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "assign", "js_int", @@ -2036,7 +2036,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.1" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "bytes", "http", @@ -2052,7 +2052,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.1" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.3.0" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "ruma-api", "ruma-common", @@ -2077,7 +2077,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.11.0" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "assign", "bytes", @@ -2097,7 +2097,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.5.4" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "indexmap", "js_int", @@ -2112,7 +2112,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.23.2" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "indoc", "js_int", @@ -2128,7 +2128,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.23.2" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2139,7 +2139,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "js_int", "ruma-api", @@ -2154,7 +2154,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.19.4" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "paste", "rand 0.8.4", @@ -2168,7 +2168,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.19.4" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2178,12 +2178,12 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.4.0" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" [[package]] name = "ruma-identity-service-api" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "js_int", "ruma-api", @@ -2196,7 +2196,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "js_int", "ruma-api", @@ -2211,7 +2211,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.4.1" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "bytes", "form_urlencoded", @@ -2225,7 +2225,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.4.1" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2236,7 +2236,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.8.0" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2253,7 +2253,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?rev=a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386#a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386" +source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 64d67a14..c62ff821 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { git = "https://github.com/ruma/ruma", rev = "c29c2b16ec114fa655e2b70bdd53c82e35859005", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/timokoesters/ruma", rev = "a3fd405d6b331c7bc4c6f366bc1b6ec303b3a386", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/timokoesters/ruma", rev = "74cf83c4ca937fa5e2709fb71e9d11848e72e487", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } # Used for long polling and federation sender, should be the same as rocket::tokio diff --git a/src/database/rooms.rs b/src/database/rooms.rs index fa121bda..aad691b4 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -653,9 +653,9 @@ impl Rooms { Ok(()) } - pub fn is_pdu_referenced(&self, pdu: &PduEvent) -> Result { - let mut key = pdu.room_id().as_bytes().to_vec(); - key.extend_from_slice(pdu.event_id().as_bytes()); + pub fn is_event_referenced(&self, room_id: &RoomId, event_id: &EventId) -> Result { + let mut key = room_id.as_bytes().to_vec(); + key.extend_from_slice(event_id.as_bytes()); Ok(self.prevevent_parent.get(&key)?.is_some()) } diff --git a/src/server_server.rs b/src/server_server.rs index 39a18473..e463bba2 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -6,7 +6,6 @@ use crate::{ use get_profile_information::v1::ProfileField; use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::{debug, error, info, trace, warn}; -use lru_cache::LruCache; use regex::Regex; use rocket::response::content::Json; use ruma::{ @@ -1174,6 +1173,9 @@ pub fn handle_incoming_pdu<'a>( } } + // Only keep those extremities we don't have in our timeline yet + extremities.retain(|id| !matches!(db.rooms.get_non_outlier_pdu_json(id), Ok(Some(_)))); + let mut extremity_statehashes = Vec::new(); for id in &extremities { @@ -1276,7 +1278,6 @@ pub fn handle_incoming_pdu<'a>( }) .collect::>(); - let auth_chain_t = Instant::now(); let mut auth_chain_sets = Vec::new(); for state in fork_states { auth_chain_sets.push( @@ -1284,9 +1285,7 @@ pub fn handle_incoming_pdu<'a>( .map_err(|_| "Failed to load auth chain.".to_owned())?, ); } - dbg!(auth_chain_t.elapsed()); - let state_res_t = Instant::now(); let state = match state_res::StateResolution::resolve( &room_id, room_version_id, @@ -1305,7 +1304,6 @@ pub fn handle_incoming_pdu<'a>( return Err("State resolution failed, either an event could not be found or deserialization".into()); } }; - dbg!(state_res_t.elapsed()); state }; @@ -1726,38 +1724,41 @@ async fn append_incoming_pdu( } fn get_auth_chain(starting_events: Vec, db: &Database) -> Result> { - let mut auth_chain_cache = db.rooms.auth_chain_cache(); + let mut full_auth_chain = HashSet::new(); - let mut auth_chain = HashSet::new(); + let mut cache = db.rooms.auth_chain_cache(); + for event_id in starting_events { + let auth_chain = if let Some(cached) = cache.get_mut(&event_id) { + cached.clone() + } else { + drop(cache); + let auth_chain = get_auth_chain_recursive(&event_id, db)?; - for event in starting_events { - auth_chain.extend(get_auth_chain_recursive(&event, &mut auth_chain_cache, db)?); - } + cache = db.rooms.auth_chain_cache(); - Ok(auth_chain) -} + cache.insert(event_id, auth_chain.clone()); -fn get_auth_chain_recursive( - event_id: &EventId, - auth_chain_cache: &mut std::sync::MutexGuard<'_, LruCache>>, - db: &Database, -) -> Result> { - if let Some(cached) = auth_chain_cache.get_mut(event_id) { - return Ok(cached.clone()); + auth_chain + }; + + full_auth_chain.extend(auth_chain); } + Ok(full_auth_chain) +} + +fn get_auth_chain_recursive(event_id: &EventId, db: &Database) -> Result> { let mut auth_chain = HashSet::new(); if let Some(pdu) = db.rooms.get_pdu(&event_id)? { + auth_chain.extend(pdu.auth_events.iter().cloned()); for auth_event in &pdu.auth_events { - auth_chain.extend(get_auth_chain_recursive(&auth_event, auth_chain_cache, db)?); + auth_chain.extend(get_auth_chain_recursive(&auth_event, db)?); } } else { warn!("Could not find pdu mentioned in auth events."); } - auth_chain_cache.insert(event_id.clone(), auth_chain.clone()); - Ok(auth_chain) } From f0ac7ed933d4b256192a7d904a37333653883cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 20 Jul 2021 15:24:18 +0200 Subject: [PATCH 5/7] chore: update ruma again --- Cargo.lock | 36 +++++++++++++++++----------------- Cargo.toml | 4 ++-- src/client_server/account.rs | 13 ++++++------ src/client_server/directory.rs | 2 +- src/client_server/room.rs | 14 +++---------- src/database/pusher.rs | 5 ++++- 6 files changed, 35 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6b87c9d..808ba4ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2015,7 +2015,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "assign", "js_int", @@ -2036,7 +2036,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.1" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "bytes", "http", @@ -2052,7 +2052,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.1" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.3.0" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "ruma-api", "ruma-common", @@ -2077,7 +2077,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.11.0" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "assign", "bytes", @@ -2097,7 +2097,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.5.4" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "indexmap", "js_int", @@ -2112,7 +2112,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.23.2" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "indoc", "js_int", @@ -2128,7 +2128,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.23.2" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2139,7 +2139,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "js_int", "ruma-api", @@ -2154,7 +2154,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.19.4" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "paste", "rand 0.8.4", @@ -2168,7 +2168,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.19.4" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2178,12 +2178,12 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.4.0" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" [[package]] name = "ruma-identity-service-api" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "js_int", "ruma-api", @@ -2196,7 +2196,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "js_int", "ruma-api", @@ -2211,7 +2211,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.4.1" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "bytes", "form_urlencoded", @@ -2225,7 +2225,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.4.1" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2236,7 +2236,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.8.0" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2253,7 +2253,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?rev=74cf83c4ca937fa5e2709fb71e9d11848e72e487#74cf83c4ca937fa5e2709fb71e9d11848e72e487" +source = "git+https://github.com/ruma/ruma?rev=eb19b0e08a901b87d11b3be0890ec788cc760492#eb19b0e08a901b87d11b3be0890ec788cc760492" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index c62ff821..2ce4b037 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ edition = "2018" rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests # Used for matrix spec type definitions and helpers -#ruma = { git = "https://github.com/ruma/ruma", rev = "c29c2b16ec114fa655e2b70bdd53c82e35859005", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/timokoesters/ruma", rev = "74cf83c4ca937fa5e2709fb71e9d11848e72e487", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "eb19b0e08a901b87d11b3be0890ec788cc760492", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +#ruma = { git = "https://github.com/timokoesters/ruma", rev = "74cf83c4ca937fa5e2709fb71e9d11848e72e487", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } # Used for long polling and federation sender, should be the same as rocket::tokio diff --git a/src/client_server/account.rs b/src/client_server/account.rs index ebaf7b17..ddb44d6e 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -20,12 +20,12 @@ use ruma::{ }, events::{ room::{ - canonical_alias, guest_access, history_visibility, join_rules, member, message, - name::{self, RoomName}, + canonical_alias, guest_access, history_visibility, join_rules, member, message, name, topic, }, EventType, }, + identifiers::RoomName, push, RoomAliasId, RoomId, RoomVersionId, UserId, }; @@ -378,13 +378,14 @@ pub async fn register_route( )?; // 6. Events implied by name and topic + let room_name = + Box::::try_from(format!("{} Admin Room", db.globals.server_name())) + .expect("Room name is valid"); db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomName, - content: serde_json::to_value(name::NameEventContent::new(Some( - RoomName::try_from("Admin Room".to_owned()).expect("Room name is valid"), - ))) - .expect("event is valid, we just created it"), + content: serde_json::to_value(name::NameEventContent::new(Some(room_name))) + .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 64375f00..d7388865 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -316,7 +316,7 @@ pub async fn get_public_rooms_filtered_helper( .map(|q| q.to_lowercase()) { if let Some(name) = &chunk.name { - if name.to_lowercase().contains(&query) { + if name.as_str().to_lowercase().contains(&query) { return true; } } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index c9b93c35..6e271308 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -9,11 +9,7 @@ use ruma::{ r0::room::{self, create_room, get_room_event, upgrade_room}, }, events::{ - room::{ - guest_access, history_visibility, join_rules, member, - name::{self, RoomName}, - topic, - }, + room::{guest_access, history_visibility, join_rules, member, name, topic}, EventType, }, serde::Raw, @@ -256,12 +252,8 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomName, - content: serde_json::to_value(name::NameEventContent::new(Some( - RoomName::try_from(name.clone()).map_err(|_| { - Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") - })?, - ))) - .expect("event is valid, we just created it"), + content: serde_json::to_value(name::NameEventContent::new(Some(name.clone()))) + .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, diff --git a/src/database/pusher.rs b/src/database/pusher.rs index 3210cb18..348f4dcb 100644 --- a/src/database/pusher.rs +++ b/src/database/pusher.rs @@ -11,6 +11,7 @@ use ruma::{ IncomingResponse, OutgoingRequest, SendAccessToken, }, events::{room::power_levels::PowerLevelsEventContent, EventType}, + identifiers::RoomName, push::{Action, PushConditionRoomCtx, PushFormat, Ruleset, Tweak}, uint, UInt, UserId, }; @@ -299,7 +300,9 @@ async fn send_notice( .rooms .room_state_get(&event.room_id, &EventType::RoomName, "")? .map(|pdu| match pdu.content.get("name") { - Some(serde_json::Value::String(s)) => Some(s.to_string()), + Some(serde_json::Value::String(s)) => { + Some(Box::::try_from(&**s).expect("room name is valid")) + } _ => None, }) .flatten(); From 56e5d5e3d2e191add0aafb5c89be62f59f33710c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 20 Jul 2021 18:38:17 +0200 Subject: [PATCH 6/7] fix: add warning for strange bug --- src/client_server/sync.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 65922bec..7ab27d91 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -1,5 +1,5 @@ use crate::{database::DatabaseGuard, ConduitResult, Database, Error, Result, Ruma, RumaResponse}; -use log::error; +use log::{error, warn}; use ruma::{ api::client::r0::{sync::sync_events, uiaa::UiaaResponse}, events::{room::member::MembershipState, AnySyncEphemeralRoomEvent, EventType}, @@ -262,9 +262,12 @@ async fn sync_helper( db.rooms .pdu_shortstatehash(&pdu.1.event_id) .transpose() - .expect("all pdus have state") + .ok_or_else(|| { + warn!("PDU without state: {}", pdu.1.event_id); + Error::bad_database("Found PDU without state") + }) }) - .transpose()?; + .transpose()?.transpose()?; // Calculates joined_member_count, invited_member_count and heroes let calculate_counts = || { From 68cc743f2a42a4f4155298a6231c48091aa401d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 20 Jul 2021 18:45:21 +0200 Subject: [PATCH 7/7] fix: use get_auth_chain method more often --- src/client_server/sync.rs | 3 ++- src/server_server.rs | 22 +--------------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 7ab27d91..427b9add 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -267,7 +267,8 @@ async fn sync_helper( Error::bad_database("Found PDU without state") }) }) - .transpose()?.transpose()?; + .transpose()? + .transpose()?; // Calculates joined_member_count, invited_member_count and heroes let calculate_counts = || { diff --git a/src/server_server.rs b/src/server_server.rs index e463bba2..d51b672e 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -2198,27 +2198,7 @@ pub async fn create_join_event_route( drop(mutex_lock); let state_ids = db.rooms.state_full_ids(shortstatehash)?; - - let mut auth_chain_ids = HashSet::::new(); - let mut todo = state_ids.iter().cloned().collect::>(); - - while let Some(event_id) = todo.iter().next().cloned() { - if let Some(pdu) = db.rooms.get_pdu(&event_id)? { - todo.extend( - pdu.auth_events - .clone() - .into_iter() - .collect::>() - .difference(&auth_chain_ids) - .cloned(), - ); - auth_chain_ids.extend(pdu.auth_events.clone().into_iter()); - } else { - warn!("Could not find pdu mentioned in auth events."); - } - - todo.remove(&event_id); - } + let auth_chain_ids = get_auth_chain(state_ids.iter().cloned().collect(), &db)?; for server in db .rooms