|
|
|
@ -1,25 +1,24 @@
|
|
|
|
/// An async function that can recursively call itself.
|
|
|
|
/// An async function that can recursively call itself.
|
|
|
|
type AsyncRecursiveType<'a, T> = Pin<Box<dyn Future<Output = T> + 'a + Send>>;
|
|
|
|
type AsyncRecursiveType<'a, T> = Pin<Box<dyn Future<Output = T> + 'a + Send>>;
|
|
|
|
|
|
|
|
|
|
|
|
use ruma::{
|
|
|
|
|
|
|
|
api::federation::discovery::{get_remote_server_keys, get_server_keys},
|
|
|
|
|
|
|
|
CanonicalJsonObject, CanonicalJsonValue, OwnedServerName, OwnedServerSigningKeyId,
|
|
|
|
|
|
|
|
RoomVersionId,
|
|
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
use std::{
|
|
|
|
use std::{
|
|
|
|
collections::{hash_map, BTreeMap, HashMap, HashSet},
|
|
|
|
collections::{hash_map, BTreeMap, HashMap, HashSet},
|
|
|
|
pin::Pin,
|
|
|
|
pin::Pin,
|
|
|
|
sync::{Arc, RwLock, RwLockWriteGuard},
|
|
|
|
sync::Arc,
|
|
|
|
time::{Duration, Instant, SystemTime},
|
|
|
|
time::{Duration, Instant, SystemTime},
|
|
|
|
};
|
|
|
|
};
|
|
|
|
use tokio::sync::Semaphore;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
use async_recursion::async_recursion;
|
|
|
|
use futures_util::{stream::FuturesUnordered, Future, StreamExt};
|
|
|
|
use futures_util::{stream::FuturesUnordered, Future, StreamExt};
|
|
|
|
use ruma::{
|
|
|
|
use ruma::{
|
|
|
|
api::{
|
|
|
|
api::{
|
|
|
|
client::error::ErrorKind,
|
|
|
|
client::error::ErrorKind,
|
|
|
|
federation::{
|
|
|
|
federation::{
|
|
|
|
discovery::get_remote_server_keys_batch::{self, v2::QueryCriteria},
|
|
|
|
discovery::{
|
|
|
|
|
|
|
|
get_remote_server_keys,
|
|
|
|
|
|
|
|
get_remote_server_keys_batch::{self, v2::QueryCriteria},
|
|
|
|
|
|
|
|
get_server_keys,
|
|
|
|
|
|
|
|
},
|
|
|
|
event::{get_event, get_room_state_ids},
|
|
|
|
event::{get_event, get_room_state_ids},
|
|
|
|
membership::create_join_event,
|
|
|
|
membership::create_join_event,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
@ -31,9 +30,11 @@ use ruma::{
|
|
|
|
int,
|
|
|
|
int,
|
|
|
|
serde::Base64,
|
|
|
|
serde::Base64,
|
|
|
|
state_res::{self, RoomVersion, StateMap},
|
|
|
|
state_res::{self, RoomVersion, StateMap},
|
|
|
|
uint, EventId, MilliSecondsSinceUnixEpoch, RoomId, ServerName,
|
|
|
|
uint, CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch,
|
|
|
|
|
|
|
|
OwnedServerName, OwnedServerSigningKeyId, RoomId, RoomVersionId, ServerName,
|
|
|
|
};
|
|
|
|
};
|
|
|
|
use serde_json::value::RawValue as RawJsonValue;
|
|
|
|
use serde_json::value::RawValue as RawJsonValue;
|
|
|
|
|
|
|
|
use tokio::sync::{RwLock, RwLockWriteGuard, Semaphore};
|
|
|
|
use tracing::{debug, error, info, trace, warn};
|
|
|
|
use tracing::{debug, error, info, trace, warn};
|
|
|
|
|
|
|
|
|
|
|
|
use crate::{service::*, services, Error, PduEvent, Result};
|
|
|
|
use crate::{service::*, services, Error, PduEvent, Result};
|
|
|
|
@ -168,7 +169,7 @@ impl Service {
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.read()
|
|
|
|
.read()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.get(&*prev_id)
|
|
|
|
.get(&*prev_id)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
// Exponential backoff
|
|
|
|
// Exponential backoff
|
|
|
|
@ -189,7 +190,7 @@ impl Service {
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.write()
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.entry((*prev_id).to_owned())
|
|
|
|
.entry((*prev_id).to_owned())
|
|
|
|
{
|
|
|
|
{
|
|
|
|
hash_map::Entry::Vacant(e) => {
|
|
|
|
hash_map::Entry::Vacant(e) => {
|
|
|
|
@ -213,7 +214,7 @@ impl Service {
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.roomid_federationhandletime
|
|
|
|
.roomid_federationhandletime
|
|
|
|
.write()
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.insert(room_id.to_owned(), ((*prev_id).to_owned(), start_time));
|
|
|
|
.insert(room_id.to_owned(), ((*prev_id).to_owned(), start_time));
|
|
|
|
|
|
|
|
|
|
|
|
if let Err(e) = self
|
|
|
|
if let Err(e) = self
|
|
|
|
@ -233,7 +234,7 @@ impl Service {
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.write()
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.entry((*prev_id).to_owned())
|
|
|
|
.entry((*prev_id).to_owned())
|
|
|
|
{
|
|
|
|
{
|
|
|
|
hash_map::Entry::Vacant(e) => {
|
|
|
|
hash_map::Entry::Vacant(e) => {
|
|
|
|
@ -249,7 +250,7 @@ impl Service {
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.roomid_federationhandletime
|
|
|
|
.roomid_federationhandletime
|
|
|
|
.write()
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.remove(&room_id.to_owned());
|
|
|
|
.remove(&room_id.to_owned());
|
|
|
|
debug!(
|
|
|
|
debug!(
|
|
|
|
"Handling prev event {} took {}m{}s",
|
|
|
|
"Handling prev event {} took {}m{}s",
|
|
|
|
@ -267,7 +268,7 @@ impl Service {
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.roomid_federationhandletime
|
|
|
|
.roomid_federationhandletime
|
|
|
|
.write()
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.insert(room_id.to_owned(), (event_id.to_owned(), start_time));
|
|
|
|
.insert(room_id.to_owned(), (event_id.to_owned(), start_time));
|
|
|
|
let r = services()
|
|
|
|
let r = services()
|
|
|
|
.rooms
|
|
|
|
.rooms
|
|
|
|
@ -285,7 +286,7 @@ impl Service {
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.roomid_federationhandletime
|
|
|
|
.roomid_federationhandletime
|
|
|
|
.write()
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.remove(&room_id.to_owned());
|
|
|
|
.remove(&room_id.to_owned());
|
|
|
|
|
|
|
|
|
|
|
|
r
|
|
|
|
r
|
|
|
|
@ -326,11 +327,8 @@ impl Service {
|
|
|
|
let room_version =
|
|
|
|
let room_version =
|
|
|
|
RoomVersion::new(room_version_id).expect("room version is supported");
|
|
|
|
RoomVersion::new(room_version_id).expect("room version is supported");
|
|
|
|
|
|
|
|
|
|
|
|
let mut val = match ruma::signatures::verify_event(
|
|
|
|
let guard = pub_key_map.read().await;
|
|
|
|
&pub_key_map.read().expect("RwLock is poisoned."),
|
|
|
|
let mut val = match ruma::signatures::verify_event(&guard, &value, room_version_id) {
|
|
|
|
&value,
|
|
|
|
|
|
|
|
room_version_id,
|
|
|
|
|
|
|
|
) {
|
|
|
|
|
|
|
|
Err(e) => {
|
|
|
|
Err(e) => {
|
|
|
|
// Drop
|
|
|
|
// Drop
|
|
|
|
warn!("Dropping bad event {}: {}", event_id, e,);
|
|
|
|
warn!("Dropping bad event {}: {}", event_id, e,);
|
|
|
|
@ -365,6 +363,8 @@ impl Service {
|
|
|
|
Ok(ruma::signatures::Verified::All) => value,
|
|
|
|
Ok(ruma::signatures::Verified::All) => value,
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
drop(guard);
|
|
|
|
|
|
|
|
|
|
|
|
// Now that we have checked the signature and hashes we can add the eventID and convert
|
|
|
|
// Now that we have checked the signature and hashes we can add the eventID and convert
|
|
|
|
// to our PduEvent type
|
|
|
|
// to our PduEvent type
|
|
|
|
val.insert(
|
|
|
|
val.insert(
|
|
|
|
@ -692,13 +692,15 @@ impl Service {
|
|
|
|
{
|
|
|
|
{
|
|
|
|
Ok(res) => {
|
|
|
|
Ok(res) => {
|
|
|
|
debug!("Fetching state events at event.");
|
|
|
|
debug!("Fetching state events at event.");
|
|
|
|
|
|
|
|
let collect = res
|
|
|
|
|
|
|
|
.pdu_ids
|
|
|
|
|
|
|
|
.iter()
|
|
|
|
|
|
|
|
.map(|x| Arc::from(&**x))
|
|
|
|
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
let state_vec = self
|
|
|
|
let state_vec = self
|
|
|
|
.fetch_and_handle_outliers(
|
|
|
|
.fetch_and_handle_outliers(
|
|
|
|
origin,
|
|
|
|
origin,
|
|
|
|
&res.pdu_ids
|
|
|
|
&collect,
|
|
|
|
.iter()
|
|
|
|
|
|
|
|
.map(|x| Arc::from(&**x))
|
|
|
|
|
|
|
|
.collect::<Vec<_>>(),
|
|
|
|
|
|
|
|
create_event,
|
|
|
|
create_event,
|
|
|
|
room_id,
|
|
|
|
room_id,
|
|
|
|
room_version_id,
|
|
|
|
room_version_id,
|
|
|
|
@ -805,7 +807,7 @@ impl Service {
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.roomid_mutex_state
|
|
|
|
.roomid_mutex_state
|
|
|
|
.write()
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.entry(room_id.to_owned())
|
|
|
|
.entry(room_id.to_owned())
|
|
|
|
.or_default(),
|
|
|
|
.or_default(),
|
|
|
|
);
|
|
|
|
);
|
|
|
|
@ -884,14 +886,18 @@ impl Service {
|
|
|
|
debug!("Starting soft fail auth check");
|
|
|
|
debug!("Starting soft fail auth check");
|
|
|
|
|
|
|
|
|
|
|
|
if soft_fail {
|
|
|
|
if soft_fail {
|
|
|
|
services().rooms.timeline.append_incoming_pdu(
|
|
|
|
services()
|
|
|
|
&incoming_pdu,
|
|
|
|
.rooms
|
|
|
|
val,
|
|
|
|
.timeline
|
|
|
|
extremities.iter().map(|e| (**e).to_owned()).collect(),
|
|
|
|
.append_incoming_pdu(
|
|
|
|
state_ids_compressed,
|
|
|
|
&incoming_pdu,
|
|
|
|
soft_fail,
|
|
|
|
val,
|
|
|
|
&state_lock,
|
|
|
|
extremities.iter().map(|e| (**e).to_owned()).collect(),
|
|
|
|
)?;
|
|
|
|
state_ids_compressed,
|
|
|
|
|
|
|
|
soft_fail,
|
|
|
|
|
|
|
|
&state_lock,
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
.await?;
|
|
|
|
|
|
|
|
|
|
|
|
// Soft fail, we keep the event as an outlier but don't add it to the timeline
|
|
|
|
// Soft fail, we keep the event as an outlier but don't add it to the timeline
|
|
|
|
warn!("Event was soft failed: {:?}", incoming_pdu);
|
|
|
|
warn!("Event was soft failed: {:?}", incoming_pdu);
|
|
|
|
@ -912,14 +918,18 @@ impl Service {
|
|
|
|
// We use the `state_at_event` instead of `state_after` so we accurately
|
|
|
|
// We use the `state_at_event` instead of `state_after` so we accurately
|
|
|
|
// represent the state for this event.
|
|
|
|
// represent the state for this event.
|
|
|
|
|
|
|
|
|
|
|
|
let pdu_id = services().rooms.timeline.append_incoming_pdu(
|
|
|
|
let pdu_id = services()
|
|
|
|
&incoming_pdu,
|
|
|
|
.rooms
|
|
|
|
val,
|
|
|
|
.timeline
|
|
|
|
extremities.iter().map(|e| (**e).to_owned()).collect(),
|
|
|
|
.append_incoming_pdu(
|
|
|
|
state_ids_compressed,
|
|
|
|
&incoming_pdu,
|
|
|
|
soft_fail,
|
|
|
|
val,
|
|
|
|
&state_lock,
|
|
|
|
extremities.iter().map(|e| (**e).to_owned()).collect(),
|
|
|
|
)?;
|
|
|
|
state_ids_compressed,
|
|
|
|
|
|
|
|
soft_fail,
|
|
|
|
|
|
|
|
&state_lock,
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
.await?;
|
|
|
|
|
|
|
|
|
|
|
|
debug!("Appended incoming pdu");
|
|
|
|
debug!("Appended incoming pdu");
|
|
|
|
|
|
|
|
|
|
|
|
@ -1034,7 +1044,8 @@ impl Service {
|
|
|
|
/// d. TODO: Ask other servers over federation?
|
|
|
|
/// d. TODO: Ask other servers over federation?
|
|
|
|
#[allow(clippy::type_complexity)]
|
|
|
|
#[allow(clippy::type_complexity)]
|
|
|
|
#[tracing::instrument(skip_all)]
|
|
|
|
#[tracing::instrument(skip_all)]
|
|
|
|
pub(crate) fn fetch_and_handle_outliers<'a>(
|
|
|
|
#[async_recursion]
|
|
|
|
|
|
|
|
pub(crate) async fn fetch_and_handle_outliers<'a>(
|
|
|
|
&'a self,
|
|
|
|
&'a self,
|
|
|
|
origin: &'a ServerName,
|
|
|
|
origin: &'a ServerName,
|
|
|
|
events: &'a [Arc<EventId>],
|
|
|
|
events: &'a [Arc<EventId>],
|
|
|
|
@ -1042,176 +1053,175 @@ impl Service {
|
|
|
|
room_id: &'a RoomId,
|
|
|
|
room_id: &'a RoomId,
|
|
|
|
room_version_id: &'a RoomVersionId,
|
|
|
|
room_version_id: &'a RoomVersionId,
|
|
|
|
pub_key_map: &'a RwLock<BTreeMap<String, BTreeMap<String, Base64>>>,
|
|
|
|
pub_key_map: &'a RwLock<BTreeMap<String, BTreeMap<String, Base64>>>,
|
|
|
|
) -> AsyncRecursiveType<'a, Vec<(Arc<PduEvent>, Option<BTreeMap<String, CanonicalJsonValue>>)>>
|
|
|
|
) -> Vec<(Arc<PduEvent>, Option<BTreeMap<String, CanonicalJsonValue>>)> {
|
|
|
|
{
|
|
|
|
let back_off = |id| async move {
|
|
|
|
Box::pin(async move {
|
|
|
|
match services()
|
|
|
|
let back_off = |id| match services()
|
|
|
|
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.write()
|
|
|
|
.write()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.entry(id)
|
|
|
|
.entry(id)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
hash_map::Entry::Vacant(e) => {
|
|
|
|
hash_map::Entry::Vacant(e) => {
|
|
|
|
e.insert((Instant::now(), 1));
|
|
|
|
e.insert((Instant::now(), 1));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
hash_map::Entry::Occupied(mut e) => *e.get_mut() = (Instant::now(), e.get().1 + 1),
|
|
|
|
hash_map::Entry::Occupied(mut e) => *e.get_mut() = (Instant::now(), e.get().1 + 1),
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
};
|
|
|
|
let mut pdus = vec![];
|
|
|
|
|
|
|
|
for id in events {
|
|
|
|
|
|
|
|
// a. Look in the main timeline (pduid_pdu tree)
|
|
|
|
|
|
|
|
// b. Look at outlier pdu tree
|
|
|
|
|
|
|
|
// (get_pdu_json checks both)
|
|
|
|
|
|
|
|
if let Ok(Some(local_pdu)) = services().rooms.timeline.get_pdu(id) {
|
|
|
|
|
|
|
|
trace!("Found {} in db", id);
|
|
|
|
|
|
|
|
pdus.push((local_pdu, None));
|
|
|
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// c. Ask origin server over federation
|
|
|
|
let mut pdus = vec![];
|
|
|
|
// We also handle its auth chain here so we don't get a stack overflow in
|
|
|
|
for id in events {
|
|
|
|
// handle_outlier_pdu.
|
|
|
|
// a. Look in the main timeline (pduid_pdu tree)
|
|
|
|
let mut todo_auth_events = vec![Arc::clone(id)];
|
|
|
|
// b. Look at outlier pdu tree
|
|
|
|
let mut events_in_reverse_order = Vec::new();
|
|
|
|
// (get_pdu_json checks both)
|
|
|
|
let mut events_all = HashSet::new();
|
|
|
|
if let Ok(Some(local_pdu)) = services().rooms.timeline.get_pdu(id) {
|
|
|
|
let mut i = 0;
|
|
|
|
trace!("Found {} in db", id);
|
|
|
|
while let Some(next_id) = todo_auth_events.pop() {
|
|
|
|
pdus.push((local_pdu, None));
|
|
|
|
if let Some((time, tries)) = services()
|
|
|
|
continue;
|
|
|
|
.globals
|
|
|
|
}
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
|
|
|
|
.read()
|
|
|
|
|
|
|
|
.unwrap()
|
|
|
|
|
|
|
|
.get(&*next_id)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
// Exponential backoff
|
|
|
|
|
|
|
|
let mut min_elapsed_duration =
|
|
|
|
|
|
|
|
Duration::from_secs(5 * 60) * (*tries) * (*tries);
|
|
|
|
|
|
|
|
if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) {
|
|
|
|
|
|
|
|
min_elapsed_duration = Duration::from_secs(60 * 60 * 24);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if time.elapsed() < min_elapsed_duration {
|
|
|
|
// c. Ask origin server over federation
|
|
|
|
info!("Backing off from {}", next_id);
|
|
|
|
// We also handle its auth chain here so we don't get a stack overflow in
|
|
|
|
continue;
|
|
|
|
// handle_outlier_pdu.
|
|
|
|
}
|
|
|
|
let mut todo_auth_events = vec![Arc::clone(id)];
|
|
|
|
|
|
|
|
let mut events_in_reverse_order = Vec::new();
|
|
|
|
|
|
|
|
let mut events_all = HashSet::new();
|
|
|
|
|
|
|
|
let mut i = 0;
|
|
|
|
|
|
|
|
while let Some(next_id) = todo_auth_events.pop() {
|
|
|
|
|
|
|
|
if let Some((time, tries)) = services()
|
|
|
|
|
|
|
|
.globals
|
|
|
|
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
|
|
|
|
.read()
|
|
|
|
|
|
|
|
.await
|
|
|
|
|
|
|
|
.get(&*next_id)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
// Exponential backoff
|
|
|
|
|
|
|
|
let mut min_elapsed_duration =
|
|
|
|
|
|
|
|
Duration::from_secs(5 * 60) * (*tries) * (*tries);
|
|
|
|
|
|
|
|
if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) {
|
|
|
|
|
|
|
|
min_elapsed_duration = Duration::from_secs(60 * 60 * 24);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if events_all.contains(&next_id) {
|
|
|
|
if time.elapsed() < min_elapsed_duration {
|
|
|
|
|
|
|
|
info!("Backing off from {}", next_id);
|
|
|
|
continue;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
i += 1;
|
|
|
|
if events_all.contains(&next_id) {
|
|
|
|
if i % 100 == 0 {
|
|
|
|
continue;
|
|
|
|
tokio::task::yield_now().await;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if let Ok(Some(_)) = services().rooms.timeline.get_pdu(&next_id) {
|
|
|
|
i += 1;
|
|
|
|
trace!("Found {} in db", next_id);
|
|
|
|
if i % 100 == 0 {
|
|
|
|
continue;
|
|
|
|
tokio::task::yield_now().await;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
info!("Fetching {} over federation.", next_id);
|
|
|
|
if let Ok(Some(_)) = services().rooms.timeline.get_pdu(&next_id) {
|
|
|
|
match services()
|
|
|
|
trace!("Found {} in db", next_id);
|
|
|
|
.sending
|
|
|
|
continue;
|
|
|
|
.send_federation_request(
|
|
|
|
}
|
|
|
|
origin,
|
|
|
|
|
|
|
|
get_event::v1::Request {
|
|
|
|
|
|
|
|
event_id: (*next_id).to_owned(),
|
|
|
|
|
|
|
|
},
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
.await
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
Ok(res) => {
|
|
|
|
|
|
|
|
info!("Got {} over federation", next_id);
|
|
|
|
|
|
|
|
let (calculated_event_id, value) =
|
|
|
|
|
|
|
|
match pdu::gen_event_id_canonical_json(&res.pdu, room_version_id) {
|
|
|
|
|
|
|
|
Ok(t) => t,
|
|
|
|
|
|
|
|
Err(_) => {
|
|
|
|
|
|
|
|
back_off((*next_id).to_owned());
|
|
|
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if calculated_event_id != *next_id {
|
|
|
|
|
|
|
|
warn!("Server didn't return event id we requested: requested: {}, we got {}. Event: {:?}",
|
|
|
|
|
|
|
|
next_id, calculated_event_id, &res.pdu);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if let Some(auth_events) =
|
|
|
|
info!("Fetching {} over federation.", next_id);
|
|
|
|
value.get("auth_events").and_then(|c| c.as_array())
|
|
|
|
match services()
|
|
|
|
{
|
|
|
|
.sending
|
|
|
|
for auth_event in auth_events {
|
|
|
|
.send_federation_request(
|
|
|
|
if let Ok(auth_event) =
|
|
|
|
origin,
|
|
|
|
serde_json::from_value(auth_event.clone().into())
|
|
|
|
get_event::v1::Request {
|
|
|
|
{
|
|
|
|
event_id: (*next_id).to_owned(),
|
|
|
|
let a: Arc<EventId> = auth_event;
|
|
|
|
},
|
|
|
|
todo_auth_events.push(a);
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
.await
|
|
|
|
warn!("Auth event id is not valid");
|
|
|
|
{
|
|
|
|
}
|
|
|
|
Ok(res) => {
|
|
|
|
|
|
|
|
info!("Got {} over federation", next_id);
|
|
|
|
|
|
|
|
let (calculated_event_id, value) =
|
|
|
|
|
|
|
|
match pdu::gen_event_id_canonical_json(&res.pdu, room_version_id) {
|
|
|
|
|
|
|
|
Ok(t) => t,
|
|
|
|
|
|
|
|
Err(_) => {
|
|
|
|
|
|
|
|
back_off((*next_id).to_owned()).await;
|
|
|
|
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
};
|
|
|
|
warn!("Auth event list invalid");
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
events_in_reverse_order.push((next_id.clone(), value));
|
|
|
|
if calculated_event_id != *next_id {
|
|
|
|
events_all.insert(next_id);
|
|
|
|
warn!("Server didn't return event id we requested: requested: {}, we got {}. Event: {:?}",
|
|
|
|
|
|
|
|
next_id, calculated_event_id, &res.pdu);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Err(_) => {
|
|
|
|
|
|
|
|
warn!("Failed to fetch event: {}", next_id);
|
|
|
|
if let Some(auth_events) =
|
|
|
|
back_off((*next_id).to_owned());
|
|
|
|
value.get("auth_events").and_then(|c| c.as_array())
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
for auth_event in auth_events {
|
|
|
|
|
|
|
|
if let Ok(auth_event) =
|
|
|
|
|
|
|
|
serde_json::from_value(auth_event.clone().into())
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
let a: Arc<EventId> = auth_event;
|
|
|
|
|
|
|
|
todo_auth_events.push(a);
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
warn!("Auth event id is not valid");
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
warn!("Auth event list invalid");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
events_in_reverse_order.push((next_id.clone(), value));
|
|
|
|
|
|
|
|
events_all.insert(next_id);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
Err(_) => {
|
|
|
|
|
|
|
|
warn!("Failed to fetch event: {}", next_id);
|
|
|
|
|
|
|
|
back_off((*next_id).to_owned()).await;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
for (next_id, value) in events_in_reverse_order.iter().rev() {
|
|
|
|
for (next_id, value) in events_in_reverse_order.iter().rev() {
|
|
|
|
if let Some((time, tries)) = services()
|
|
|
|
if let Some((time, tries)) = services()
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.read()
|
|
|
|
.read()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.get(&**next_id)
|
|
|
|
.get(&**next_id)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
// Exponential backoff
|
|
|
|
// Exponential backoff
|
|
|
|
let mut min_elapsed_duration =
|
|
|
|
let mut min_elapsed_duration =
|
|
|
|
Duration::from_secs(5 * 60) * (*tries) * (*tries);
|
|
|
|
Duration::from_secs(5 * 60) * (*tries) * (*tries);
|
|
|
|
if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) {
|
|
|
|
if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) {
|
|
|
|
min_elapsed_duration = Duration::from_secs(60 * 60 * 24);
|
|
|
|
min_elapsed_duration = Duration::from_secs(60 * 60 * 24);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if time.elapsed() < min_elapsed_duration {
|
|
|
|
if time.elapsed() < min_elapsed_duration {
|
|
|
|
info!("Backing off from {}", next_id);
|
|
|
|
info!("Backing off from {}", next_id);
|
|
|
|
continue;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
match self
|
|
|
|
match self
|
|
|
|
.handle_outlier_pdu(
|
|
|
|
.handle_outlier_pdu(
|
|
|
|
origin,
|
|
|
|
origin,
|
|
|
|
create_event,
|
|
|
|
create_event,
|
|
|
|
next_id,
|
|
|
|
next_id,
|
|
|
|
room_id,
|
|
|
|
room_id,
|
|
|
|
value.clone(),
|
|
|
|
value.clone(),
|
|
|
|
true,
|
|
|
|
true,
|
|
|
|
pub_key_map,
|
|
|
|
pub_key_map,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
.await
|
|
|
|
.await
|
|
|
|
{
|
|
|
|
{
|
|
|
|
Ok((pdu, json)) => {
|
|
|
|
Ok((pdu, json)) => {
|
|
|
|
if next_id == id {
|
|
|
|
if next_id == id {
|
|
|
|
pdus.push((pdu, Some(json)));
|
|
|
|
pdus.push((pdu, Some(json)));
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
Err(e) => {
|
|
|
|
|
|
|
|
warn!("Authentication of event {} failed: {:?}", next_id, e);
|
|
|
|
|
|
|
|
back_off((**next_id).to_owned());
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Err(e) => {
|
|
|
|
|
|
|
|
warn!("Authentication of event {} failed: {:?}", next_id, e);
|
|
|
|
|
|
|
|
back_off((**next_id).to_owned()).await;
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
pdus
|
|
|
|
}
|
|
|
|
})
|
|
|
|
pdus
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn fetch_unknown_prev_events(
|
|
|
|
async fn fetch_unknown_prev_events(
|
|
|
|
@ -1360,7 +1370,7 @@ impl Service {
|
|
|
|
|
|
|
|
|
|
|
|
pub_key_map
|
|
|
|
pub_key_map
|
|
|
|
.write()
|
|
|
|
.write()
|
|
|
|
.map_err(|_| Error::bad_database("RwLock is poisoned."))?
|
|
|
|
.await
|
|
|
|
.insert(signature_server.clone(), keys);
|
|
|
|
.insert(signature_server.clone(), keys);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@ -1369,7 +1379,7 @@ impl Service {
|
|
|
|
|
|
|
|
|
|
|
|
// Gets a list of servers for which we don't have the signing key yet. We go over
|
|
|
|
// Gets a list of servers for which we don't have the signing key yet. We go over
|
|
|
|
// the PDUs and either cache the key or add it to the list that needs to be retrieved.
|
|
|
|
// the PDUs and either cache the key or add it to the list that needs to be retrieved.
|
|
|
|
fn get_server_keys_from_cache(
|
|
|
|
async fn get_server_keys_from_cache(
|
|
|
|
&self,
|
|
|
|
&self,
|
|
|
|
pdu: &RawJsonValue,
|
|
|
|
pdu: &RawJsonValue,
|
|
|
|
servers: &mut BTreeMap<OwnedServerName, BTreeMap<OwnedServerSigningKeyId, QueryCriteria>>,
|
|
|
|
servers: &mut BTreeMap<OwnedServerName, BTreeMap<OwnedServerSigningKeyId, QueryCriteria>>,
|
|
|
|
@ -1393,7 +1403,7 @@ impl Service {
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.bad_event_ratelimiter
|
|
|
|
.read()
|
|
|
|
.read()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.get(event_id)
|
|
|
|
.get(event_id)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
// Exponential backoff
|
|
|
|
// Exponential backoff
|
|
|
|
@ -1469,17 +1479,19 @@ impl Service {
|
|
|
|
> = BTreeMap::new();
|
|
|
|
> = BTreeMap::new();
|
|
|
|
|
|
|
|
|
|
|
|
{
|
|
|
|
{
|
|
|
|
let mut pkm = pub_key_map
|
|
|
|
let mut pkm = pub_key_map.write().await;
|
|
|
|
.write()
|
|
|
|
|
|
|
|
.map_err(|_| Error::bad_database("RwLock is poisoned."))?;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Try to fetch keys, failure is okay
|
|
|
|
// Try to fetch keys, failure is okay
|
|
|
|
// Servers we couldn't find in the cache will be added to `servers`
|
|
|
|
// Servers we couldn't find in the cache will be added to `servers`
|
|
|
|
for pdu in &event.room_state.state {
|
|
|
|
for pdu in &event.room_state.state {
|
|
|
|
let _ = self.get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm);
|
|
|
|
let _ = self
|
|
|
|
|
|
|
|
.get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm)
|
|
|
|
|
|
|
|
.await;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for pdu in &event.room_state.auth_chain {
|
|
|
|
for pdu in &event.room_state.auth_chain {
|
|
|
|
let _ = self.get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm);
|
|
|
|
let _ = self
|
|
|
|
|
|
|
|
.get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm)
|
|
|
|
|
|
|
|
.await;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
drop(pkm);
|
|
|
|
drop(pkm);
|
|
|
|
@ -1503,9 +1515,7 @@ impl Service {
|
|
|
|
.await
|
|
|
|
.await
|
|
|
|
{
|
|
|
|
{
|
|
|
|
trace!("Got signing keys: {:?}", keys);
|
|
|
|
trace!("Got signing keys: {:?}", keys);
|
|
|
|
let mut pkm = pub_key_map
|
|
|
|
let mut pkm = pub_key_map.write().await;
|
|
|
|
.write()
|
|
|
|
|
|
|
|
.map_err(|_| Error::bad_database("RwLock is poisoned."))?;
|
|
|
|
|
|
|
|
for k in keys.server_keys {
|
|
|
|
for k in keys.server_keys {
|
|
|
|
let k = match k.deserialize() {
|
|
|
|
let k = match k.deserialize() {
|
|
|
|
Ok(key) => key,
|
|
|
|
Ok(key) => key,
|
|
|
|
@ -1564,10 +1574,7 @@ impl Service {
|
|
|
|
.into_iter()
|
|
|
|
.into_iter()
|
|
|
|
.map(|(k, v)| (k.to_string(), v.key))
|
|
|
|
.map(|(k, v)| (k.to_string(), v.key))
|
|
|
|
.collect();
|
|
|
|
.collect();
|
|
|
|
pub_key_map
|
|
|
|
pub_key_map.write().await.insert(origin.to_string(), result);
|
|
|
|
.write()
|
|
|
|
|
|
|
|
.map_err(|_| Error::bad_database("RwLock is poisoned."))?
|
|
|
|
|
|
|
|
.insert(origin.to_string(), result);
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
info!("Done handling result");
|
|
|
|
info!("Done handling result");
|
|
|
|
@ -1632,14 +1639,14 @@ impl Service {
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.servername_ratelimiter
|
|
|
|
.servername_ratelimiter
|
|
|
|
.read()
|
|
|
|
.read()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.get(origin)
|
|
|
|
.get(origin)
|
|
|
|
.map(|s| Arc::clone(s).acquire_owned());
|
|
|
|
.map(|s| Arc::clone(s).acquire_owned());
|
|
|
|
|
|
|
|
|
|
|
|
let permit = match permit {
|
|
|
|
let permit = match permit {
|
|
|
|
Some(p) => p,
|
|
|
|
Some(p) => p,
|
|
|
|
None => {
|
|
|
|
None => {
|
|
|
|
let mut write = services().globals.servername_ratelimiter.write().unwrap();
|
|
|
|
let mut write = services().globals.servername_ratelimiter.write().await;
|
|
|
|
let s = Arc::clone(
|
|
|
|
let s = Arc::clone(
|
|
|
|
write
|
|
|
|
write
|
|
|
|
.entry(origin.to_owned())
|
|
|
|
.entry(origin.to_owned())
|
|
|
|
@ -1651,24 +1658,26 @@ impl Service {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
.await;
|
|
|
|
.await;
|
|
|
|
|
|
|
|
|
|
|
|
let back_off = |id| match services()
|
|
|
|
let back_off = |id| async {
|
|
|
|
.globals
|
|
|
|
match services()
|
|
|
|
.bad_signature_ratelimiter
|
|
|
|
.globals
|
|
|
|
.write()
|
|
|
|
.bad_signature_ratelimiter
|
|
|
|
.unwrap()
|
|
|
|
.write()
|
|
|
|
.entry(id)
|
|
|
|
.await
|
|
|
|
{
|
|
|
|
.entry(id)
|
|
|
|
hash_map::Entry::Vacant(e) => {
|
|
|
|
{
|
|
|
|
e.insert((Instant::now(), 1));
|
|
|
|
hash_map::Entry::Vacant(e) => {
|
|
|
|
|
|
|
|
e.insert((Instant::now(), 1));
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
hash_map::Entry::Occupied(mut e) => *e.get_mut() = (Instant::now(), e.get().1 + 1),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
hash_map::Entry::Occupied(mut e) => *e.get_mut() = (Instant::now(), e.get().1 + 1),
|
|
|
|
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
if let Some((time, tries)) = services()
|
|
|
|
if let Some((time, tries)) = services()
|
|
|
|
.globals
|
|
|
|
.globals
|
|
|
|
.bad_signature_ratelimiter
|
|
|
|
.bad_signature_ratelimiter
|
|
|
|
.read()
|
|
|
|
.read()
|
|
|
|
.unwrap()
|
|
|
|
.await
|
|
|
|
.get(&signature_ids)
|
|
|
|
.get(&signature_ids)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
// Exponential backoff
|
|
|
|
// Exponential backoff
|
|
|
|
@ -1775,7 +1784,7 @@ impl Service {
|
|
|
|
|
|
|
|
|
|
|
|
drop(permit);
|
|
|
|
drop(permit);
|
|
|
|
|
|
|
|
|
|
|
|
back_off(signature_ids);
|
|
|
|
back_off(signature_ids).await;
|
|
|
|
|
|
|
|
|
|
|
|
warn!("Failed to find public key for server: {}", origin);
|
|
|
|
warn!("Failed to find public key for server: {}", origin);
|
|
|
|
Err(Error::BadServerResponse(
|
|
|
|
Err(Error::BadServerResponse(
|
|
|
|
|
Matthias Ahouansou
4 months ago