From 96dd3b2880fedd218d7faa052e2f38febb90c690 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Tue, 27 Oct 2020 19:10:09 -0400 Subject: [PATCH 01/36] Update ruma to latest, fix unstable origin feature in ruma --- Cargo.lock | 485 ++++++++++++++++++++++++------ Cargo.toml | 17 +- src/client_server/backup.rs | 23 +- src/client_server/capabilities.rs | 8 +- src/client_server/filter.rs | 8 +- src/client_server/keys.rs | 8 +- src/client_server/media.rs | 24 +- src/client_server/membership.rs | 58 ++-- src/client_server/state.rs | 54 ++-- src/database/rooms.rs | 48 +-- src/database/users.rs | 6 +- src/ruma_wrapper.rs | 47 +-- src/server_server.rs | 23 +- 13 files changed, 576 insertions(+), 233 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6fed2aff..461972bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,9 +44,9 @@ checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" [[package]] name = "arrayvec" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "assign" @@ -221,7 +221,7 @@ dependencies = [ "reqwest", "ring", "rocket", - "ruma", + "ruma 0.0.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", "rust-argon2", "serde", "serde_json", @@ -273,11 +273,11 @@ checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" [[package]] name = "crc32fast" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] @@ -464,9 +464,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8e3078b7b2a8a671cb7a3d17b4760e4181ea243227776ba83fd043b4ca034e" +checksum = "95314d38584ffbfda215621d723e0a3906f032e03ae5551e650058dac83d4797" dependencies = [ "futures-channel", "futures-core", @@ -479,9 +479,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a4d35f7401e948629c9c3d6638fb9bf94e0b2121e96c3b428cc4e631f3eb74" +checksum = "0448174b01148032eed37ac4aed28963aaaa8cfa93569a08e5b479bbc6c2c151" dependencies = [ "futures-core", "futures-sink", @@ -489,15 +489,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d674eaa0056896d5ada519900dbf97ead2e46a7b6621e8160d79e2f2e1e2784b" +checksum = "18eaa56102984bed2c88ea39026cff3ce3b4c7f508ca970cedf2450ea10d4e46" [[package]] name = "futures-executor" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc709ca1da6f66143b8c9bec8e6260181869893714e9b5a490b169b0414144ab" +checksum = "f5f8e0c9258abaea85e78ebdda17ef9666d390e987f006be6080dfe354b708cb" dependencies = [ "futures-core", "futures-task", @@ -506,15 +506,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc94b64bb39543b4e432f1790b6bf18e3ee3b74653c5449f63310e9a74b123c" +checksum = "6e1798854a4727ff944a7b12aa999f58ce7aa81db80d2dfaaf2ba06f065ddd2b" [[package]] name = "futures-macro" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f57ed14da4603b2554682e9f2ff3c65d7567b53188db96cb71538217fc64581b" +checksum = "e36fccf3fc58563b4a14d265027c627c3b665d7fed489427e88e7cc929559efe" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -524,24 +524,24 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d8764258ed64ebc5d9ed185cf86a95db5cac810269c5d20ececb32e0088abbd" +checksum = "0e3ca3f17d6e8804ae5d3df7a7d35b2b3a6fe89dac84b31872720fc3060a0b11" [[package]] name = "futures-task" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd26820a9f3637f1302da8bceba3ff33adbe53464b54ca24d4e2d4f1db30f94" +checksum = "96d502af37186c4fef99453df03e374683f8a1eec9dcc1e66b3b82dc8278ce3c" dependencies = [ "once_cell", ] [[package]] name = "futures-util" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a894a0acddba51a2d49a6f4263b1e64b8c579ece8af50fa86503d52cd1eea34" +checksum = "abcb44342f62e6f3e8ac427b8aa815f724fd705dfad060b18ac7866c15bb8e34" dependencies = [ "futures-channel", "futures-core", @@ -550,7 +550,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project", + "pin-project 1.0.1", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -566,6 +566,19 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generator" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cdc09201b2e8ca1b19290cf7e65de2246b8e91fb6874279722189c4de7b94dc" +dependencies = [ + "cc", + "libc", + "log", + "rustc_version", + "winapi 0.3.9", +] + [[package]] name = "getrandom" version = "0.1.15" @@ -601,9 +614,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993f9e0baeed60001cf565546b0d3dbe6a6ad23f2bd31644a133c641eccf6d53" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" dependencies = [ "bytes", "fnv", @@ -616,6 +629,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", + "tracing-futures", ] [[package]] @@ -702,7 +716,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project", + "pin-project 0.4.27", "socket2", "tokio", "tower-service", @@ -736,12 +750,13 @@ dependencies = [ [[package]] name = "image" -version = "0.23.10" +version = "0.23.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985fc06b1304d19c28d5c562ed78ef5316183f2b0053b46763a0b94862373c34" +checksum = "b4f0a8345b33b082aedec2f4d7d4a926b845cee184cbe78b703413066564431b" dependencies = [ "bytemuck", "byteorder", + "color_quant", "gif", "jpeg-decoder", "num-iter", @@ -768,11 +783,11 @@ checksum = "cb6ee2a7da03bfc3b66ca47c92c2e392fcc053ea040a85561749b026f7aad09a" [[package]] name = "instant" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" +checksum = "cb1fc4429a33e1f80d41dc9fea4d108a88bec1de8053878898ae448a0b52f613" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] @@ -862,9 +877,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" +checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" [[package]] name = "linked-hash-map" @@ -890,6 +905,19 @@ dependencies = [ "cfg-if 0.1.10", ] +[[package]] +name = "loom" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0e8460f2f2121162705187214720353c517b97bdfb3494c0b1e33d83ebe4bed" +dependencies = [ + "cfg-if 0.1.10", + "generator", + "scoped-tls", + "serde", + "serde_json", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -1218,7 +1246,16 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" dependencies = [ - "pin-project-internal", + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841" +dependencies = [ + "pin-project-internal 1.0.1", ] [[package]] @@ -1232,11 +1269,22 @@ dependencies = [ "syn", ] +[[package]] +name = "pin-project-internal" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e555d9e657502182ac97b539fb3dae8b79cda19e3e4f8ffb5e8de4f18df93c95" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" [[package]] name = "pin-utils" @@ -1559,38 +1607,88 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +dependencies = [ + "assign", + "js_int", + "ruma-api 0.17.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-appservice-api 0.2.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-client-api 0.10.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-events 0.22.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-federation-api 0.0.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-signatures 0.6.0-dev.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", +] + +[[package]] +name = "ruma" +version = "0.0.1" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" +dependencies = [ + "assign", + "js_int", + "ruma-api 0.17.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-appservice-api 0.2.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-client-api 0.10.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-events 0.22.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-federation-api 0.0.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-signatures 0.6.0-dev.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", +] + +[[package]] +name = "ruma-api" +version = "0.17.0-alpha.1" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" dependencies = [ - "ruma-api", - "ruma-appservice-api", - "ruma-client-api", - "ruma-common", - "ruma-events", - "ruma-federation-api", - "ruma-identifiers", - "ruma-serde", - "ruma-signatures", + "http", + "percent-encoding", + "ruma-api-macros 0.17.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "serde", + "serde_json", + "strum", + "thiserror", ] [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" dependencies = [ "http", "percent-encoding", - "ruma-api-macros", - "ruma-identifiers", - "ruma-serde", + "ruma-api-macros 0.17.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", "serde", "serde_json", "strum", + "thiserror", +] + +[[package]] +name = "ruma-api-macros" +version = "0.17.0-alpha.1" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1601,30 +1699,78 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +dependencies = [ + "ruma-api 0.17.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-events 0.22.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "serde", + "serde_json", +] + +[[package]] +name = "ruma-appservice-api" +version = "0.2.0-alpha.1" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" dependencies = [ - "ruma-api", - "ruma-common", - "ruma-events", - "ruma-identifiers", + "ruma-api 0.17.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-events 0.22.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "serde", + "serde_json", +] + +[[package]] +name = "ruma-client-api" +version = "0.10.0-alpha.1" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +dependencies = [ + "assign", + "http", + "js_int", + "maplit", + "percent-encoding", + "ruma-api 0.17.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-events 0.22.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", "serde", "serde_json", + "strum", ] [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" dependencies = [ "assign", "http", "js_int", + "maplit", "percent-encoding", - "ruma-api", - "ruma-common", - "ruma-events", - "ruma-identifiers", - "ruma-serde", + "ruma-api 0.17.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-events 0.22.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "serde", + "serde_json", + "strum", +] + +[[package]] +name = "ruma-common" +version = "0.2.0" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +dependencies = [ + "js_int", + "ruma-common-macros 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", "serde", "serde_json", "strum", @@ -1633,36 +1779,84 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" dependencies = [ "js_int", - "ruma-api", - "ruma-identifiers", - "ruma-serde", + "ruma-common-macros 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", "serde", "serde_json", "strum", ] +[[package]] +name = "ruma-common-macros" +version = "0.2.0" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ruma-common-macros" +version = "0.2.0" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" dependencies = [ "js_int", - "ruma-common", - "ruma-events-macros", - "ruma-identifiers", - "ruma-serde", + "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-events-macros 0.22.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", "serde", "serde_json", "strum", ] +[[package]] +name = "ruma-events" +version = "0.22.0-alpha.1" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" +dependencies = [ + "js_int", + "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-events-macros 0.22.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "serde", + "serde_json", + "strum", +] + +[[package]] +name = "ruma-events-macros" +version = "0.22.0-alpha.1" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1673,14 +1867,29 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" dependencies = [ "js_int", - "ruma-api", - "ruma-common", - "ruma-events", - "ruma-identifiers", - "ruma-serde", + "ruma-api 0.17.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-events 0.22.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "serde", + "serde_json", +] + +[[package]] +name = "ruma-federation-api" +version = "0.0.3" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" +dependencies = [ + "js_int", + "ruma-api 0.17.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-events 0.22.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", "serde", "serde_json", ] @@ -1688,11 +1897,24 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +dependencies = [ + "ruma-identifiers-macros 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-identifiers-validation 0.1.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "serde", + "strum", +] + +[[package]] +name = "ruma-identifiers" +version = "0.17.4" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" dependencies = [ "rand", - "ruma-identifiers-macros", - "ruma-identifiers-validation", + "ruma-identifiers-macros 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-identifiers-validation 0.1.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", "serde", "strum", ] @@ -1700,27 +1922,59 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +dependencies = [ + "proc-macro2", + "quote", + "ruma-identifiers-validation 0.1.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "syn", +] + +[[package]] +name = "ruma-identifiers-macros" +version = "0.17.4" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" dependencies = [ "proc-macro2", "quote", - "ruma-identifiers-validation", + "ruma-identifiers-validation 0.1.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", "syn", ] [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" dependencies = [ "serde", "strum", ] +[[package]] +name = "ruma-identifiers-validation" +version = "0.1.1" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" +dependencies = [ + "serde", + "strum", +] + +[[package]] +name = "ruma-serde" +version = "0.2.3" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +dependencies = [ + "form_urlencoded", + "itoa", + "js_int", + "serde", + "serde_json", +] + [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" dependencies = [ "form_urlencoded", "itoa", @@ -1732,10 +1986,25 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" +source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" dependencies = [ "base64", "ring", + "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "serde_json", + "untrusted", +] + +[[package]] +name = "ruma-signatures" +version = "0.6.0-dev.1" +source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" +dependencies = [ + "base64", + "ring", + "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", "serde_json", "untrusted", ] @@ -1754,9 +2023,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2610b7f643d18c87dff3b489950269617e6601a51f1f05aa5daefee36f64f0b" +checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" [[package]] name = "rustc_version" @@ -1796,6 +2065,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "scoped-tls" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1901,11 +2176,12 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sharded-slab" -version = "0.0.9" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06d5a3f5166fb5b42a5439f2eee8b9de149e235961e3eb21c5808fc3ea17ff3e" +checksum = "7b4921be914e16899a80adefb821f8ddb7974e3f1250223575a44ed994882127" dependencies = [ "lazy_static", + "loom", ] [[package]] @@ -1982,12 +2258,11 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#a7d76935f12757aecfee305838069c9bcbe7d34a" dependencies = [ "itertools", "js_int", "maplit", - "ruma", + "ruma 0.0.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", "serde", "serde_json", "thiserror", @@ -2067,9 +2342,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.44" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e03e57e4fcbfe7749842d53e24ccb9aa12b7252dbe5e91d2acad31834c8b8fdd" +checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" dependencies = [ "proc-macro2", "quote", @@ -2292,6 +2567,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + [[package]] name = "tracing-log" version = "0.1.1" @@ -2315,9 +2600,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ef0a5e15477aa303afbfac3a44cba9b6430fdaad52423b1e6c0dbbe28c3eedd" +checksum = "2810660b9d5b18895d140caba6401765749a6a162e5d0736cfc44ea50db9d79d" dependencies = [ "ansi_term", "chrono", @@ -2561,9 +2846,9 @@ dependencies = [ [[package]] name = "weezl" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e26e7a4d998e3d7949c69444b8b4916bac810da0d3a82ae612c89e952782f4" +checksum = "8795d6e0e17485803cc10ef126bb8c0d59b7c61b219d66cfe0b3216dd0e8580a" [[package]] name = "widestring" diff --git a/Cargo.toml b/Cargo.toml index 8b29be8f..a6f89b90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,12 +19,13 @@ rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_p # Used for matrix spec type definitions and helpers #ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } -ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fed-fixes" } -#ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } +ruma = { git = "https://github.com/DevinR528/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "unstable-join" } +# ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } # Used when doing state resolution -state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] } -#state-res = { path = "../state-res", features = ["unstable-pre-spec"] } +# state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] } +state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec"] } +# state-res = { path = "../../state-res", features = ["unstable-pre-spec"] } # Used for long polling tokio = "0.2.22" @@ -72,3 +73,11 @@ required-features = ["conduit_bin"] [lib] name = "conduit" path = "src/lib.rs" + +# [patch."https://github.com/timokoesters/ruma"] +# ruma = { path = "../ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"] } +# # ruma = { git = "https://github.com/ruma/ruma", rev = "64b9c646d15a359d62ab464a95176ff94adb2554", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"]} + +[patch."https://github.com/ruma/state-res"] +state-res = { path = "../../state-res", features = ["unstable-pre-spec"] } +# state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp" } \ No newline at end of file diff --git a/src/client_server/backup.rs b/src/client_server/backup.rs index c84af0a6..607fa39e 100644 --- a/src/client_server/backup.rs +++ b/src/client_server/backup.rs @@ -107,7 +107,7 @@ pub async fn get_backup_route( )] pub async fn delete_backup_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -158,7 +158,7 @@ pub async fn add_backup_keys_route( )] pub async fn add_backup_key_sessions_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -189,7 +189,7 @@ pub async fn add_backup_key_sessions_route( )] pub async fn add_backup_key_session_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -232,7 +232,7 @@ pub async fn get_backup_keys_route( )] pub async fn get_backup_key_sessions_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -249,13 +249,14 @@ pub async fn get_backup_key_sessions_route( )] pub async fn get_backup_key_session_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let key_data = - db.key_backups - .get_session(&sender_user, &body.version, &body.room_id, &body.session_id)?; + let key_data = db + .key_backups + .get_session(&sender_user, &body.version, &body.room_id, &body.session_id)? + .ok_or_else(|| Error::BadDatabase("Backup key not found for this user's session"))?; Ok(get_backup_key_session::Response { key_data }.into()) } @@ -266,7 +267,7 @@ pub async fn get_backup_key_session_route( )] pub async fn delete_backup_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -288,7 +289,7 @@ pub async fn delete_backup_keys_route( )] pub async fn delete_backup_key_sessions_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -310,7 +311,7 @@ pub async fn delete_backup_key_sessions_route( )] pub async fn delete_backup_key_session_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); diff --git a/src/client_server/capabilities.rs b/src/client_server/capabilities.rs index 54c08ba8..fa12a08e 100644 --- a/src/client_server/capabilities.rs +++ b/src/client_server/capabilities.rs @@ -22,11 +22,11 @@ pub async fn get_capabilities_route() -> ConduitResult ConduitResult { // TODO Ok(get_filter::Response::new(filter::IncomingFilterDefinition { event_fields: None, - event_format: None, - account_data: None, - room: None, - presence: None, + event_format: filter::EventFormat::default(), + account_data: filter::IncomingFilter::default(), + room: filter::IncomingRoomFilter::default(), + presence: filter::IncomingFilter::default(), }) .into()) } diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index 58c79dad..8426518b 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -11,7 +11,7 @@ use ruma::{ uiaa::{AuthFlow, UiaaInfo}, }, }, - encryption::IncomingUnsignedDeviceInfo, + encryption::UnsignedDeviceInfo, }; use std::collections::{BTreeMap, HashSet}; @@ -24,7 +24,7 @@ use rocket::{get, post}; )] pub async fn upload_keys_route( db: State<'_, Database>, - body: Ruma>, + body: Ruma, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -94,7 +94,7 @@ pub async fn get_keys_route( Error::bad_database("all_device_keys contained nonexistent device.") })?; - keys.unsigned = IncomingUnsignedDeviceInfo { + keys.unsigned = UnsignedDeviceInfo { device_display_name: metadata.display_name, }; @@ -113,7 +113,7 @@ pub async fn get_keys_route( ), )?; - keys.unsigned = IncomingUnsignedDeviceInfo { + keys.unsigned = UnsignedDeviceInfo { device_display_name: metadata.display_name, }; diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 96874cc4..af7880cd 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -39,7 +39,7 @@ pub async fn create_content_route( db.media.create( mxc.clone(), &body.filename.as_deref(), - &body.content_type, + body.content_type.as_deref().unwrap_or("img"), // TODO this is now optional handle &body.file, )?; @@ -66,8 +66,8 @@ pub async fn get_content_route( { Ok(get_content::Response { file, - content_type, - content_disposition: filename.unwrap_or_default(), // TODO: Spec says this should be optional + content_type: Some(content_type), + content_disposition: filename, } .into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { @@ -84,8 +84,11 @@ pub async fn get_content_route( db.media.create( mxc, - &Some(&get_content_response.content_disposition), - &get_content_response.content_type, + &get_content_response.content_disposition.as_deref(), + get_content_response // TODO this is now optional handle + .content_type + .as_deref() + .unwrap_or("img"), &get_content_response.file, )?; @@ -116,7 +119,11 @@ pub async fn get_content_thumbnail_route( .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, )? { - Ok(get_content_thumbnail::Response { file, content_type }.into()) + Ok(get_content_thumbnail::Response { + file, + content_type: Some(content_type), + } + .into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_thumbnail_response = server_server::send_request( &db.globals, @@ -135,7 +142,10 @@ pub async fn get_content_thumbnail_route( db.media.upload_thumbnail( mxc, &None, - &get_thumbnail_response.content_type, + get_thumbnail_response + .content_type + .as_deref() + .unwrap_or("img"), // TODO now optional, deal with it somehow body.width.try_into().expect("all UInts are valid u32s"), body.height.try_into().expect("all UInts are valid u32s"), &get_thumbnail_response.file, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 25cad85c..50a8cca0 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -23,7 +23,11 @@ use ruma::{ }; use state_res::StateEvent; use std::{ - collections::BTreeMap, collections::HashMap, collections::HashSet, convert::TryFrom, iter, + collections::BTreeMap, + collections::HashMap, + collections::HashSet, + convert::{TryFrom, TryInto}, + iter, sync::Arc, }; @@ -509,31 +513,42 @@ async fn join_room_by_id_helper( .expect("event is valid, we just created it"), ); + // TODO fixup CanonicalJsonValue + // use that instead of serde_json::Map... maybe? + let mut canon_json_stub = + serde_json::from_value(join_event_stub_value).expect("json Value is canonical JSON"); // Generate event id let event_id = EventId::try_from(&*format!( "${}", - ruma::signatures::reference_hash(&join_event_stub_value) + ruma::signatures::reference_hash(&canon_json_stub, &RoomVersionId::Version6) .expect("ruma can calculate reference hashes") )) .expect("ruma's reference hashes are valid event ids"); // We don't leave the event id into the pdu because that's only allowed in v1 or v2 rooms - let join_event_stub = join_event_stub_value.as_object_mut().unwrap(); - join_event_stub.remove("event_id"); + // let join_event_stub = join_event_stub_value.as_object_mut().unwrap(); + // join_event_stub.remove("event_id"); + + canon_json_stub.remove("event_id"); ruma::signatures::hash_and_sign_event( db.globals.server_name().as_str(), db.globals.keypair(), - &mut join_event_stub_value, + &mut canon_json_stub, + &RoomVersionId::Version6, ) .expect("event is valid, we just created it"); // Add event_id back - let join_event_stub = join_event_stub_value.as_object_mut().unwrap(); - join_event_stub.insert("event_id".to_owned(), event_id.to_string().into()); + canon_json_stub.insert( + "event_id".to_owned(), + serde_json::json!(event_id) + .try_into() + .expect("EventId is a valid CanonicalJsonValue"), + ); // It has enough fields to be called a proper event now - let join_event = join_event_stub_value; + let join_event = canon_json_stub; let send_join_response = server_server::send_request( &db.globals, @@ -541,27 +556,31 @@ async fn join_room_by_id_helper( federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, - pdu_stub: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), + pdu_stub: PduEvent::convert_to_outgoing_federation_event( + serde_json::to_value(&join_event) + .expect("we just validated and ser/de this event"), + ), }, ) .await?; - let add_event_id = |pdu: &Raw| { + let add_event_id = |pdu: &Raw| -> Result<(EventId, serde_json::Value)> { let mut value = serde_json::from_str(pdu.json().get()) .expect("converting raw jsons to values always works"); let event_id = EventId::try_from(&*format!( "${}", - ruma::signatures::reference_hash(&value) + ruma::signatures::reference_hash(&value, &RoomVersionId::Version6) .expect("ruma can calculate reference hashes") )) .expect("ruma's reference hashes are valid event ids"); - value - .as_object_mut() - .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? - .insert("event_id".to_owned(), event_id.to_string().into()); + value.insert( + "event_id".to_owned(), + serde_json::from_value(serde_json::json!(event_id)) + .expect("a valid EventId can be converted to CanonicalJsonValue"), + ); - Ok((event_id, value)) + Ok((event_id, serde_json::json!(value))) // TODO CanonicalJsonValue fixup? }; let room_state = send_join_response.room_state.state.iter().map(add_event_id); @@ -580,7 +599,10 @@ async fn join_room_by_id_helper( let mut event_map = room_state .chain(auth_chain) - .chain(iter::once(Ok((event_id, join_event)))) // Add join event we just created + .chain(iter::once(Ok(( + event_id, + serde_json::to_value(join_event).unwrap(), + )))) // Add join event we just created .map(|r| { let (event_id, value) = r?; serde_json::from_value::(value.clone()) @@ -595,7 +617,7 @@ async fn join_room_by_id_helper( let control_events = event_map .values() .filter(|pdu| pdu.is_power_event()) - .map(|pdu| pdu.event_id().clone()) + .map(|pdu| pdu.event_id()) .collect::>(); // These events are not guaranteed to be sorted but they are resolved according to spec diff --git a/src/client_server/state.rs b/src/client_server/state.rs index ca6bdf7e..37778623 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -99,14 +99,14 @@ pub async fn send_state_event_for_empty_key_route( )] pub async fn get_state_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); // Users not in the room should not be able to access the state unless history_visibility is // WorldReadable - if !db.rooms.is_joined(sender_user, &body.room_id)? { - if !matches!( + if !db.rooms.is_joined(sender_user, &body.room_id)? + && !matches!( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|(_, event)| { @@ -119,12 +119,12 @@ pub async fn get_state_events_route( .map(|e| e.history_visibility) }), Some(Ok(HistoryVisibility::WorldReadable)) - ) { - return Err(Error::BadRequest( - ErrorKind::Forbidden, - "You don't have permission to view the room state.", - )); - } + ) + { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "You don't have permission to view the room state.", + )); } Ok(get_state_events::Response { @@ -144,14 +144,14 @@ pub async fn get_state_events_route( )] pub async fn get_state_events_for_key_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); // Users not in the room should not be able to access the state unless history_visibility is // WorldReadable - if !db.rooms.is_joined(sender_user, &body.room_id)? { - if !matches!( + if !db.rooms.is_joined(sender_user, &body.room_id)? + && !matches!( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|(_, event)| { @@ -164,12 +164,12 @@ pub async fn get_state_events_for_key_route( .map(|e| e.history_visibility) }), Some(Ok(HistoryVisibility::WorldReadable)) - ) { - return Err(Error::BadRequest( - ErrorKind::Forbidden, - "You don't have permission to view the room state.", - )); - } + ) + { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "You don't have permission to view the room state.", + )); } let event = db @@ -194,14 +194,14 @@ pub async fn get_state_events_for_key_route( )] pub async fn get_state_events_for_empty_key_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); // Users not in the room should not be able to access the state unless history_visibility is // WorldReadable - if !db.rooms.is_joined(sender_user, &body.room_id)? { - if !matches!( + if !db.rooms.is_joined(sender_user, &body.room_id)? + && !matches!( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|(_, event)| { @@ -214,12 +214,12 @@ pub async fn get_state_events_for_empty_key_route( .map(|e| e.history_visibility) }), Some(Ok(HistoryVisibility::WorldReadable)) - ) { - return Err(Error::BadRequest( - ErrorKind::Forbidden, - "You don't have permission to view the room state.", - )); - } + ) + { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "You don't have permission to view the room state.", + )); } let event = db diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 8ab900fd..d9238a9d 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -15,7 +15,7 @@ use ruma::{ }, EventType, }, - EventId, Raw, RoomAliasId, RoomId, ServerName, UserId, + EventId, Raw, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId, }; use sled::IVec; use state_res::{event_auth, Error as StateError, Requester, StateEvent, StateMap, StateStore}; @@ -196,7 +196,7 @@ impl Rooms { Ok(self.pduid_statehash.get(pdu_id)?) } - /// Returns the last state hash key added to the db. + /// Returns the last state hash key added to the db for the given room. pub fn current_state_hash(&self, room_id: &RoomId) -> Result> { Ok(self.roomid_statehash.get(room_id.as_bytes())?) } @@ -249,7 +249,7 @@ impl Rooms { .is_some()) } - /// Returns the full room state. + /// Force the creation of a new StateHash and insert it into the db. pub fn force_state( &self, room_id: &RoomId, @@ -436,6 +436,7 @@ impl Rooms { Ok(()) } + #[allow(clippy::too_many_arguments)] /// Creates a new persisted data unit and adds it to a room. pub fn append_pdu( &self, @@ -687,7 +688,7 @@ impl Rooms { } EventType::RoomMember => { let prev_event = self - .get_pdu(prev_events.iter().next().ok_or(Error::BadRequest( + .get_pdu(prev_events.get(0).ok_or(Error::BadRequest( ErrorKind::Unknown, "Membership can't be the first event", ))?)? @@ -703,7 +704,7 @@ impl Rooms { sender: &sender, }, prev_event, - None, + None, // TODO: third party invite &auth_events .iter() .map(|((ty, key), pdu)| { @@ -761,7 +762,7 @@ impl Rooms { } let mut pdu = PduEvent { - event_id: EventId::try_from("$thiswillbefilledinlater").expect("we know this is valid"), + event_id: ruma::event_id!("$thiswillbefilledinlater"), room_id: room_id.clone(), sender: sender.clone(), origin_server_ts: utils::millis_since_unix_epoch() @@ -787,37 +788,42 @@ impl Rooms { }; // Hash and sign - let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); - pdu_json - .as_object_mut() - .expect("json is object") - .remove("event_id"); + let mut pdu_json: BTreeMap = + serde_json::from_value(serde_json::json!(&pdu)) + .expect("event is valid, we just created it"); + + pdu_json.remove("event_id"); // Add origin because synapse likes that (and it's required in the spec) - pdu_json - .as_object_mut() - .expect("json is object") - .insert("origin".to_owned(), globals.server_name().as_str().into()); + pdu_json.insert( + "origin".to_owned(), + serde_json::json!(globals.server_name()) + .try_into() + .expect("server name is a valid CanonicalJsonValue"), + ); ruma::signatures::hash_and_sign_event( globals.server_name().as_str(), globals.keypair(), &mut pdu_json, + &RoomVersionId::Version6, ) .expect("event is valid, we just created it"); // Generate event id pdu.event_id = EventId::try_from(&*format!( "${}", - ruma::signatures::reference_hash(&pdu_json) + ruma::signatures::reference_hash(&pdu_json, &RoomVersionId::Version6) .expect("ruma can calculate reference hashes") )) .expect("ruma's reference hashes are valid event ids"); - pdu_json - .as_object_mut() - .expect("json is object") - .insert("event_id".to_owned(), pdu.event_id.to_string().into()); + pdu_json.insert( + "event_id".to_owned(), + serde_json::json!(pdu.event_id) + .try_into() + .expect("EventId is a valid CanonicalJsonValue"), + ); // Increment the last index and use that // This is also the next_batch/since value @@ -832,7 +838,7 @@ impl Rooms { self.append_pdu( &pdu, - &pdu_json, + &serde_json::json!(pdu_json), // TODO fixup CanonicalJsonValue count, pdu_id.clone().into(), globals, diff --git a/src/database/users.rs b/src/database/users.rs index 2a039602..885c0415 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -8,7 +8,7 @@ use ruma::{ keys::{CrossSigningKey, OneTimeKey}, }, }, - encryption::IncomingDeviceKeys, + encryption::DeviceKeys, events::{AnyToDeviceEvent, EventType}, DeviceId, DeviceKeyAlgorithm, DeviceKeyId, Raw, UserId, }; @@ -401,7 +401,7 @@ impl Users { &self, user_id: &UserId, device_id: &DeviceId, - device_keys: &IncomingDeviceKeys, + device_keys: &DeviceKeys, rooms: &super::rooms::Rooms, globals: &super::globals::Globals, ) -> Result<()> { @@ -631,7 +631,7 @@ impl Users { &self, user_id: &UserId, device_id: &DeviceId, - ) -> Result> { + ) -> Result> { let mut key = user_id.to_string().as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(device_id.as_bytes()); diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 8da3e17b..8dfd2080 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -1,7 +1,8 @@ use crate::Error; use ruma::{ - api::{Outgoing, OutgoingRequest}, + api::{AuthScheme, OutgoingRequest}, identifiers::{DeviceId, UserId}, + Outgoing, }; use std::{convert::TryFrom, convert::TryInto, ops::Deref}; @@ -61,29 +62,31 @@ where .await .expect("database was loaded"); - let (sender_user, sender_device) = if T::METADATA.requires_authentication { - // Get token from header or query value - let token = match request - .headers() - .get_one("Authorization") - .map(|s| s[7..].to_owned()) // Split off "Bearer " - .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok())) - { - // TODO: M_MISSING_TOKEN - None => return Failure((Status::Unauthorized, ())), - Some(token) => token, + let (sender_user, sender_device) = + // TODO: Do we need to matches! anything else here? ServerSignatures + if matches!(T::METADATA.authentication, AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken) { + // Get token from header or query value + let token = match request + .headers() + .get_one("Authorization") + .map(|s| s[7..].to_owned()) // Split off "Bearer " + .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok())) + { + // TODO: M_MISSING_TOKEN + None => return Failure((Status::Unauthorized, ())), + Some(token) => token, + }; + + // Check if token is valid + match db.users.find_from_token(&token).unwrap() { + // TODO: M_UNKNOWN_TOKEN + None => return Failure((Status::Unauthorized, ())), + Some((user_id, device_id)) => (Some(user_id), Some(device_id.into())), + } + } else { + (None, None) }; - // Check if token is valid - match db.users.find_from_token(&token).unwrap() { - // TODO: M_UNKNOWN_TOKEN - None => return Failure((Status::Unauthorized, ())), - Some((user_id, device_id)) => (Some(user_id), Some(device_id.into())), - } - } else { - (None, None) - }; - let mut http_request = http::Request::builder() .uri(request.uri().to_string()) .method(&*request.method().to_string()); diff --git a/src/server_server.rs b/src/server_server.rs index d8d0fa52..5b70780c 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -17,11 +17,11 @@ use ruma::{ OutgoingRequest, }, directory::{IncomingFilter, IncomingRoomNetwork}, - EventId, ServerName, + EventId, RoomVersionId, ServerName, }; use std::{ collections::BTreeMap, - convert::TryFrom, + convert::{TryFrom, TryInto}, fmt::Debug, time::{Duration, SystemTime}, }; @@ -95,7 +95,7 @@ where let mut http_request = request .try_into_http_request(&actual_destination, Some("")) .map_err(|e| { - warn!("{}: {}", actual_destination, e); + warn!("failed to find destination {}: {}", actual_destination, e); Error::BadServerResponse("Invalid destination") })?; @@ -122,7 +122,9 @@ where request_map.insert("origin".to_owned(), globals.server_name().as_str().into()); request_map.insert("destination".to_owned(), destination.as_str().into()); - let mut request_json = request_map.into(); + let mut request_json = + serde_json::from_value(request_map.into()).expect("valid JSON is valid BTreeMap"); + ruma::signatures::sign_json( globals.server_name().as_str(), globals.keypair(), @@ -130,6 +132,9 @@ where ) .expect("our request json is what ruma expects"); + let request_json: serde_json::Map = + serde_json::from_slice(&serde_json::to_vec(&request_json).unwrap()).unwrap(); + let signatures = request_json["signatures"] .as_object() .unwrap() @@ -234,7 +239,9 @@ pub fn get_server_keys(db: State<'_, Database>) -> Json { let mut verify_keys = BTreeMap::new(); verify_keys.insert( - format!("ed25519:{}", db.globals.keypair().version()), + format!("ed25519:{}", db.globals.keypair().version()) + .try_into() + .expect("DB stores valid ServerKeyId's"), VerifyKey { key: base64::encode_config(db.globals.keypair().public_key(), base64::STANDARD_NO_PAD), }, @@ -259,7 +266,7 @@ pub fn get_server_keys(db: State<'_, Database>) -> Json { &mut response, ) .unwrap(); - Json(response.to_string()) + Json(ruma::serde::to_canonical_json_string(&response).expect("JSON is canonical")) } #[cfg_attr(feature = "conduit_bin", get("/_matrix/key/v2/server/<_>"))] @@ -365,7 +372,7 @@ pub async fn get_public_rooms_route( feature = "conduit_bin", put("/_matrix/federation/v1/send/<_>", data = "") )] -pub fn send_transaction_message_route<'a>( +pub async fn send_transaction_message_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { @@ -451,7 +458,7 @@ pub fn get_missing_events_route<'a>( ) .map_err(|_| Error::bad_database("Invalid prev_events content in pdu in db."))?, ); - events.push(PduEvent::convert_to_outgoing_federation_event(pdu)); + events.push(serde_json::from_value(pdu).expect("Raw<..> is always valid")); } i += 1; } From db8a0c5d69ada292879a8356e10eb96c6f961c8f Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 8 Nov 2020 14:44:02 -0500 Subject: [PATCH 02/36] Add closest_parent method to Rooms Db insert in order /send pdus --- Cargo.lock | 1 + Cargo.toml | 8 +- src/database/rooms.rs | 57 +++++++- src/server_server.rs | 296 ++++++++++++++++++++++++++++++++++++++---- 4 files changed, 334 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 461972bb..3f8703e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2258,6 +2258,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" +source = "git+https://github.com/ruma/state-res?branch=timo-spec-comp#1cd1a16cdefabb126a781a50b3d5eb1fdb3d3afb" dependencies = [ "itertools", "js_int", diff --git a/Cargo.toml b/Cargo.toml index a6f89b90..b72b92f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ ruma = { git = "https://github.com/DevinR528/ruma", features = ["rand", "client- # Used when doing state resolution # state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] } -state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec"] } +state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec", "gen-eventid"] } # state-res = { path = "../../state-res", features = ["unstable-pre-spec"] } # Used for long polling @@ -78,6 +78,6 @@ path = "src/lib.rs" # ruma = { path = "../ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"] } # # ruma = { git = "https://github.com/ruma/ruma", rev = "64b9c646d15a359d62ab464a95176ff94adb2554", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"]} -[patch."https://github.com/ruma/state-res"] -state-res = { path = "../../state-res", features = ["unstable-pre-spec"] } -# state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp" } \ No newline at end of file +# [patch."https://github.com/ruma/state-res"] +# state-res = { path = "../../state-res", features = ["unstable-pre-spec", "gen-eventid"] } +# # state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp" } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index d9238a9d..28e1f608 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -35,6 +35,11 @@ use super::admin::AdminCommand; /// hashing the entire state. pub type StateHashId = IVec; +pub enum ClosestParent { + Append, + Insert(u64), +} + #[derive(Clone)] pub struct Rooms { pub edus: edus::RoomEdus, @@ -74,7 +79,10 @@ impl StateStore for Rooms { .get_pdu_id(event_id) .map_err(StateError::custom)? .ok_or_else(|| { - StateError::NotFound("PDU via room_id and event_id not found in the db.".into()) + StateError::NotFound(format!( + "PDU via room_id and event_id not found in the db.\n{}", + event_id.as_str() + )) })?; serde_json::from_slice( @@ -395,6 +403,47 @@ impl Rooms { } } + pub fn get_closest_parent( + &self, + incoming_prev_ids: &[EventId], + their_state: &BTreeMap>, + ) -> Result> { + match self.pduid_pdu.last()? { + Some(val) + if incoming_prev_ids.contains( + &serde_json::from_slice::(&val.1) + .map_err(|_| { + Error::bad_database("last DB entry contains invalid PDU bytes") + })? + .event_id, + ) => + { + Ok(Some(ClosestParent::Append)) + } + _ => { + let mut prev_ids = incoming_prev_ids.to_vec(); + while let Some(id) = prev_ids.pop() { + match self.get_pdu_id(&id)? { + Some(pdu_id) => { + return Ok(Some(ClosestParent::Insert(self.pdu_count(&pdu_id)?))); + } + None => { + prev_ids.extend(their_state.get(&id).map_or( + Err(Error::BadServerResponse( + "Failed to find previous event for PDU in state", + )), + // `prev_event_ids` will return an empty Vec instead of failing + // so it works perfect for our use here + |pdu| Ok(pdu.prev_event_ids()), + )?); + } + } + } + Ok(None) + } + } + } + /// Returns the leaf pdus of a room. pub fn get_pdu_leaves(&self, room_id: &RoomId) -> Result> { let mut prefix = room_id.as_bytes().to_vec(); @@ -438,6 +487,9 @@ impl Rooms { #[allow(clippy::too_many_arguments)] /// Creates a new persisted data unit and adds it to a room. + /// + /// By this point the incoming event should be fully authenticated, no auth happens + /// in `append_pdu`. pub fn append_pdu( &self, pdu: &PduEvent, @@ -554,6 +606,7 @@ impl Rooms { self.stateid_pduid .scan_prefix(&prefix) .filter_map(|pdu| pdu.map_err(|e| error!("{}", e)).ok()) + // Chop the old state_hash out leaving behind the (EventType, StateKey) .map(|(k, v)| (k.subslice(prefix.len(), k.len() - prefix.len()), v)) .collect::>() } else { @@ -851,7 +904,7 @@ impl Rooms { .filter_map(|r| r.ok()) .filter(|server| &**server != globals.server_name()) { - sending.send_pdu(server, &pdu_id)?; + sending.send_pdu(&server, &pdu_id)?; } Ok(pdu.event_id) diff --git a/src/server_server.rs b/src/server_server.rs index 5b70780c..41520fbd 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,4 +1,7 @@ -use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; +use crate::{ + client_server, database::rooms::ClosestParent, utils, ConduitResult, Database, Error, PduEvent, + Result, Ruma, +}; use get_profile_information::v1::ProfileField; use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::warn; @@ -381,44 +384,266 @@ pub async fn send_transaction_message_route<'a>( } //dbg!(&*body); + // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we? + // SPEC: + // Servers MUST strictly enforce the JSON format specified in the appendices. + // This translates to a 400 M_BAD_JSON error on most endpoints, or discarding of + // events over federation. For example, the Federation API's /send endpoint would + // discard the event whereas the Client Server API's /send/{eventType} endpoint + // would return a M_BAD_JSON error. + let mut resolved_map = BTreeMap::new(); for pdu in &body.pdus { - let mut value = serde_json::from_str(pdu.json().get()) - .expect("converting raw jsons to values always works"); - - let event_id = EventId::try_from(&*format!( - "${}", - ruma::signatures::reference_hash(&value).expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); - - value - .as_object_mut() - .expect("ruma pdus are json objects") - .insert("event_id".to_owned(), event_id.to_string().into()); - + println!("LOOP"); + let (event_id, value) = process_incoming_pdu(pdu); let pdu = serde_json::from_value::(value.clone()) .expect("all ruma pdus are conduit pdus"); - if db.rooms.exists(&pdu.room_id)? { + let room_id = &pdu.room_id; + + if value.get("state_key").is_none() { + if !db.rooms.is_joined(&pdu.sender, &pdu.room_id)? { + // TODO: auth rules apply to all events, not only those with a state key + log::error!("Unauthorized {}", pdu.kind); + return Err(Error::BadRequest( + ruma::api::client::error::ErrorKind::Forbidden, + "Event is not authorized", + )); + } + + // TODO: We should be doing the same get_closest_parent thing here too? + // same as for state events ~100 lines down let count = db.globals.next_count()?; + let mut pdu_id = pdu.room_id.as_bytes().to_vec(); pdu_id.push(0xff); pdu_id.extend_from_slice(&count.to_be_bytes()); - db.rooms.append_to_state(&pdu_id, &pdu)?; db.rooms.append_pdu( &pdu, &value, count, - pdu_id.clone().into(), + pdu_id.into(), &db.globals, &db.account_data, &db.admin, )?; + + resolved_map.insert(event_id, Ok::<(), String>(())); + continue; } + + let now = std::time::Instant::now(); + let get_state_response = match send_request( + &db.globals, + body.body.origin.clone(), + ruma::api::federation::event::get_room_state::v1::Request { + room_id, + event_id: &event_id, + }, + ) + .await + { + Ok(res) => res, + // We can't hard fail because there are some valid errors, just + // keep checking PDU's + // + // As an example a possible error + // {"errcode":"M_FORBIDDEN","error":"Host not in room."} + Err(err) => { + log::error!("Request failed: {}", err); + resolved_map.insert(event_id, Err(err.to_string())); + dbg!(now.elapsed()); + continue; + } + }; + dbg!(now.elapsed()); + + let their_current_state = get_state_response + .pdus + .iter() + .chain(get_state_response.auth_chain.iter()) // add auth events + .map(|pdu| { + let (event_id, json) = process_incoming_pdu(pdu); + ( + event_id.clone(), + std::sync::Arc::new( + // When creating a StateEvent the event_id arg will be used + // over any found in the json and it will not use ruma::reference_hash + // to generate one + state_res::StateEvent::from_id_value(event_id, json) + .expect("valid pdu json"), + ), + ) + }) + .collect::>(); + + if value.get("state_key").is_none() { + if !db.rooms.is_joined(&pdu.sender, &pdu.room_id)? { + // TODO: auth rules apply to all events, not only those with a state key + log::error!("Sender is not joined {}", pdu.kind); + + resolved_map.insert(event_id, Err("User is not in this room".into())); + continue; + } + + // // TODO: We should be doing the same get_closest_parent thing here too? + // // same as for state events ~100 lines down + // let count = db.globals.next_count()?; + // let mut pdu_id = pdu.room_id.as_bytes().to_vec(); + // pdu_id.push(0xff); + // pdu_id.extend_from_slice(&count.to_be_bytes()); + // db.rooms.append_pdu( + // &pdu, + // &value, + // count, + // pdu_id.into(), + // &db.globals, + // &db.account_data, + // &db.sending, + // )?; + + // If the event is older than the last event in pduid_pdu Tree then find the + // closest ancestor we know of and insert after the known ancestor by + // altering the known events pduid to = same roomID + same count bytes + 0x1 + // pushing a single byte every time a simple append cannot be done. + match db + .rooms + .get_closest_parent(&pdu.prev_events, &their_current_state)? + { + Some(ClosestParent::Append) => { + let count = db.globals.next_count()?; + dbg!(&count); + let mut pdu_id = room_id.as_bytes().to_vec(); + pdu_id.push(0xff); + pdu_id.extend_from_slice(&count.to_be_bytes()); + + db.rooms.append_pdu( + &pdu, + &value, + count, + pdu_id.into(), + &db.globals, + &db.account_data, + &db.sending, + )?; + } + Some(ClosestParent::Insert(old_count)) => { + let count = old_count; + let mut pdu_id = room_id.as_bytes().to_vec(); + pdu_id.push(0xff); + pdu_id.extend_from_slice(&count.to_be_bytes()); + // Create a new count that is after old_count but before + // the pdu appended after + pdu_id.push(1); + + db.rooms.append_pdu( + &pdu, + &value, + count, + pdu_id.into(), + &db.globals, + &db.account_data, + &db.sending, + )?; + } + _ => panic!("Not a sequential event or no parents found"), + }; + resolved_map.insert(event_id, Ok::<(), String>(())); + continue; + } + + let our_current_state = db.rooms.room_state_full(room_id)?; + match state_res::StateResolution::resolve( + room_id, + &ruma::RoomVersionId::Version6, + &[ + our_current_state + .iter() + .map(|((ev, sk), v)| ((ev.clone(), sk.to_owned()), v.event_id.clone())) + .collect::>(), + // TODO we may not want the auth events chained in here for resolution? + their_current_state + .iter() + .map(|(_id, v)| ((v.kind(), v.state_key()), v.event_id().clone())) + .collect::>(), + ], + Some( + our_current_state + .iter() + .map(|(_k, v)| (v.event_id.clone(), v.convert_for_state_res())) + .chain( + their_current_state + .iter() + .map(|(id, ev)| (id.clone(), ev.clone())), + ) + .collect::>(), + ), + &db.rooms, + ) { + Ok(resolved) if resolved.values().any(|id| &event_id == id) => { + // If the event is older than the last event in pduid_pdu Tree then find the + // closest ancestor we know of and insert after the known ancestor by + // altering the known events pduid to = same roomID + same count bytes + 0x1 + // pushing a single byte every time a simple append cannot be done. + match db + .rooms + .get_closest_parent(&pdu.prev_events, &their_current_state)? + { + Some(ClosestParent::Append) => { + let count = db.globals.next_count()?; + dbg!(&count); + let mut pdu_id = room_id.as_bytes().to_vec(); + pdu_id.push(0xff); + pdu_id.extend_from_slice(&count.to_be_bytes()); + + db.rooms.append_pdu( + &pdu, + &value, + count, + pdu_id.into(), + &db.globals, + &db.account_data, + &db.sending, + )?; + } + Some(ClosestParent::Insert(old_count)) => { + println!("INSERT PDU FOUND {}", old_count); + let count = old_count; + let mut pdu_id = room_id.as_bytes().to_vec(); + pdu_id.push(0xff); + pdu_id.extend_from_slice(&count.to_be_bytes()); + // Create a new count that is after old_count but before + // the pdu appended after + pdu_id.push(1); + + db.rooms.append_pdu( + &pdu, + &value, + count, + pdu_id.into(), + &db.globals, + &db.account_data, + &db.sending, + )?; + } + _ => panic!("Not a sequential event or no parents found"), + } + + resolved_map.insert(event_id, Ok::<(), String>(())); + } + // If the eventId is not found in the resolved state auth has failed + Ok(_) => { + // TODO have state_res give the actual auth error in this case + resolved_map.insert( + event_id, + Err("This event failed authentication, not found in resolved set".into()), + ); + } + Err(e) => { + resolved_map.insert(event_id, Err(e.to_string())); + } + }; } - Ok(send_transaction_message::v1::Response { - pdus: BTreeMap::new(), - } - .into()) + + Ok(send_transaction_message::v1::Response { pdus: resolved_map }.into()) } #[cfg_attr( @@ -529,3 +754,30 @@ pub fn get_user_devices_route<'a>( .into()) } */ + +/// Generates a correct eventId for the incoming pdu. +/// +/// Returns a `state_res::StateEvent` which can be converted freely and has accessor methods. +fn process_incoming_pdu(pdu: &ruma::Raw) -> (EventId, serde_json::Value) { + let mut value = serde_json::from_str(pdu.json().get()) + .expect("converting raw jsons to values always works"); + + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&value, &RoomVersionId::Version6) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + value.insert( + "event_id".to_owned(), + serde_json::json!(event_id) + .try_into() + .expect("EventId is a valid CanonicalJsonValue"), + ); + + ( + event_id, + serde_json::to_value(value).expect("JSON Value is a CanonicalJsonValue"), + ) +} From 0d69ebdc6c7259a544dc83958eaba1d50312e128 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 8 Nov 2020 14:45:52 -0500 Subject: [PATCH 03/36] Reformat imports and fix clippy warnings --- src/client_server/account.rs | 11 ++++------- src/client_server/directory.rs | 4 +--- src/client_server/membership.rs | 7 ++----- src/client_server/state.rs | 8 +++++--- src/database/sending.rs | 5 +++-- src/ruma_wrapper.rs | 5 ++++- src/utils.rs | 16 +++++++++++++++- 7 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/client_server/account.rs b/src/client_server/account.rs index f48543e2..ab90de5b 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -15,13 +15,10 @@ use ruma::{ }, }, events::{ - room::canonical_alias, - room::guest_access, - room::history_visibility, - room::join_rules, - room::member, - room::name, - room::{message, topic}, + room::{ + canonical_alias, guest_access, history_visibility, join_rules, member, message, name, + topic, + }, EventType, }, RoomAliasId, RoomId, RoomVersionId, UserId, diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 048b4100..d8af2e3f 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -15,9 +15,7 @@ use ruma::{ }, federation, }, - directory::Filter, - directory::RoomNetwork, - directory::{IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk}, + directory::{Filter, IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk, RoomNetwork}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 50a8cca0..0e6235f4 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -17,15 +17,12 @@ use ruma::{ }, federation, }, - events::pdu::Pdu, - events::{room::member, EventType}, + events::{pdu::Pdu, room::member, EventType}, EventId, Raw, RoomId, RoomVersionId, ServerName, UserId, }; use state_res::StateEvent; use std::{ - collections::BTreeMap, - collections::HashMap, - collections::HashSet, + collections::{BTreeMap, HashMap, HashSet}, convert::{TryFrom, TryInto}, iter, sync::Arc, diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 37778623..010b20d3 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -9,9 +9,8 @@ use ruma::{ }, }, events::{ - room::history_visibility::HistoryVisibility, - room::history_visibility::HistoryVisibilityEventContent, AnyStateEventContent, - EventContent, EventType, + room::history_visibility::{HistoryVisibility, HistoryVisibilityEventContent}, + AnyStateEventContent, EventContent, EventType, }, EventId, RoomId, UserId, }; @@ -103,6 +102,7 @@ pub async fn get_state_events_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + #[allow(clippy::blocks_in_if_conditions)] // Users not in the room should not be able to access the state unless history_visibility is // WorldReadable if !db.rooms.is_joined(sender_user, &body.room_id)? @@ -148,6 +148,7 @@ pub async fn get_state_events_for_key_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + #[allow(clippy::blocks_in_if_conditions)] // Users not in the room should not be able to access the state unless history_visibility is // WorldReadable if !db.rooms.is_joined(sender_user, &body.room_id)? @@ -198,6 +199,7 @@ pub async fn get_state_events_for_empty_key_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + #[allow(clippy::blocks_in_if_conditions)] // Users not in the room should not be able to access the state unless history_visibility is // WorldReadable if !db.rooms.is_joined(sender_user, &body.room_id)? diff --git a/src/database/sending.rs b/src/database/sending.rs index e3fca4f0..c108e7ee 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -116,6 +116,7 @@ impl Sending { } } Err((_server, _e)) => { + log::error!("server: {}\nerror: {}", _server, _e) // TODO: exponential backoff } }; @@ -131,7 +132,7 @@ impl Sending { .expect("splitn will always return 1 or more elements"), ) .map_err(|_| Error::bad_database("ServerName in servernamepduid bytes are invalid.")) - .and_then(|server_str|Box::::try_from(server_str) + .and_then(|server_str| Box::::try_from(server_str) .map_err(|_| Error::bad_database("ServerName in servernamepduid is invalid."))) .ok() .and_then(|server| parts @@ -162,7 +163,7 @@ impl Sending { }); } - pub fn send_pdu(&self, server: Box, pdu_id: &[u8]) -> Result<()> { + pub fn send_pdu(&self, server: &ServerName, pdu_id: &[u8]) -> Result<()> { let mut key = server.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(pdu_id); diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 8dfd2080..1c5529a3 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -4,7 +4,10 @@ use ruma::{ identifiers::{DeviceId, UserId}, Outgoing, }; -use std::{convert::TryFrom, convert::TryInto, ops::Deref}; +use std::{ + convert::{TryFrom, TryInto}, + ops::Deref, +}; #[cfg(feature = "conduit_bin")] use { diff --git a/src/utils.rs b/src/utils.rs index 452b7c5a..e65ec864 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -89,9 +89,23 @@ pub fn common_elements( } } } - false }) .all(|b| b) })) } + +#[test] +fn sled_tests() { + let db = sled::Config::new().temporary(true).open().unwrap(); + + db.insert(1_u64.to_be_bytes(), vec![10]).unwrap(); + db.insert(2_u64.to_be_bytes(), vec![20]).unwrap(); + db.insert(3_u64.to_be_bytes(), vec![30]).unwrap(); + + let mut key = 1_u64.to_be_bytes().to_vec(); + key.push(1); + db.insert(key, vec![40]).unwrap(); + + println!("{:?}", db.iter().collect::, _>>().unwrap()) +} From c9a6ce54cb9d54009779878b821e82e76e6e1b6a Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 8 Nov 2020 14:46:26 -0500 Subject: [PATCH 04/36] Add basic handling of EDUs for /send/txn --- src/server_server.rs | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/server_server.rs b/src/server_server.rs index 41520fbd..6550b8f0 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -383,7 +383,41 @@ pub async fn send_transaction_message_route<'a>( return Err(Error::bad_config("Federation is disabled.")); } - //dbg!(&*body); + for edu in &body.edus { + match serde_json::from_str::(edu.json().get()) { + Ok(edu) => match edu.edu_type.as_str() { + "m.typing" => { + if let Some(typing) = edu.content.get("typing") { + if typing.as_bool().unwrap_or_default() { + db.rooms.edus.typing_add( + &UserId::try_from(edu.content["user_id"].as_str().unwrap()) + .unwrap(), + &RoomId::try_from(edu.content["room_id"].as_str().unwrap()) + .unwrap(), + 3000 + utils::millis_since_unix_epoch(), + &db.globals, + )?; + } else { + db.rooms.edus.typing_remove( + &UserId::try_from(edu.content["user_id"].as_str().unwrap()) + .unwrap(), + &RoomId::try_from(edu.content["room_id"].as_str().unwrap()) + .unwrap(), + &db.globals, + )?; + } + } + } + "m.presence" => {} + "m.receipt" => {} + _ => {} + }, + Err(_err) => { + log::error!("{}", _err); + continue; + } + } + } // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we? // SPEC: // Servers MUST strictly enforce the JSON format specified in the appendices. From dcd1163806e8736dd758cdef77c862b8b823088b Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Tue, 3 Nov 2020 21:20:35 +0100 Subject: [PATCH 05/36] All outgoing pdus in Sending must be PduStubs --- src/database/sending.rs | 36 ++++++++++++++++++++++++------------ src/server_server.rs | 25 +++++++------------------ 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/database/sending.rs b/src/database/sending.rs index c108e7ee..14558e3c 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -73,7 +73,7 @@ impl Sending { loop { select! { Some(server) = futures.next() => { - debug!("response: {:?}", &server); + debug!("sending response: {:?}", &server); match server { Ok((server, _response)) => { let mut prefix = server.as_bytes().to_vec(); @@ -184,17 +184,29 @@ impl Sending { let pdu_jsons = pdu_ids .iter() .map(|pdu_id| { - Ok::<_, (Box, Error)>(PduEvent::convert_to_outgoing_federation_event( - rooms - .get_pdu_json_from_id(pdu_id) - .map_err(|e| (server.clone(), e))? - .ok_or_else(|| { - ( - server.clone(), - Error::bad_database("Event in servernamepduids not found in db."), - ) - })?, - )) + Ok::<_, (Box, Error)>( + // TODO: this was a PduStub + // In order for sending to work these actually do have to be + // PduStub but, since they are Raw<..> we can fake it. + serde_json::from_str( + PduEvent::convert_to_outgoing_federation_event( + rooms + .get_pdu_json_from_id(pdu_id) + .map_err(|e| (server.clone(), e))? + .ok_or_else(|| { + ( + server.clone(), + Error::bad_database( + "Event in servernamepduids not found in db.", + ), + ) + })?, + ) + .json() + .get(), + ) + .expect("Raw<..> is always valid"), + ) }) .filter_map(|r| r.ok()) .collect::>(); diff --git a/src/server_server.rs b/src/server_server.rs index 6550b8f0..de24c6d4 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -20,7 +20,7 @@ use ruma::{ OutgoingRequest, }, directory::{IncomingFilter, IncomingRoomNetwork}, - EventId, RoomVersionId, ServerName, + EventId, RoomId, RoomVersionId, ServerName, UserId, }; use std::{ collections::BTreeMap, @@ -412,19 +412,10 @@ pub async fn send_transaction_message_route<'a>( "m.receipt" => {} _ => {} }, - Err(_err) => { - log::error!("{}", _err); - continue; - } + Err(_err) => continue, } } - // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we? - // SPEC: - // Servers MUST strictly enforce the JSON format specified in the appendices. - // This translates to a 400 M_BAD_JSON error on most endpoints, or discarding of - // events over federation. For example, the Federation API's /send endpoint would - // discard the event whereas the Client Server API's /send/{eventType} endpoint - // would return a M_BAD_JSON error. + let mut resolved_map = BTreeMap::new(); for pdu in &body.pdus { println!("LOOP"); @@ -437,16 +428,14 @@ pub async fn send_transaction_message_route<'a>( if !db.rooms.is_joined(&pdu.sender, &pdu.room_id)? { // TODO: auth rules apply to all events, not only those with a state key log::error!("Unauthorized {}", pdu.kind); - return Err(Error::BadRequest( - ruma::api::client::error::ErrorKind::Forbidden, - "Event is not authorized", - )); + + resolved_map.insert(event_id, Err("User is not in this room".into())); + continue; } // TODO: We should be doing the same get_closest_parent thing here too? // same as for state events ~100 lines down let count = db.globals.next_count()?; - let mut pdu_id = pdu.room_id.as_bytes().to_vec(); pdu_id.push(0xff); pdu_id.extend_from_slice(&count.to_be_bytes()); @@ -677,7 +666,7 @@ pub async fn send_transaction_message_route<'a>( }; } - Ok(send_transaction_message::v1::Response { pdus: resolved_map }.into()) + Ok(dbg!(send_transaction_message::v1::Response { pdus: resolved_map }).into()) } #[cfg_attr( From eca0bbb35a9c062c21392c60455f624374c5cd17 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 8 Nov 2020 13:49:02 -0500 Subject: [PATCH 06/36] Fix federated join miss hashing the join event created --- src/server_server.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index de24c6d4..ccdeb749 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -412,10 +412,19 @@ pub async fn send_transaction_message_route<'a>( "m.receipt" => {} _ => {} }, - Err(_err) => continue, + Err(_err) => { + log::error!("{}", _err); + continue; + } } } - + // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we? + // SPEC: + // Servers MUST strictly enforce the JSON format specified in the appendices. + // This translates to a 400 M_BAD_JSON error on most endpoints, or discarding of + // events over federation. For example, the Federation API's /send endpoint would + // discard the event whereas the Client Server API's /send/{eventType} endpoint + // would return a M_BAD_JSON error. let mut resolved_map = BTreeMap::new(); for pdu in &body.pdus { println!("LOOP"); From b13049a6fac23a41f726d7f0173554e1a40394af Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 8 Nov 2020 13:49:02 -0500 Subject: [PATCH 07/36] Fix federated join miss hashing the join event created --- src/client_server/membership.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 0e6235f4..5d621fa5 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -512,22 +512,16 @@ async fn join_room_by_id_helper( // TODO fixup CanonicalJsonValue // use that instead of serde_json::Map... maybe? - let mut canon_json_stub = + let mut canon_json_stub: BTreeMap<_, ruma::signatures::CanonicalJsonValue> = serde_json::from_value(join_event_stub_value).expect("json Value is canonical JSON"); - // Generate event id - let event_id = EventId::try_from(&*format!( - "${}", - ruma::signatures::reference_hash(&canon_json_stub, &RoomVersionId::Version6) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); // We don't leave the event id into the pdu because that's only allowed in v1 or v2 rooms // let join_event_stub = join_event_stub_value.as_object_mut().unwrap(); // join_event_stub.remove("event_id"); - canon_json_stub.remove("event_id"); + // In order to create a compatible ref hash (EventID) the `hashes` field needs to be present + // who the hell knew... ruma::signatures::hash_and_sign_event( db.globals.server_name().as_str(), db.globals.keypair(), @@ -536,6 +530,14 @@ async fn join_room_by_id_helper( ) .expect("event is valid, we just created it"); + // Generate event id + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&canon_json_stub, &RoomVersionId::Version6) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + // Add event_id back canon_json_stub.insert( "event_id".to_owned(), @@ -545,7 +547,7 @@ async fn join_room_by_id_helper( ); // It has enough fields to be called a proper event now - let join_event = canon_json_stub; + let join_event = dbg!(canon_json_stub); let send_join_response = server_server::send_request( &db.globals, @@ -606,7 +608,7 @@ async fn join_room_by_id_helper( .map(|ev| (event_id, Arc::new(ev))) .map_err(|e| { warn!("{}: {}", value, e); - Error::BadServerResponse("Invalid PDU bytes in send_join response.") + Error::BadServerResponse("Invalid PDU in send_join response.") }) }) .collect::>>>()?; From acd144e93413dd509e2d75bdb381243d5d8187b8 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 8 Nov 2020 13:54:59 -0500 Subject: [PATCH 08/36] Fix get_closest_parent and cleanup federation/send/:txn --- Cargo.lock | 286 +++++--------------------------- Cargo.toml | 12 +- src/client_server/membership.rs | 4 +- src/database/rooms.rs | 5 +- src/server_server.rs | 81 +++------ src/utils.rs | 15 -- 6 files changed, 69 insertions(+), 334 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f8703e0..d2eef800 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -221,7 +221,7 @@ dependencies = [ "reqwest", "ring", "rocket", - "ruma 0.0.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma", "rust-argon2", "serde", "serde_json", @@ -1611,33 +1611,15 @@ source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a dependencies = [ "assign", "js_int", - "ruma-api 0.17.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-appservice-api 0.2.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-client-api 0.10.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-events 0.22.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-federation-api 0.0.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-signatures 0.6.0-dev.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", -] - -[[package]] -name = "ruma" -version = "0.0.1" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "assign", - "js_int", - "ruma-api 0.17.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-appservice-api 0.2.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-client-api 0.10.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-events 0.22.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-federation-api 0.0.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-signatures 0.6.0-dev.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-api", + "ruma-appservice-api", + "ruma-client-api", + "ruma-common", + "ruma-events", + "ruma-federation-api", + "ruma-identifiers", + "ruma-serde", + "ruma-signatures", ] [[package]] @@ -1647,27 +1629,10 @@ source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a dependencies = [ "http", "percent-encoding", - "ruma-api-macros 0.17.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "serde", - "serde_json", - "strum", - "thiserror", -] - -[[package]] -name = "ruma-api" -version = "0.17.0-alpha.1" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "http", - "percent-encoding", - "ruma-api-macros 0.17.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-api-macros", + "ruma-common", + "ruma-identifiers", + "ruma-serde", "serde", "serde_json", "strum", @@ -1685,39 +1650,15 @@ dependencies = [ "syn", ] -[[package]] -name = "ruma-api-macros" -version = "0.17.0-alpha.1" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" dependencies = [ - "ruma-api 0.17.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-events 0.22.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "serde", - "serde_json", -] - -[[package]] -name = "ruma-appservice-api" -version = "0.2.0-alpha.1" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "ruma-api 0.17.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-events 0.22.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-api", + "ruma-common", + "ruma-events", + "ruma-identifiers", "serde", "serde_json", ] @@ -1732,31 +1673,11 @@ dependencies = [ "js_int", "maplit", "percent-encoding", - "ruma-api 0.17.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-events 0.22.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "serde", - "serde_json", - "strum", -] - -[[package]] -name = "ruma-client-api" -version = "0.10.0-alpha.1" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "assign", - "http", - "js_int", - "maplit", - "percent-encoding", - "ruma-api 0.17.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-events 0.22.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-api", + "ruma-common", + "ruma-events", + "ruma-identifiers", + "ruma-serde", "serde", "serde_json", "strum", @@ -1768,23 +1689,9 @@ version = "0.2.0" source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" dependencies = [ "js_int", - "ruma-common-macros 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "serde", - "serde_json", - "strum", -] - -[[package]] -name = "ruma-common" -version = "0.2.0" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "js_int", - "ruma-common-macros 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-common-macros", + "ruma-identifiers", + "ruma-serde", "serde", "serde_json", "strum", @@ -1801,42 +1708,16 @@ dependencies = [ "syn", ] -[[package]] -name = "ruma-common-macros" -version = "0.2.0" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "ruma-events" version = "0.22.0-alpha.1" source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" dependencies = [ "js_int", - "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-events-macros 0.22.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "serde", - "serde_json", - "strum", -] - -[[package]] -name = "ruma-events" -version = "0.22.0-alpha.1" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "js_int", - "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-events-macros 0.22.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-common", + "ruma-events-macros", + "ruma-identifiers", + "ruma-serde", "serde", "serde_json", "strum", @@ -1853,43 +1734,17 @@ dependencies = [ "syn", ] -[[package]] -name = "ruma-events-macros" -version = "0.22.0-alpha.1" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "ruma-federation-api" version = "0.0.3" source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" dependencies = [ "js_int", - "ruma-api 0.17.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-common 0.2.0 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-events 0.22.0-alpha.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "serde", - "serde_json", -] - -[[package]] -name = "ruma-federation-api" -version = "0.0.3" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "js_int", - "ruma-api 0.17.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-common 0.2.0 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-events 0.22.0-alpha.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-api", + "ruma-common", + "ruma-events", + "ruma-identifiers", + "ruma-serde", "serde", "serde_json", ] @@ -1898,23 +1753,11 @@ dependencies = [ name = "ruma-identifiers" version = "0.17.4" source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" -dependencies = [ - "ruma-identifiers-macros 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-identifiers-validation 0.1.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "serde", - "strum", -] - -[[package]] -name = "ruma-identifiers" -version = "0.17.4" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" dependencies = [ "rand", - "ruma-identifiers-macros 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-identifiers-validation 0.1.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-identifiers-macros", + "ruma-identifiers-validation", + "ruma-serde", "serde", "strum", ] @@ -1926,18 +1769,7 @@ source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a dependencies = [ "proc-macro2", "quote", - "ruma-identifiers-validation 0.1.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "syn", -] - -[[package]] -name = "ruma-identifiers-macros" -version = "0.17.4" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "proc-macro2", - "quote", - "ruma-identifiers-validation 0.1.1 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-identifiers-validation", "syn", ] @@ -1950,15 +1782,6 @@ dependencies = [ "strum", ] -[[package]] -name = "ruma-identifiers-validation" -version = "0.1.1" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "serde", - "strum", -] - [[package]] name = "ruma-serde" version = "0.2.3" @@ -1971,18 +1794,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "ruma-serde" -version = "0.2.3" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "form_urlencoded", - "itoa", - "js_int", - "serde", - "serde_json", -] - [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" @@ -1990,21 +1801,8 @@ source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a dependencies = [ "base64", "ring", - "ruma-identifiers 0.17.4 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "ruma-serde 0.2.3 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", - "serde_json", - "untrusted", -] - -[[package]] -name = "ruma-signatures" -version = "0.6.0-dev.1" -source = "git+https://github.com/DevinR528/ruma?branch=unstable-join#424b138d84ccc47c0b212708a54f66bf88f7d57a" -dependencies = [ - "base64", - "ring", - "ruma-identifiers 0.17.4 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", - "ruma-serde 0.2.3 (git+https://github.com/DevinR528/ruma?branch=unstable-join)", + "ruma-identifiers", + "ruma-serde", "serde_json", "untrusted", ] @@ -2263,7 +2061,7 @@ dependencies = [ "itertools", "js_int", "maplit", - "ruma 0.0.1 (git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889)", + "ruma", "serde", "serde_json", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index b72b92f8..e7b87fa1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ edition = "2018" rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] } # Used for matrix spec type definitions and helpers -#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } -ruma = { git = "https://github.com/DevinR528/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "unstable-join" } +ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks", "unstable-exhaustive-types"], rev = "c15382ca41262058302959eac4029ab4a1ea5889" } +# ruma = { git = "https://github.com/DevinR528/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "unstable-join" } # ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } # Used when doing state resolution @@ -73,11 +73,3 @@ required-features = ["conduit_bin"] [lib] name = "conduit" path = "src/lib.rs" - -# [patch."https://github.com/timokoesters/ruma"] -# ruma = { path = "../ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"] } -# # ruma = { git = "https://github.com/ruma/ruma", rev = "64b9c646d15a359d62ab464a95176ff94adb2554", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"]} - -# [patch."https://github.com/ruma/state-res"] -# state-res = { path = "../../state-res", features = ["unstable-pre-spec", "gen-eventid"] } -# # state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp" } diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 5d621fa5..a07fe727 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -515,9 +515,7 @@ async fn join_room_by_id_helper( let mut canon_json_stub: BTreeMap<_, ruma::signatures::CanonicalJsonValue> = serde_json::from_value(join_event_stub_value).expect("json Value is canonical JSON"); - // We don't leave the event id into the pdu because that's only allowed in v1 or v2 rooms - // let join_event_stub = join_event_stub_value.as_object_mut().unwrap(); - // join_event_stub.remove("event_id"); + // We don't leave the event id in the pdu because that's only allowed in v1 or v2 rooms canon_json_stub.remove("event_id"); // In order to create a compatible ref hash (EventID) the `hashes` field needs to be present diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 28e1f608..08734b1b 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -405,11 +405,12 @@ impl Rooms { pub fn get_closest_parent( &self, + room: &RoomId, incoming_prev_ids: &[EventId], their_state: &BTreeMap>, ) -> Result> { - match self.pduid_pdu.last()? { - Some(val) + match self.pduid_pdu.scan_prefix(room.as_bytes()).last() { + Some(Ok(val)) if incoming_prev_ids.contains( &serde_json::from_slice::(&val.1) .map_err(|_| { diff --git a/src/server_server.rs b/src/server_server.rs index ccdeb749..c865b028 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -418,6 +418,7 @@ pub async fn send_transaction_message_route<'a>( } } } + // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we? // SPEC: // Servers MUST strictly enforce the JSON format specified in the appendices. @@ -427,42 +428,20 @@ pub async fn send_transaction_message_route<'a>( // would return a M_BAD_JSON error. let mut resolved_map = BTreeMap::new(); for pdu in &body.pdus { - println!("LOOP"); let (event_id, value) = process_incoming_pdu(pdu); let pdu = serde_json::from_value::(value.clone()) .expect("all ruma pdus are conduit pdus"); let room_id = &pdu.room_id; - if value.get("state_key").is_none() { - if !db.rooms.is_joined(&pdu.sender, &pdu.room_id)? { - // TODO: auth rules apply to all events, not only those with a state key - log::error!("Unauthorized {}", pdu.kind); - - resolved_map.insert(event_id, Err("User is not in this room".into())); - continue; - } - - // TODO: We should be doing the same get_closest_parent thing here too? - // same as for state events ~100 lines down - let count = db.globals.next_count()?; - let mut pdu_id = pdu.room_id.as_bytes().to_vec(); - pdu_id.push(0xff); - pdu_id.extend_from_slice(&count.to_be_bytes()); - db.rooms.append_pdu( - &pdu, - &value, - count, - pdu_id.into(), - &db.globals, - &db.account_data, - &db.admin, - )?; - - resolved_map.insert(event_id, Ok::<(), String>(())); + // If we have no idea about this room + // TODO: Does a server only send us events that we should know about or + // when everyone on this server leaves a room can we ignore those events? + if !db.rooms.exists(&pdu.room_id)? { + log::error!("Room does not exist on this server"); + resolved_map.insert(event_id, Err("Room is unknown to this server".into())); continue; } - let now = std::time::Instant::now(); let get_state_response = match send_request( &db.globals, body.body.origin.clone(), @@ -482,11 +461,9 @@ pub async fn send_transaction_message_route<'a>( Err(err) => { log::error!("Request failed: {}", err); resolved_map.insert(event_id, Err(err.to_string())); - dbg!(now.elapsed()); continue; } }; - dbg!(now.elapsed()); let their_current_state = get_state_response .pdus @@ -509,40 +486,21 @@ pub async fn send_transaction_message_route<'a>( if value.get("state_key").is_none() { if !db.rooms.is_joined(&pdu.sender, &pdu.room_id)? { - // TODO: auth rules apply to all events, not only those with a state key log::error!("Sender is not joined {}", pdu.kind); - resolved_map.insert(event_id, Err("User is not in this room".into())); continue; } - // // TODO: We should be doing the same get_closest_parent thing here too? - // // same as for state events ~100 lines down - // let count = db.globals.next_count()?; - // let mut pdu_id = pdu.room_id.as_bytes().to_vec(); - // pdu_id.push(0xff); - // pdu_id.extend_from_slice(&count.to_be_bytes()); - // db.rooms.append_pdu( - // &pdu, - // &value, - // count, - // pdu_id.into(), - // &db.globals, - // &db.account_data, - // &db.sending, - // )?; - // If the event is older than the last event in pduid_pdu Tree then find the // closest ancestor we know of and insert after the known ancestor by // altering the known events pduid to = same roomID + same count bytes + 0x1 // pushing a single byte every time a simple append cannot be done. match db .rooms - .get_closest_parent(&pdu.prev_events, &their_current_state)? + .get_closest_parent(room_id, &pdu.prev_events, &their_current_state)? { Some(ClosestParent::Append) => { let count = db.globals.next_count()?; - dbg!(&count); let mut pdu_id = room_id.as_bytes().to_vec(); pdu_id.push(0xff); pdu_id.extend_from_slice(&count.to_be_bytes()); @@ -554,10 +512,12 @@ pub async fn send_transaction_message_route<'a>( pdu_id.into(), &db.globals, &db.account_data, - &db.sending, + &db.admin, )?; } Some(ClosestParent::Insert(old_count)) => { + println!("INSERT PDU FOUND {}", old_count); + let count = old_count; let mut pdu_id = room_id.as_bytes().to_vec(); pdu_id.push(0xff); @@ -573,7 +533,7 @@ pub async fn send_transaction_message_route<'a>( pdu_id.into(), &db.globals, &db.account_data, - &db.sending, + &db.admin, )?; } _ => panic!("Not a sequential event or no parents found"), @@ -615,13 +575,13 @@ pub async fn send_transaction_message_route<'a>( // closest ancestor we know of and insert after the known ancestor by // altering the known events pduid to = same roomID + same count bytes + 0x1 // pushing a single byte every time a simple append cannot be done. - match db - .rooms - .get_closest_parent(&pdu.prev_events, &their_current_state)? - { + match db.rooms.get_closest_parent( + room_id, + &pdu.prev_events, + &their_current_state, + )? { Some(ClosestParent::Append) => { let count = db.globals.next_count()?; - dbg!(&count); let mut pdu_id = room_id.as_bytes().to_vec(); pdu_id.push(0xff); pdu_id.extend_from_slice(&count.to_be_bytes()); @@ -633,11 +593,12 @@ pub async fn send_transaction_message_route<'a>( pdu_id.into(), &db.globals, &db.account_data, - &db.sending, + &db.admin, )?; } Some(ClosestParent::Insert(old_count)) => { - println!("INSERT PDU FOUND {}", old_count); + println!("INSERT STATE PDU FOUND {}", old_count); + let count = old_count; let mut pdu_id = room_id.as_bytes().to_vec(); pdu_id.push(0xff); @@ -653,7 +614,7 @@ pub async fn send_transaction_message_route<'a>( pdu_id.into(), &db.globals, &db.account_data, - &db.sending, + &db.admin, )?; } _ => panic!("Not a sequential event or no parents found"), diff --git a/src/utils.rs b/src/utils.rs index e65ec864..edcf48a1 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -94,18 +94,3 @@ pub fn common_elements( .all(|b| b) })) } - -#[test] -fn sled_tests() { - let db = sled::Config::new().temporary(true).open().unwrap(); - - db.insert(1_u64.to_be_bytes(), vec![10]).unwrap(); - db.insert(2_u64.to_be_bytes(), vec![20]).unwrap(); - db.insert(3_u64.to_be_bytes(), vec![30]).unwrap(); - - let mut key = 1_u64.to_be_bytes().to_vec(); - key.push(1); - db.insert(key, vec![40]).unwrap(); - - println!("{:?}", db.iter().collect::, _>>().unwrap()) -} From 234b2264680a7c8e514e51672503519a08b05257 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 11 Nov 2020 14:30:12 -0500 Subject: [PATCH 09/36] Fix review issues, Remove EventHash's in prev/auth_events in StateEvent The latest state-res crate uses ruma's PduRoomV3 PDU's which don't have tuples of (EventId, EventHashs) like previous versions did (this was left from rebasing onto master). The Media DB now takes an optional content_type like the updated ruma structs. --- src/client_server/media.rs | 12 +++--------- src/database/media.rs | 18 ++++++++++++++---- src/database/rooms.rs | 1 + src/pdu.rs | 14 +++----------- src/server_server.rs | 4 ++-- 5 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index af7880cd..6d72107e 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -39,7 +39,7 @@ pub async fn create_content_route( db.media.create( mxc.clone(), &body.filename.as_deref(), - body.content_type.as_deref().unwrap_or("img"), // TODO this is now optional handle + &body.content_type.as_deref(), // TODO this is now optional handle &body.file, )?; @@ -85,10 +85,7 @@ pub async fn get_content_route( db.media.create( mxc, &get_content_response.content_disposition.as_deref(), - get_content_response // TODO this is now optional handle - .content_type - .as_deref() - .unwrap_or("img"), + &get_content_response.content_type.as_deref(), &get_content_response.file, )?; @@ -142,10 +139,7 @@ pub async fn get_content_thumbnail_route( db.media.upload_thumbnail( mxc, &None, - get_thumbnail_response - .content_type - .as_deref() - .unwrap_or("img"), // TODO now optional, deal with it somehow + &get_thumbnail_response.content_type, body.width.try_into().expect("all UInts are valid u32s"), body.height.try_into().expect("all UInts are valid u32s"), &get_thumbnail_response.file, diff --git a/src/database/media.rs b/src/database/media.rs index 8c59aa4d..bfc62078 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -20,7 +20,7 @@ impl Media { &self, mxc: String, filename: &Option<&str>, - content_type: &str, + content_type: &Option<&str>, file: &[u8], ) -> Result<()> { let mut key = mxc.as_bytes().to_vec(); @@ -30,7 +30,12 @@ impl Media { key.push(0xff); key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default()); key.push(0xff); - key.extend_from_slice(content_type.as_bytes()); + key.extend_from_slice( + content_type + .as_ref() + .map(|c| c.as_bytes()) + .unwrap_or_default(), + ); self.mediaid_file.insert(key, file)?; @@ -42,7 +47,7 @@ impl Media { &self, mxc: String, filename: &Option, - content_type: &str, + content_type: &Option, width: u32, height: u32, file: &[u8], @@ -54,7 +59,12 @@ impl Media { key.push(0xff); key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default()); key.push(0xff); - key.extend_from_slice(content_type.as_bytes()); + key.extend_from_slice( + content_type + .as_ref() + .map(|c| c.as_bytes()) + .unwrap_or_default(), + ); self.mediaid_file.insert(key, file)?; diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 08734b1b..d8e61316 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -647,6 +647,7 @@ impl Rooms { } /// Creates a new persisted data unit and adds it to a room. + #[allow(clippy::too_many_arguments)] pub fn build_and_append_pdu( &self, pdu_builder: PduBuilder, diff --git a/src/pdu.rs b/src/pdu.rs index 7118bfc8..a213d2cf 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -225,7 +225,7 @@ impl PduEvent { impl From<&state_res::StateEvent> for PduEvent { fn from(pdu: &state_res::StateEvent) -> Self { Self { - event_id: pdu.event_id().clone(), + event_id: pdu.event_id(), room_id: pdu.room_id().unwrap().clone(), sender: pdu.sender().clone(), origin_server_ts: (pdu @@ -260,17 +260,9 @@ impl PduEvent { "type": self.kind, "content": self.content, "state_key": self.state_key, - "prev_events": self.prev_events - .iter() - // TODO How do we create one of these - .map(|id| (id, EventHash { sha256: "hello".into() })) - .collect::>(), + "prev_events": self.prev_events, "depth": self.depth, - "auth_events": self.auth_events - .iter() - // TODO How do we create one of these - .map(|id| (id, EventHash { sha256: "hello".into() })) - .collect::>(), + "auth_events": self.auth_events, "redacts": self.redacts, "unsigned": self.unsigned, "hashes": self.hashes, diff --git a/src/server_server.rs b/src/server_server.rs index c865b028..19a73f9a 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -419,7 +419,7 @@ pub async fn send_transaction_message_route<'a>( } } - // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we? + // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we anywhere? // SPEC: // Servers MUST strictly enforce the JSON format specified in the appendices. // This translates to a 400 M_BAD_JSON error on most endpoints, or discarding of @@ -554,7 +554,7 @@ pub async fn send_transaction_message_route<'a>( // TODO we may not want the auth events chained in here for resolution? their_current_state .iter() - .map(|(_id, v)| ((v.kind(), v.state_key()), v.event_id().clone())) + .map(|(_id, v)| ((v.kind(), v.state_key()), v.event_id())) .collect::>(), ], Some( From 86bb93f8cf8eddee51ecf9188c8358ddfd7f175a Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sat, 14 Nov 2020 16:18:15 -0500 Subject: [PATCH 10/36] Remove outdated TODOs, use StateEvent::from_id_value consistently --- src/client_server/membership.rs | 11 ++++++---- src/pdu.rs | 37 +++++++++++++++++++-------------- src/server_server.rs | 8 +++---- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index a07fe727..39d69cdd 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -545,7 +545,7 @@ async fn join_room_by_id_helper( ); // It has enough fields to be called a proper event now - let join_event = dbg!(canon_json_stub); + let join_event = canon_json_stub; let send_join_response = server_server::send_request( &db.globals, @@ -577,7 +577,7 @@ async fn join_room_by_id_helper( .expect("a valid EventId can be converted to CanonicalJsonValue"), ); - Ok((event_id, serde_json::json!(value))) // TODO CanonicalJsonValue fixup? + Ok((event_id, serde_json::json!(value))) }; let room_state = send_join_response.room_state.state.iter().map(add_event_id); @@ -602,7 +602,8 @@ async fn join_room_by_id_helper( )))) // Add join event we just created .map(|r| { let (event_id, value) = r?; - serde_json::from_value::(value.clone()) + // TODO remove .clone when I remove debug logging + state_res::StateEvent::from_id_value(event_id.clone(), value.clone()) .map(|ev| (event_id, Arc::new(ev))) .map_err(|e| { warn!("{}: {}", value, e); @@ -642,7 +643,9 @@ async fn join_room_by_id_helper( .expect("iterative auth check failed on resolved events"); // This removes the control events that failed auth, leaving the resolved - // to be mainline sorted + // to be mainline sorted. In the actual `state_res::StateResolution::resolve` + // function both are removed since these are all events we don't know of + // we must keep track of everything to add to our DB. let events_to_sort = event_map .keys() .filter(|id| { diff --git a/src/pdu.rs b/src/pdu.rs index a213d2cf..effbc5d4 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -252,22 +252,27 @@ impl From<&state_res::StateEvent> for PduEvent { impl PduEvent { pub fn convert_for_state_res(&self) -> Arc { Arc::new( - serde_json::from_value(json!({ - "event_id": self.event_id, - "room_id": self.room_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "type": self.kind, - "content": self.content, - "state_key": self.state_key, - "prev_events": self.prev_events, - "depth": self.depth, - "auth_events": self.auth_events, - "redacts": self.redacts, - "unsigned": self.unsigned, - "hashes": self.hashes, - "signatures": self.signatures, - })) + // For consistency of eventId (just in case) we use the one + // generated by conduit for everything. + state_res::StateEvent::from_id_value( + self.event_id.clone(), + json!({ + "event_id": self.event_id, + "room_id": self.room_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "type": self.kind, + "content": self.content, + "state_key": self.state_key, + "prev_events": self.prev_events, + "depth": self.depth, + "auth_events": self.auth_events, + "redacts": self.redacts, + "unsigned": self.unsigned, + "hashes": self.hashes, + "signatures": self.signatures, + }), + ) .expect("all conduit PDUs are state events"), ) } diff --git a/src/server_server.rs b/src/server_server.rs index 19a73f9a..b9d26fd9 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -26,6 +26,7 @@ use std::{ collections::BTreeMap, convert::{TryFrom, TryInto}, fmt::Debug, + sync::Arc, time::{Duration, SystemTime}, }; use trust_dns_resolver::AsyncResolver; @@ -551,7 +552,6 @@ pub async fn send_transaction_message_route<'a>( .iter() .map(|((ev, sk), v)| ((ev.clone(), sk.to_owned()), v.event_id.clone())) .collect::>(), - // TODO we may not want the auth events chained in here for resolution? their_current_state .iter() .map(|(_id, v)| ((v.kind(), v.state_key()), v.event_id())) @@ -750,10 +750,10 @@ pub fn get_user_devices_route<'a>( /// Generates a correct eventId for the incoming pdu. /// -/// Returns a `state_res::StateEvent` which can be converted freely and has accessor methods. +/// Returns a tuple of the new `EventId` and the PDU with the eventId inserted as a `serde_json::Value`. fn process_incoming_pdu(pdu: &ruma::Raw) -> (EventId, serde_json::Value) { - let mut value = serde_json::from_str(pdu.json().get()) - .expect("converting raw jsons to values always works"); + let mut value = + serde_json::from_str(pdu.json().get()).expect("A Raw<...> is always valid JSON"); let event_id = EventId::try_from(&*format!( "${}", From bb24f6ad90513326582a908672543749571a2054 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 15 Nov 2020 16:48:43 -0500 Subject: [PATCH 11/36] Address some review issues fmt, errors, comments --- src/client_server/backup.rs | 2 +- src/client_server/media.rs | 2 +- src/client_server/membership.rs | 2 -- src/database/rooms.rs | 15 +++++++++++- src/database/sending.rs | 6 ++--- src/ruma_wrapper.rs | 41 +++++++++++++++++---------------- src/server_server.rs | 21 ++++++++--------- 7 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/client_server/backup.rs b/src/client_server/backup.rs index 607fa39e..676b5a3f 100644 --- a/src/client_server/backup.rs +++ b/src/client_server/backup.rs @@ -256,7 +256,7 @@ pub async fn get_backup_key_session_route( let key_data = db .key_backups .get_session(&sender_user, &body.version, &body.room_id, &body.session_id)? - .ok_or_else(|| Error::BadDatabase("Backup key not found for this user's session"))?; + .ok_or_else(|| Error::bad_database("Backup key not found for this user's session."))?; Ok(get_backup_key_session::Response { key_data }.into()) } diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 6d72107e..0c234884 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -39,7 +39,7 @@ pub async fn create_content_route( db.media.create( mxc.clone(), &body.filename.as_deref(), - &body.content_type.as_deref(), // TODO this is now optional handle + &body.content_type.as_deref(), &body.file, )?; diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 39d69cdd..849fb7e2 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -519,7 +519,6 @@ async fn join_room_by_id_helper( canon_json_stub.remove("event_id"); // In order to create a compatible ref hash (EventID) the `hashes` field needs to be present - // who the hell knew... ruma::signatures::hash_and_sign_event( db.globals.server_name().as_str(), db.globals.keypair(), @@ -602,7 +601,6 @@ async fn join_room_by_id_helper( )))) // Add join event we just created .map(|r| { let (event_id, value) = r?; - // TODO remove .clone when I remove debug logging state_res::StateEvent::from_id_value(event_id.clone(), value.clone()) .map(|ev| (event_id, Arc::new(ev))) .map_err(|e| { diff --git a/src/database/rooms.rs b/src/database/rooms.rs index d8e61316..3d5b890b 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -35,6 +35,11 @@ use super::admin::AdminCommand; /// hashing the entire state. pub type StateHashId = IVec; +/// An enum that represents the two valid states when searching +/// for an events "parent". +/// +/// An events parent is any event we are aware of that is part of +/// the events `prev_events` array. pub enum ClosestParent { Append, Insert(u64), @@ -80,7 +85,7 @@ impl StateStore for Rooms { .map_err(StateError::custom)? .ok_or_else(|| { StateError::NotFound(format!( - "PDU via room_id and event_id not found in the db.\n{}", + "PDU via room_id and event_id not found in the db: {}", event_id.as_str() )) })?; @@ -258,6 +263,8 @@ impl Rooms { } /// Force the creation of a new StateHash and insert it into the db. + /// + /// Whatever `state` is supplied to `force_state` __is__ the current room state snapshot. pub fn force_state( &self, room_id: &RoomId, @@ -403,6 +410,12 @@ impl Rooms { } } + /// Recursively search for a PDU from our DB that is also in the + /// `prev_events` field of the incoming PDU. + /// + /// First we check if the last PDU inserted to the given room is a parent + /// if not we recursively check older `prev_events` to insert the incoming + /// event after. pub fn get_closest_parent( &self, room: &RoomId, diff --git a/src/database/sending.rs b/src/database/sending.rs index 14558e3c..6b9e0fe7 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -2,7 +2,7 @@ use std::{collections::HashMap, convert::TryFrom, time::SystemTime}; use crate::{server_server, utils, Error, PduEvent, Result}; use federation::transactions::send_transaction_message; -use log::debug; +use log::{debug, error}; use rocket::futures::stream::{FuturesUnordered, StreamExt}; use ruma::{api::federation, ServerName}; use sled::IVec; @@ -115,8 +115,8 @@ impl Sending { // servercurrentpdus with the prefix should be empty now } } - Err((_server, _e)) => { - log::error!("server: {}\nerror: {}", _server, _e) + Err((server, e)) => { + error!("server: {}\nerror: {}", server, e) // TODO: exponential backoff } }; diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 1c5529a3..a68b09d0 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -67,27 +67,28 @@ where let (sender_user, sender_device) = // TODO: Do we need to matches! anything else here? ServerSignatures - if matches!(T::METADATA.authentication, AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken) { - // Get token from header or query value - let token = match request - .headers() - .get_one("Authorization") - .map(|s| s[7..].to_owned()) // Split off "Bearer " - .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok())) - { - // TODO: M_MISSING_TOKEN - None => return Failure((Status::Unauthorized, ())), - Some(token) => token, - }; - - // Check if token is valid - match db.users.find_from_token(&token).unwrap() { - // TODO: M_UNKNOWN_TOKEN - None => return Failure((Status::Unauthorized, ())), - Some((user_id, device_id)) => (Some(user_id), Some(device_id.into())), + match T::METADATA.authentication { + AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => { + // Get token from header or query value + let token = match request + .headers() + .get_one("Authorization") + .map(|s| s[7..].to_owned()) // Split off "Bearer " + .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok())) + { + // TODO: M_MISSING_TOKEN + None => return Failure((Status::Unauthorized, ())), + Some(token) => token, + }; + + // Check if token is valid + match db.users.find_from_token(&token).unwrap() { + // TODO: M_UNKNOWN_TOKEN + None => return Failure((Status::Unauthorized, ())), + Some((user_id, device_id)) => (Some(user_id), Some(device_id.into())), + } } - } else { - (None, None) + _ => (None, None) }; let mut http_request = http::Request::builder() diff --git a/src/server_server.rs b/src/server_server.rs index b9d26fd9..89d8eb16 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -4,7 +4,7 @@ use crate::{ }; use get_profile_information::v1::ProfileField; use http::header::{HeaderValue, AUTHORIZATION, HOST}; -use log::warn; +use log::{error, warn}; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ api::{ @@ -26,7 +26,6 @@ use std::{ collections::BTreeMap, convert::{TryFrom, TryInto}, fmt::Debug, - sync::Arc, time::{Duration, SystemTime}, }; use trust_dns_resolver::AsyncResolver; @@ -99,7 +98,7 @@ where let mut http_request = request .try_into_http_request(&actual_destination, Some("")) .map_err(|e| { - warn!("failed to find destination {}: {}", actual_destination, e); + warn!("Failed to find destination {}: {}", actual_destination, e); Error::BadServerResponse("Invalid destination") })?; @@ -264,12 +263,14 @@ pub fn get_server_keys(db: State<'_, Database>) -> Json { .body(), ) .unwrap(); + ruma::signatures::sign_json( db.globals.server_name().as_str(), db.globals.keypair(), &mut response, ) .unwrap(); + Json(ruma::serde::to_canonical_json_string(&response).expect("JSON is canonical")) } @@ -413,8 +414,8 @@ pub async fn send_transaction_message_route<'a>( "m.receipt" => {} _ => {} }, - Err(_err) => { - log::error!("{}", _err); + Err(err) => { + error!("{}", err); continue; } } @@ -434,11 +435,9 @@ pub async fn send_transaction_message_route<'a>( .expect("all ruma pdus are conduit pdus"); let room_id = &pdu.room_id; - // If we have no idea about this room - // TODO: Does a server only send us events that we should know about or - // when everyone on this server leaves a room can we ignore those events? + // If we have no idea about this room skip the PDU if !db.rooms.exists(&pdu.room_id)? { - log::error!("Room does not exist on this server"); + error!("Room does not exist on this server."); resolved_map.insert(event_id, Err("Room is unknown to this server".into())); continue; } @@ -460,7 +459,7 @@ pub async fn send_transaction_message_route<'a>( // As an example a possible error // {"errcode":"M_FORBIDDEN","error":"Host not in room."} Err(err) => { - log::error!("Request failed: {}", err); + error!("Request failed: {}", err); resolved_map.insert(event_id, Err(err.to_string())); continue; } @@ -487,7 +486,7 @@ pub async fn send_transaction_message_route<'a>( if value.get("state_key").is_none() { if !db.rooms.is_joined(&pdu.sender, &pdu.room_id)? { - log::error!("Sender is not joined {}", pdu.kind); + error!("Sender is not joined {}", pdu.kind); resolved_map.insert(event_id, Err("User is not in this room".into())); continue; } From b6d721374f970cca912477a3972bb857758d983d Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 18 Nov 2020 08:36:12 -0500 Subject: [PATCH 12/36] Have Media db return optional content_type, conversion fixes --- src/client_server/media.rs | 8 ++---- src/client_server/membership.rs | 3 +-- src/database/media.rs | 44 +++++++++++++++++++-------------- src/database/rooms.rs | 6 ++--- src/utils.rs | 17 +++++++++++++ 5 files changed, 47 insertions(+), 31 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 0c234884..e6bd182b 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -66,7 +66,7 @@ pub async fn get_content_route( { Ok(get_content::Response { file, - content_type: Some(content_type), + content_type, content_disposition: filename, } .into()) @@ -116,11 +116,7 @@ pub async fn get_content_thumbnail_route( .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, )? { - Ok(get_content_thumbnail::Response { - file, - content_type: Some(content_type), - } - .into()) + Ok(get_content_thumbnail::Response { file, content_type }.into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_thumbnail_response = server_server::send_request( &db.globals, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 849fb7e2..47fcde1a 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -510,8 +510,7 @@ async fn join_room_by_id_helper( .expect("event is valid, we just created it"), ); - // TODO fixup CanonicalJsonValue - // use that instead of serde_json::Map... maybe? + // Convert `serde_json;:Value` to `CanonicalJsonObj` for hashing/signing let mut canon_json_stub: BTreeMap<_, ruma::signatures::CanonicalJsonValue> = serde_json::from_value(join_event_stub_value).expect("json Value is canonical JSON"); diff --git a/src/database/media.rs b/src/database/media.rs index bfc62078..89d48e16 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -5,7 +5,7 @@ use std::mem; pub struct FileMeta { pub filename: Option, - pub content_type: String, + pub content_type: Option, pub file: Vec, } @@ -83,12 +83,14 @@ impl Media { let (key, file) = r?; let mut parts = key.rsplit(|&b| b == 0xff); - let content_type = utils::string_from_bytes( - parts - .next() - .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?, - ) - .map_err(|_| Error::bad_database("Content type in mediaid_file is invalid unicode."))?; + let content_type = parts + .next() + .map(|bytes| { + Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| { + Error::bad_database("Content type in mediaid_file is invalid unicode.") + })?) + }) + .transpose()?; let filename_bytes = parts .next() @@ -158,12 +160,14 @@ impl Media { let (key, file) = r?; let mut parts = key.rsplit(|&b| b == 0xff); - let content_type = utils::string_from_bytes( - parts - .next() - .ok_or_else(|| Error::bad_database("Invalid Media ID in db"))?, - ) - .map_err(|_| Error::bad_database("Content type in mediaid_file is invalid unicode."))?; + let content_type = parts + .next() + .map(|bytes| { + Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| { + Error::bad_database("Content type in mediaid_file is invalid unicode.") + })?) + }) + .transpose()?; let filename_bytes = parts .next() @@ -189,12 +193,14 @@ impl Media { let (key, file) = r?; let mut parts = key.rsplit(|&b| b == 0xff); - let content_type = utils::string_from_bytes( - parts - .next() - .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?, - ) - .map_err(|_| Error::bad_database("Content type in mediaid_file is invalid unicode."))?; + let content_type = parts + .next() + .map(|bytes| { + Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| { + Error::bad_database("Content type in mediaid_file is invalid unicode.") + })?) + }) + .transpose()?; let filename_bytes = parts .next() diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 3d5b890b..a95587cc 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -499,7 +499,6 @@ impl Rooms { Ok(()) } - #[allow(clippy::too_many_arguments)] /// Creates a new persisted data unit and adds it to a room. /// /// By this point the incoming event should be fully authenticated, no auth happens @@ -856,9 +855,8 @@ impl Rooms { }; // Hash and sign - let mut pdu_json: BTreeMap = - serde_json::from_value(serde_json::json!(&pdu)) - .expect("event is valid, we just created it"); + let mut pdu_json = + utils::to_canonical_object(&pdu).expect("event is valid, we just created it"); pdu_json.remove("event_id"); diff --git a/src/utils.rs b/src/utils.rs index edcf48a1..c82e6feb 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,6 +1,7 @@ use argon2::{Config, Variant}; use cmp::Ordering; use rand::prelude::*; +use ruma::serde::{try_from_json_map, CanonicalJsonError, CanonicalJsonObject}; use sled::IVec; use std::{ cmp, @@ -94,3 +95,19 @@ pub fn common_elements( .all(|b| b) })) } + +/// Fallible conversion from any value that implements `Serialize` to a `CanonicalJsonObject`. +/// +/// `value` must serialize to an `serde_json::Value::Object`. +pub fn to_canonical_object( + value: T, +) -> Result { + use serde::ser::Error; + + match serde_json::to_value(value).map_err(CanonicalJsonError::SerDe)? { + serde_json::Value::Object(map) => try_from_json_map(map), + _ => Err(CanonicalJsonError::SerDe(serde_json::Error::custom( + "Value must be an object", + ))), + } +} From 27e686f9ff03718c166b8e9af3536bc36d22083e Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Mon, 30 Nov 2020 12:10:33 -0500 Subject: [PATCH 13/36] Convert uses of serde_json::Value to CanonicalJsonObject --- Cargo.lock | 429 +++++++++++++++++--------------- src/client_server/membership.rs | 59 ++--- src/database/rooms.rs | 18 +- src/database/sending.rs | 4 +- src/server_server.rs | 21 +- 5 files changed, 276 insertions(+), 255 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2eef800..afb3abf3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,9 +2,9 @@ # It is not intended for manual editing. [[package]] name = "addr2line" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" +checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" dependencies = [ "gimli", ] @@ -30,12 +30,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "arc-swap" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" - [[package]] name = "arrayref" version = "0.3.6" @@ -56,9 +50,9 @@ checksum = "4af5687fe33aec5e70ef14caac5e0d363e335e5e5d6385fb75978d0c241b1d67" [[package]] name = "async-trait" -version = "0.1.41" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b246867b8b3b6ae56035f1eb1ed557c1d8eae97f0d53696138a50fa0e3a3b8c0" +checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" dependencies = [ "proc-macro2", "quote", @@ -90,9 +84,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" -version = "0.3.53" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707b586e0e2f247cbde68cdd2c3ce69ea7b7be43e1c5b426e37c9319c4b9838e" +checksum = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" dependencies = [ "addr2line", "cfg-if 1.0.0", @@ -104,9 +98,9 @@ dependencies = [ [[package]] name = "base-x" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" [[package]] name = "base64" @@ -114,6 +108,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "binascii" version = "0.1.4" @@ -128,9 +128,9 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "blake2b_simd" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" dependencies = [ "arrayref", "arrayvec", @@ -163,9 +163,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cc" -version = "1.0.61" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" +checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15" [[package]] name = "cfg-if" @@ -211,7 +211,7 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" name = "conduit" version = "0.1.0" dependencies = [ - "base64", + "base64 0.12.3", "directories", "http", "image", @@ -232,11 +232,21 @@ dependencies = [ "trust-dns-resolver", ] +[[package]] +name = "console_error_panic_hook" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d976903543e0c48546a91908f21588a680a8c8f984df9a5d69feccb2b2a211" +dependencies = [ + "cfg-if 0.1.10", + "wasm-bindgen", +] + [[package]] name = "const_fn" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" +checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab" [[package]] name = "constant_time_eq" @@ -246,20 +256,20 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "cookie" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" +checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" dependencies = [ "percent-encoding", - "time 0.2.22", + "time 0.2.23", "version_check", ] [[package]] name = "core-foundation" -version = "0.7.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" dependencies = [ "core-foundation-sys", "libc", @@ -267,9 +277,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.7.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" [[package]] name = "crc32fast" @@ -282,27 +292,26 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.8.2" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" dependencies = [ - "autocfg", - "cfg-if 0.1.10", + "cfg-if 1.0.0", + "const_fn", "crossbeam-utils", "lazy_static", - "maybe-uninit", "memoffset", "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.7.2" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" dependencies = [ "autocfg", - "cfg-if 0.1.10", + "cfg-if 1.0.0", "lazy_static", ] @@ -372,12 +381,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" -[[package]] -name = "dtoa" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" - [[package]] name = "either" version = "1.6.1" @@ -386,11 +389,11 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] @@ -464,9 +467,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95314d38584ffbfda215621d723e0a3906f032e03ae5551e650058dac83d4797" +checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" dependencies = [ "futures-channel", "futures-core", @@ -479,9 +482,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0448174b01148032eed37ac4aed28963aaaa8cfa93569a08e5b479bbc6c2c151" +checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" dependencies = [ "futures-core", "futures-sink", @@ -489,15 +492,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18eaa56102984bed2c88ea39026cff3ce3b4c7f508ca970cedf2450ea10d4e46" +checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" [[package]] name = "futures-executor" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5f8e0c9258abaea85e78ebdda17ef9666d390e987f006be6080dfe354b708cb" +checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" dependencies = [ "futures-core", "futures-task", @@ -506,15 +509,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1798854a4727ff944a7b12aa999f58ce7aa81db80d2dfaaf2ba06f065ddd2b" +checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" [[package]] name = "futures-macro" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36fccf3fc58563b4a14d265027c627c3b665d7fed489427e88e7cc929559efe" +checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -524,24 +527,24 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3ca3f17d6e8804ae5d3df7a7d35b2b3a6fe89dac84b31872720fc3060a0b11" +checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" [[package]] name = "futures-task" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d502af37186c4fef99453df03e374683f8a1eec9dcc1e66b3b82dc8278ce3c" +checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" dependencies = [ "once_cell", ] [[package]] name = "futures-util" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abcb44342f62e6f3e8ac427b8aa815f724fd705dfad060b18ac7866c15bb8e34" +checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" dependencies = [ "futures-channel", "futures-core", @@ -550,7 +553,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project 1.0.1", + "pin-project 1.0.2", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -602,9 +605,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" [[package]] name = "glob" @@ -702,9 +705,9 @@ checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" [[package]] name = "hyper" -version = "0.13.8" +version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" +checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" dependencies = [ "bytes", "futures-channel", @@ -716,7 +719,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 0.4.27", + "pin-project 1.0.2", "socket2", "tokio", "tower-service", @@ -750,9 +753,9 @@ dependencies = [ [[package]] name = "image" -version = "0.23.11" +version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f0a8345b33b082aedec2f4d7d4a926b845cee184cbe78b703413066564431b" +checksum = "7ce04077ead78e39ae8610ad26216aed811996b043d47beed5090db674f9e9b5" dependencies = [ "bytemuck", "byteorder", @@ -777,15 +780,15 @@ dependencies = [ [[package]] name = "inlinable_string" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6ee2a7da03bfc3b66ca47c92c2e392fcc053ea040a85561749b026f7aad09a" +checksum = "3094308123a0e9fd59659ce45e22de9f53fc1d2ac6e1feb9fef988e4f76cad77" [[package]] name = "instant" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb1fc4429a33e1f80d41dc9fea4d108a88bec1de8053878898ae448a0b52f613" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" dependencies = [ "cfg-if 1.0.0", ] @@ -889,9 +892,9 @@ checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" [[package]] name = "lock_api" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" dependencies = [ "scopeguard", ] @@ -954,23 +957,17 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "memchr" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" [[package]] name = "memoffset" -version = "0.5.6" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" dependencies = [ "autocfg", ] @@ -1042,9 +1039,9 @@ dependencies = [ [[package]] name = "miow" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" dependencies = [ "kernel32-sys", "net2", @@ -1054,9 +1051,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" +checksum = "6fcc7939b5edc4e4f86b1b4a04bb1498afaaf871b1a6691838ed06fcb48d3a3f" dependencies = [ "lazy_static", "libc", @@ -1072,9 +1069,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.35" +version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" +checksum = "d7cf75f38f16cb05ea017784dc6dbfd354f76c223dba37701734c4f5a9337d02" dependencies = [ "cfg-if 0.1.10", "libc", @@ -1083,9 +1080,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ "autocfg", "num-traits", @@ -1093,9 +1090,9 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.41" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6e6b7c748f995c4c29c5f5ae0248536e04a5739927c74ec0fa564805094b9f" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" dependencies = [ "autocfg", "num-integer", @@ -1104,9 +1101,9 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5b4d7360f362cfb50dde8143501e6940b22f644be75a4cc90b2d81968908138" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" dependencies = [ "autocfg", "num-integer", @@ -1115,9 +1112,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ "autocfg", ] @@ -1134,15 +1131,15 @@ dependencies = [ [[package]] name = "object" -version = "0.21.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37fd5004feb2ce328a52b0b3d01dbf4ffff72583493900ed15f22d4111c51693" +checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" [[package]] name = "once_cell" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" +checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" [[package]] name = "openssl" @@ -1189,9 +1186,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" dependencies = [ "instant", "lock_api", @@ -1251,11 +1248,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841" +checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" dependencies = [ - "pin-project-internal 1.0.1", + "pin-project-internal 1.0.2", ] [[package]] @@ -1271,9 +1268,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86" +checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" dependencies = [ "proc-macro2", "quote", @@ -1286,6 +1283,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" +[[package]] +name = "pin-project-lite" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" + [[package]] name = "pin-utils" version = "0.1.0" @@ -1312,9 +1315,9 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" [[package]] name = "proc-macro-crate" @@ -1327,9 +1330,9 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.18" +version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro-nested" @@ -1433,18 +1436,18 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745c1787167ddae5569661d5ffb8b25ae5fedbf46717eaa92d652221cec72623" +checksum = "e17626b2f4bcf35b84bf379072a66e28cfe5c3c6ae58b38e4914bb8891dabece" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d21b475ab879ef0e315ad99067fa25778c3b0377f57f1b00207448dac1a3144" +checksum = "0c523ccaed8ac4b0288948849a350b37d3035827413c458b6a40ddb614bb4f72" dependencies = [ "proc-macro2", "quote", @@ -1453,9 +1456,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" +checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" dependencies = [ "regex-syntax", ] @@ -1472,9 +1475,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.20" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" +checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" [[package]] name = "remove_dir_all" @@ -1487,11 +1490,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" +checksum = "fb15d6255c792356a0f578d8a645c677904dc02e862bebe2ecc18e0c01b9a0ce" dependencies = [ - "base64", + "base64 0.13.0", "bytes", "encoding_rs", "futures-core", @@ -1508,7 +1511,7 @@ dependencies = [ "mime_guess", "native-tls", "percent-encoding", - "pin-project-lite", + "pin-project-lite 0.2.0", "serde", "serde_urlencoded", "tokio", @@ -1516,15 +1519,16 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-bindgen-test", "web-sys", "winreg 0.7.0", ] [[package]] name = "resolv-conf" -version = "0.6.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11834e137f3b14e309437a8276714eed3a80d1ef894869e510f2c0c0b98b9f4a" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" dependencies = [ "hostname", "quick-error", @@ -1532,9 +1536,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.16.15" +version = "0.16.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "952cd6b98c85bbc30efa1ba5783b8abf12fec8b3287ffa52605b9432313e34e4" +checksum = "70017ed5c555d79ee3538fc63ca09c70ad8f317dcadc1adc2c496b60c22bb24f" dependencies = [ "cc", "libc", @@ -1563,7 +1567,7 @@ dependencies = [ "rocket_codegen", "rocket_http", "state", - "time 0.2.22", + "time 0.2.23", "tokio", "toml", "version_check", @@ -1598,7 +1602,7 @@ dependencies = [ "ref-cast", "smallvec", "state", - "time 0.2.22", + "time 0.2.23", "tokio", "tokio-rustls", "unicode-xid", @@ -1799,7 +1803,7 @@ name = "ruma-signatures" version = "0.6.0-dev.1" source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" dependencies = [ - "base64", + "base64 0.12.3", "ring", "ruma-identifiers", "ruma-serde", @@ -1809,11 +1813,11 @@ dependencies = [ [[package]] name = "rust-argon2" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19" +checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" dependencies = [ - "base64", + "base64 0.13.0", "blake2b_simd", "constant_time_eq", "crossbeam-utils", @@ -1840,7 +1844,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" dependencies = [ - "base64", + "base64 0.12.3", "log", "ring", "sct", @@ -1887,9 +1891,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "0.4.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535" +checksum = "c1759c2e3c8580017a484a7ac56d3abc5a6c1feadf88db2f3633f12ae4268c69" dependencies = [ "bitflags", "core-foundation", @@ -1900,9 +1904,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "0.4.3" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" +checksum = "f99b9d5e26d2a71633cc4f2ebae7cc9f874044e0c351a27e17892d76dce5678b" dependencies = [ "core-foundation-sys", "libc", @@ -1956,14 +1960,14 @@ dependencies = [ [[package]] name = "serde_urlencoded" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" dependencies = [ - "dtoa", + "form_urlencoded", "itoa", + "ryu", "serde", - "url", ] [[package]] @@ -1984,11 +1988,10 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e12110bc539e657a646068aaf5eb5b63af9d0c1f7b29c97113fad80e15f035" +checksum = "ce32ea0c6c56d5eacaeb814fbed9960547021d3edd010ded1425f180536b20ab" dependencies = [ - "arc-swap", "libc", ] @@ -2000,9 +2003,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "sled" -version = "0.34.4" +version = "0.34.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f72c064e63fbca3138ad07f3588c58093f1684f3a99f60dcfa6d46b87e60fde7" +checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc" dependencies = [ "crc32fast", "crossbeam-epoch", @@ -2016,17 +2019,17 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" +checksum = "7acad6f34eb9e8a259d3283d1e8c1d34d7415943d4895f65cc73813c7396fc85" [[package]] name = "socket2" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" +checksum = "2c29947abdee2a218277abeca306f25789c938e500ea5a9d4b12a5a504466902" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", "redox_syscall", "winapi 0.3.9", @@ -2040,23 +2043,23 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.11" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4e0831040d2cf2bdfd51b844be71885783d489898a192f254ae25d57cce725c" +checksum = "cf906c8b8fc3f6ecd1046e01da1d8ddec83e48c8b08b84dcc02b585a6bedf5a8" dependencies = [ "version_check", ] [[package]] name = "state" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" +checksum = "3015a7d0a5fd5105c91c3710d42f9ccf0abfb287d62206484dcc67f9569a6483" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res?branch=timo-spec-comp#1cd1a16cdefabb126a781a50b3d5eb1fdb3d3afb" +source = "git+https://github.com/ruma/state-res?branch=timo-spec-comp#d2a85669cc6056679ce6ca0fde4658a879ad2b08" dependencies = [ "itertools", "js_int", @@ -2141,9 +2144,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.48" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" +checksum = "8833e20724c24de12bbaba5ad230ea61c3eafb05b881c7c9d3cfe8638b187e68" dependencies = [ "proc-macro2", "quote", @@ -2166,18 +2169,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42" +checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab" +checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" dependencies = [ "proc-macro2", "quote", @@ -2206,9 +2209,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55b7151c9065e80917fbf285d9a5d1432f60db41d170ccafc749a136b41a93af" +checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" dependencies = [ "const_fn", "libc", @@ -2244,15 +2247,24 @@ dependencies = [ [[package]] name = "tinyvec" -version = "0.3.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" +checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" +checksum = "a6d7ad61edd59bfcc7e80dababf0f4aed2e6d5e0ba1659356ae889752dfc12ff" dependencies = [ "bytes", "fnv", @@ -2264,7 +2276,7 @@ dependencies = [ "mio", "mio-uds", "num_cpus", - "pin-project-lite", + "pin-project-lite 0.1.11", "signal-hook-registry", "slab", "tokio-macros", @@ -2273,9 +2285,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" dependencies = [ "proc-macro2", "quote", @@ -2314,7 +2326,7 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project-lite", + "pin-project-lite 0.1.11", "tokio", ] @@ -2335,13 +2347,13 @@ checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" [[package]] name = "tracing" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" +checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "log", - "pin-project-lite", + "pin-project-lite 0.2.0", "tracing-attributes", "tracing-core", ] @@ -2399,9 +2411,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2810660b9d5b18895d140caba6401765749a6a162e5d0736cfc44ea50db9d79d" +checksum = "a1fa8f0c8f4c594e4fc9debc1990deab13238077271ba84dd853d54902ee3401" dependencies = [ "ansi_term", "chrono", @@ -2421,9 +2433,9 @@ dependencies = [ [[package]] name = "trust-dns-proto" -version = "0.19.5" +version = "0.19.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdd7061ba6f4d4d9721afedffbfd403f20f39a4301fee1b70d6fcd09cca69f28" +checksum = "53861fcb288a166aae4c508ae558ed18b53838db728d4d310aad08270a7d4c2b" dependencies = [ "async-trait", "backtrace", @@ -2441,9 +2453,9 @@ dependencies = [ [[package]] name = "trust-dns-resolver" -version = "0.19.5" +version = "0.19.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f23cdfdc3d8300b3c50c9e84302d3bd6d860fb9529af84ace6cf9665f181b77" +checksum = "6759e8efc40465547b0dfce9500d733c65f969a4cbbfbe3ccf68daaa46ef179e" dependencies = [ "backtrace", "cfg-if 0.1.10", @@ -2485,18 +2497,18 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.13" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" [[package]] name = "unicode-xid" @@ -2512,10 +2524,11 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" dependencies = [ + "form_urlencoded", "idna", "matches", "percent-encoding", @@ -2623,6 +2636,30 @@ version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" +[[package]] +name = "wasm-bindgen-test" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34d1cdc8b98a557f24733d50a1199c4b0635e465eecba9c45b214544da197f64" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "scoped-tls", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fb9c67be7439ee8ab1b7db502a49c05e51e2835b66796c705134d9b8e1a585" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "web-sys" version = "0.3.45" @@ -2645,9 +2682,9 @@ dependencies = [ [[package]] name = "weezl" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8795d6e0e17485803cc10ef126bb8c0d59b7c61b219d66cfe0b3216dd0e8580a" +checksum = "3e2bb9fc8309084dd7cd651336673844c1d47f8ef6d2091ec160b27f5c4aa277" [[package]] name = "widestring" diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 47fcde1a..d5c50e5d 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -18,12 +18,13 @@ use ruma::{ federation, }, events::{pdu::Pdu, room::member, EventType}, + serde::{to_canonical_value, CanonicalJsonObject}, EventId, Raw, RoomId, RoomVersionId, ServerName, UserId, }; use state_res::StateEvent; use std::{ collections::{BTreeMap, HashMap, HashSet}, - convert::{TryFrom, TryInto}, + convert::TryFrom, iter, sync::Arc, }; @@ -477,30 +478,25 @@ async fn join_room_by_id_helper( let (make_join_response, remote_server) = make_join_response_and_server?; - let mut join_event_stub_value = - serde_json::from_str::(make_join_response.event.json().get()) + let mut join_event_stub = + serde_json::from_str::(make_join_response.event.json().get()) .map_err(|_| { Error::BadServerResponse("Invalid make_join event json received from server.") })?; - let join_event_stub = - join_event_stub_value - .as_object_mut() - .ok_or(Error::BadServerResponse( - "Invalid make join event object received from server.", - ))?; - join_event_stub.insert( "origin".to_owned(), - db.globals.server_name().to_owned().to_string().into(), + to_canonical_value(db.globals.server_name()) + .map_err(|_| Error::bad_database("Invalid server name found"))?, ); join_event_stub.insert( "origin_server_ts".to_owned(), - utils::millis_since_unix_epoch().into(), + to_canonical_value(utils::millis_since_unix_epoch()) + .expect("Timestamp is valid js_int value"), ); join_event_stub.insert( "content".to_owned(), - serde_json::to_value(member::MemberEventContent { + to_canonical_value(member::MemberEventContent { membership: member::MembershipState::Join, displayname: db.users.displayname(&sender_user)?, avatar_url: db.users.avatar_url(&sender_user)?, @@ -510,18 +506,14 @@ async fn join_room_by_id_helper( .expect("event is valid, we just created it"), ); - // Convert `serde_json;:Value` to `CanonicalJsonObj` for hashing/signing - let mut canon_json_stub: BTreeMap<_, ruma::signatures::CanonicalJsonValue> = - serde_json::from_value(join_event_stub_value).expect("json Value is canonical JSON"); - // We don't leave the event id in the pdu because that's only allowed in v1 or v2 rooms - canon_json_stub.remove("event_id"); + join_event_stub.remove("event_id"); // In order to create a compatible ref hash (EventID) the `hashes` field needs to be present ruma::signatures::hash_and_sign_event( db.globals.server_name().as_str(), db.globals.keypair(), - &mut canon_json_stub, + &mut join_event_stub, &RoomVersionId::Version6, ) .expect("event is valid, we just created it"); @@ -529,21 +521,19 @@ async fn join_room_by_id_helper( // Generate event id let event_id = EventId::try_from(&*format!( "${}", - ruma::signatures::reference_hash(&canon_json_stub, &RoomVersionId::Version6) + ruma::signatures::reference_hash(&join_event_stub, &RoomVersionId::Version6) .expect("ruma can calculate reference hashes") )) .expect("ruma's reference hashes are valid event ids"); // Add event_id back - canon_json_stub.insert( + join_event_stub.insert( "event_id".to_owned(), - serde_json::json!(event_id) - .try_into() - .expect("EventId is a valid CanonicalJsonValue"), + to_canonical_value(&event_id).expect("EventId is a valid CanonicalJsonValue"), ); // It has enough fields to be called a proper event now - let join_event = canon_json_stub; + let join_event = join_event_stub; let send_join_response = server_server::send_request( &db.globals, @@ -559,7 +549,7 @@ async fn join_room_by_id_helper( ) .await?; - let add_event_id = |pdu: &Raw| -> Result<(EventId, serde_json::Value)> { + let add_event_id = |pdu: &Raw| -> Result<(EventId, CanonicalJsonObject)> { let mut value = serde_json::from_str(pdu.json().get()) .expect("converting raw jsons to values always works"); let event_id = EventId::try_from(&*format!( @@ -571,18 +561,18 @@ async fn join_room_by_id_helper( value.insert( "event_id".to_owned(), - serde_json::from_value(serde_json::json!(event_id)) + to_canonical_value(&event_id) .expect("a valid EventId can be converted to CanonicalJsonValue"), ); - Ok((event_id, serde_json::json!(value))) + Ok((event_id, value)) }; let room_state = send_join_response.room_state.state.iter().map(add_event_id); let state_events = room_state .clone() - .map(|pdu: Result<(EventId, serde_json::Value)>| Ok(pdu?.0)) + .map(|pdu: Result<(EventId, CanonicalJsonObject)>| Ok(pdu?.0)) .chain(iter::once(Ok(event_id.clone()))) // Add join event we just created .collect::>>()?; @@ -594,16 +584,13 @@ async fn join_room_by_id_helper( let mut event_map = room_state .chain(auth_chain) - .chain(iter::once(Ok(( - event_id, - serde_json::to_value(join_event).unwrap(), - )))) // Add join event we just created + .chain(iter::once(Ok((event_id, join_event)))) // Add join event we just created .map(|r| { let (event_id, value) = r?; - state_res::StateEvent::from_id_value(event_id.clone(), value.clone()) + state_res::StateEvent::from_id_canon_obj(event_id.clone(), value.clone()) .map(|ev| (event_id, Arc::new(ev))) .map_err(|e| { - warn!("{}: {}", value, e); + warn!("{:?}: {}", value, e); Error::BadServerResponse("Invalid PDU in send_join response.") }) }) @@ -692,7 +679,7 @@ async fn join_room_by_id_helper( pdu_id.extend_from_slice(&count.to_be_bytes()); db.rooms.append_pdu( &PduEvent::from(&**pdu), - &serde_json::to_value(&**pdu).expect("PDU is valid value"), + &utils::to_canonical_object(&**pdu).expect("Pdu is valid canonical object"), count, pdu_id.clone().into(), &db.globals, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index a95587cc..651a596d 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -15,6 +15,7 @@ use ruma::{ }, EventType, }, + serde::{to_canonical_value, CanonicalJsonObject}, EventId, Raw, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId, }; use sled::IVec; @@ -506,7 +507,7 @@ impl Rooms { pub fn append_pdu( &self, pdu: &PduEvent, - pdu_json: &serde_json::Value, + pdu_json: &CanonicalJsonObject, count: u64, pdu_id: IVec, globals: &super::globals::Globals, @@ -520,7 +521,11 @@ impl Rooms { self.edus .private_read_set(&pdu.room_id, &pdu.sender, count, &globals)?; - self.pduid_pdu.insert(&pdu_id, &*pdu_json.to_string())?; + self.pduid_pdu.insert( + &pdu_id, + &*serde_json::to_string(pdu_json) + .expect("CanonicalJsonObject is always a valid String"), + )?; self.eventid_pduid .insert(pdu.event_id.as_bytes(), &*pdu_id)?; @@ -863,8 +868,7 @@ impl Rooms { // Add origin because synapse likes that (and it's required in the spec) pdu_json.insert( "origin".to_owned(), - serde_json::json!(globals.server_name()) - .try_into() + to_canonical_value(globals.server_name()) .expect("server name is a valid CanonicalJsonValue"), ); @@ -886,9 +890,7 @@ impl Rooms { pdu_json.insert( "event_id".to_owned(), - serde_json::json!(pdu.event_id) - .try_into() - .expect("EventId is a valid CanonicalJsonValue"), + to_canonical_value(&pdu.event_id).expect("EventId is a valid CanonicalJsonValue"), ); // Increment the last index and use that @@ -904,7 +906,7 @@ impl Rooms { self.append_pdu( &pdu, - &serde_json::json!(pdu_json), // TODO fixup CanonicalJsonValue + &pdu_json, count, pdu_id.clone().into(), globals, diff --git a/src/database/sending.rs b/src/database/sending.rs index 6b9e0fe7..0be14f85 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -185,9 +185,7 @@ impl Sending { .iter() .map(|pdu_id| { Ok::<_, (Box, Error)>( - // TODO: this was a PduStub - // In order for sending to work these actually do have to be - // PduStub but, since they are Raw<..> we can fake it. + // TODO: check room version and remove event_id if needed serde_json::from_str( PduEvent::convert_to_outgoing_federation_event( rooms diff --git a/src/server_server.rs b/src/server_server.rs index 89d8eb16..8d3de1e4 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -20,6 +20,7 @@ use ruma::{ OutgoingRequest, }, directory::{IncomingFilter, IncomingRoomNetwork}, + serde::{to_canonical_value, CanonicalJsonObject}, EventId, RoomId, RoomVersionId, ServerName, UserId, }; use std::{ @@ -431,12 +432,13 @@ pub async fn send_transaction_message_route<'a>( let mut resolved_map = BTreeMap::new(); for pdu in &body.pdus { let (event_id, value) = process_incoming_pdu(pdu); - let pdu = serde_json::from_value::(value.clone()) + // TODO: this is an unfortunate conversion dance... + let pdu = serde_json::from_value::(serde_json::to_value(&value).expect("msg")) .expect("all ruma pdus are conduit pdus"); let room_id = &pdu.room_id; // If we have no idea about this room skip the PDU - if !db.rooms.exists(&pdu.room_id)? { + if !db.rooms.exists(room_id)? { error!("Room does not exist on this server."); resolved_map.insert(event_id, Err("Room is unknown to this server".into())); continue; @@ -477,7 +479,7 @@ pub async fn send_transaction_message_route<'a>( // When creating a StateEvent the event_id arg will be used // over any found in the json and it will not use ruma::reference_hash // to generate one - state_res::StateEvent::from_id_value(event_id, json) + state_res::StateEvent::from_id_canon_obj(event_id, json) .expect("valid pdu json"), ), ) @@ -485,7 +487,7 @@ pub async fn send_transaction_message_route<'a>( .collect::>(); if value.get("state_key").is_none() { - if !db.rooms.is_joined(&pdu.sender, &pdu.room_id)? { + if !db.rooms.is_joined(&pdu.sender, room_id)? { error!("Sender is not joined {}", pdu.kind); resolved_map.insert(event_id, Err("User is not in this room".into())); continue; @@ -750,7 +752,7 @@ pub fn get_user_devices_route<'a>( /// Generates a correct eventId for the incoming pdu. /// /// Returns a tuple of the new `EventId` and the PDU with the eventId inserted as a `serde_json::Value`. -fn process_incoming_pdu(pdu: &ruma::Raw) -> (EventId, serde_json::Value) { +fn process_incoming_pdu(pdu: &ruma::Raw) -> (EventId, CanonicalJsonObject) { let mut value = serde_json::from_str(pdu.json().get()).expect("A Raw<...> is always valid JSON"); @@ -763,13 +765,8 @@ fn process_incoming_pdu(pdu: &ruma::Raw) -> (EventId, se value.insert( "event_id".to_owned(), - serde_json::json!(event_id) - .try_into() - .expect("EventId is a valid CanonicalJsonValue"), + to_canonical_value(&event_id).expect("EventId is a valid CanonicalJsonValue"), ); - ( - event_id, - serde_json::to_value(value).expect("JSON Value is a CanonicalJsonValue"), - ) + (event_id, value) } From c173ce43a5e6ca6abe0b23a590c124e85173593c Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Mon, 30 Nov 2020 14:46:47 -0500 Subject: [PATCH 14/36] convert_to_outgoing_federation_event takes CanonicalJsonObj --- src/client_server/membership.rs | 5 +---- src/database/rooms.rs | 4 ++-- src/pdu.rs | 31 ++++++++++++++++--------------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index d5c50e5d..b5e4042f 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -541,10 +541,7 @@ async fn join_room_by_id_helper( federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, - pdu_stub: PduEvent::convert_to_outgoing_federation_event( - serde_json::to_value(&join_event) - .expect("we just validated and ser/de this event"), - ), + pdu_stub: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), }, ) .await?; diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 651a596d..e7b6eaac 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -385,8 +385,8 @@ impl Rooms { }) } - /// Returns the pdu. - pub fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result> { + /// Returns the pdu as a `BTreeMap`. + pub fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result> { self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| { Ok(Some( serde_json::from_slice(&pdu) diff --git a/src/pdu.rs b/src/pdu.rs index effbc5d4..cffd4a32 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -5,6 +5,7 @@ use ruma::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, + serde::{CanonicalJsonObject, CanonicalJsonValue}, EventId, Raw, RoomId, ServerKeyId, ServerName, UserId, }; use serde::{Deserialize, Serialize}; @@ -200,25 +201,25 @@ impl PduEvent { } pub fn convert_to_outgoing_federation_event( - mut pdu_json: serde_json::Value, + mut pdu_json: CanonicalJsonObject, ) -> Raw { - if let Some(unsigned) = pdu_json - .as_object_mut() - .expect("json is object") - .get_mut("unsigned") - { - unsigned - .as_object_mut() - .expect("unsigned is object") - .remove("transaction_id"); + if let Some(CanonicalJsonValue::Object(unsigned)) = pdu_json.get_mut("unsigned") { + unsigned.remove("transaction_id"); } - pdu_json - .as_object_mut() - .expect("json is object") - .remove("event_id"); + pdu_json.remove("event_id"); - serde_json::from_value::>(pdu_json).expect("Raw::from_value always works") + // TODO: another option would be to convert it to a canonical string to validate size + // and return a Result> + // serde_json::from_str::>( + // ruma::serde::to_canonical_json_string(pdu_json).expect("CanonicalJson is valid serde_json::Value"), + // ) + // .expect("Raw::from_value always works") + + serde_json::from_value::>( + serde_json::to_value(pdu_json).expect("CanonicalJson is valid serde_json::Value"), + ) + .expect("Raw::from_value always works") } } From b869aab5d03b0ec606d664939782f42ea57ce624 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Fri, 4 Dec 2020 17:16:29 -0500 Subject: [PATCH 15/36] Cleanup removing debug printing and logging, append non state events --- src/client_server/backup.rs | 7 +- src/database/rooms.rs | 4 +- src/database/sending.rs | 4 +- src/pdu.rs | 35 ++++++++- src/ruma_wrapper.rs | 46 ++++++----- src/server_server.rs | 153 +++++++++++++----------------------- 6 files changed, 119 insertions(+), 130 deletions(-) diff --git a/src/client_server/backup.rs b/src/client_server/backup.rs index 676b5a3f..0f34ba78 100644 --- a/src/client_server/backup.rs +++ b/src/client_server/backup.rs @@ -256,7 +256,12 @@ pub async fn get_backup_key_session_route( let key_data = db .key_backups .get_session(&sender_user, &body.version, &body.room_id, &body.session_id)? - .ok_or_else(|| Error::bad_database("Backup key not found for this user's session."))?; + .ok_or_else(|| { + Error::BadRequest( + ErrorKind::NotFound, + "Backup key not found for this user's session.", + ) + })?; Ok(get_backup_key_session::Response { key_data }.into()) } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index e7b6eaac..0618fd6e 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -41,7 +41,7 @@ pub type StateHashId = IVec; /// /// An events parent is any event we are aware of that is part of /// the events `prev_events` array. -pub enum ClosestParent { +pub(crate) enum ClosestParent { Append, Insert(u64), } @@ -417,7 +417,7 @@ impl Rooms { /// First we check if the last PDU inserted to the given room is a parent /// if not we recursively check older `prev_events` to insert the incoming /// event after. - pub fn get_closest_parent( + pub(crate) fn get_latest_pduid_before( &self, room: &RoomId, incoming_prev_ids: &[EventId], diff --git a/src/database/sending.rs b/src/database/sending.rs index 0be14f85..cd88e088 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -2,7 +2,7 @@ use std::{collections::HashMap, convert::TryFrom, time::SystemTime}; use crate::{server_server, utils, Error, PduEvent, Result}; use federation::transactions::send_transaction_message; -use log::{debug, error}; +use log::{debug, warn}; use rocket::futures::stream::{FuturesUnordered, StreamExt}; use ruma::{api::federation, ServerName}; use sled::IVec; @@ -116,7 +116,7 @@ impl Sending { } } Err((server, e)) => { - error!("server: {}\nerror: {}", server, e) + warn!("Couldn't send transaction to {}: {}", server, e) // TODO: exponential backoff } }; diff --git a/src/pdu.rs b/src/pdu.rs index cffd4a32..e56e81ad 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -5,12 +5,17 @@ use ruma::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, - serde::{CanonicalJsonObject, CanonicalJsonValue}, - EventId, Raw, RoomId, ServerKeyId, ServerName, UserId, + serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue}, + EventId, Raw, RoomId, RoomVersionId, ServerKeyId, ServerName, UserId, }; use serde::{Deserialize, Serialize}; use serde_json::json; -use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::UNIX_EPOCH}; +use std::{ + collections::BTreeMap, + convert::{TryFrom, TryInto}, + sync::Arc, + time::UNIX_EPOCH, +}; #[derive(Deserialize, Serialize, Debug)] pub struct PduEvent { @@ -279,6 +284,30 @@ impl PduEvent { } } +/// Generates a correct eventId for the incoming pdu. +/// +/// Returns a tuple of the new `EventId` and the PDU with the eventId inserted as a `serde_json::Value`. +pub(crate) fn process_incoming_pdu( + pdu: &ruma::Raw, +) -> (EventId, CanonicalJsonObject) { + let mut value = + serde_json::from_str(pdu.json().get()).expect("A Raw<...> is always valid JSON"); + + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&value, &RoomVersionId::Version6) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + value.insert( + "event_id".to_owned(), + to_canonical_value(&event_id).expect("EventId is a valid CanonicalJsonValue"), + ); + + (event_id, value) +} + /// Build the start of a PDU in order to add it to the `Database`. #[derive(Debug, Deserialize)] pub struct PduBuilder { diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index a68b09d0..4b3d08d5 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -65,31 +65,29 @@ where .await .expect("database was loaded"); - let (sender_user, sender_device) = - // TODO: Do we need to matches! anything else here? ServerSignatures - match T::METADATA.authentication { - AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => { - // Get token from header or query value - let token = match request - .headers() - .get_one("Authorization") - .map(|s| s[7..].to_owned()) // Split off "Bearer " - .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok())) - { - // TODO: M_MISSING_TOKEN - None => return Failure((Status::Unauthorized, ())), - Some(token) => token, - }; - - // Check if token is valid - match db.users.find_from_token(&token).unwrap() { - // TODO: M_UNKNOWN_TOKEN - None => return Failure((Status::Unauthorized, ())), - Some((user_id, device_id)) => (Some(user_id), Some(device_id.into())), - } + let (sender_user, sender_device) = match T::METADATA.authentication { + AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => { + // Get token from header or query value + let token = match request + .headers() + .get_one("Authorization") + .map(|s| s[7..].to_owned()) // Split off "Bearer " + .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok())) + { + // TODO: M_MISSING_TOKEN + None => return Failure((Status::Unauthorized, ())), + Some(token) => token, + }; + + // Check if token is valid + match db.users.find_from_token(&token).unwrap() { + // TODO: M_UNKNOWN_TOKEN + None => return Failure((Status::Unauthorized, ())), + Some((user_id, device_id)) => (Some(user_id), Some(device_id.into())), } - _ => (None, None) - }; + } + _ => (None, None), + }; let mut http_request = http::Request::builder() .uri(request.uri().to_string()) diff --git a/src/server_server.rs b/src/server_server.rs index 8d3de1e4..a7f6391d 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -20,13 +20,13 @@ use ruma::{ OutgoingRequest, }, directory::{IncomingFilter, IncomingRoomNetwork}, - serde::{to_canonical_value, CanonicalJsonObject}, - EventId, RoomId, RoomVersionId, ServerName, UserId, + EventId, RoomId, ServerName, UserId, }; use std::{ collections::BTreeMap, convert::{TryFrom, TryInto}, fmt::Debug, + sync::Arc, time::{Duration, SystemTime}, }; use trust_dns_resolver::AsyncResolver; @@ -415,8 +415,7 @@ pub async fn send_transaction_message_route<'a>( "m.receipt" => {} _ => {} }, - Err(err) => { - error!("{}", err); + Err(_err) => { continue; } } @@ -431,19 +430,53 @@ pub async fn send_transaction_message_route<'a>( // would return a M_BAD_JSON error. let mut resolved_map = BTreeMap::new(); for pdu in &body.pdus { - let (event_id, value) = process_incoming_pdu(pdu); - // TODO: this is an unfortunate conversion dance... - let pdu = serde_json::from_value::(serde_json::to_value(&value).expect("msg")) - .expect("all ruma pdus are conduit pdus"); + // Ruma/PduEvent/StateEvent satifies - 1. Is a valid event, otherwise it is dropped. + + // state-res checks signatures - 2. Passes signature checks, otherwise event is dropped. + + // 3. Passes hash checks, otherwise it is redacted before being processed further. + // TODO: redact event if hashing fails + let (event_id, value) = crate::pdu::process_incoming_pdu(pdu); + + let pdu = serde_json::from_value::( + serde_json::to_value(&value).expect("CanonicalJsonObj is a valid JsonValue"), + ) + .expect("all ruma pdus are conduit pdus"); let room_id = &pdu.room_id; // If we have no idea about this room skip the PDU if !db.rooms.exists(room_id)? { - error!("Room does not exist on this server."); resolved_map.insert(event_id, Err("Room is unknown to this server".into())); continue; } + // If it is not a state event, we can skip state-res + if value.get("state_key").is_none() { + if !db.rooms.is_joined(&pdu.sender, room_id)? { + warn!("Sender is not joined {}", pdu.kind); + resolved_map.insert(event_id, Err("User is not in this room".into())); + continue; + } + + let count = db.globals.next_count()?; + let mut pdu_id = room_id.as_bytes().to_vec(); + pdu_id.push(0xff); + pdu_id.extend_from_slice(&count.to_be_bytes()); + db.rooms.append_pdu( + &pdu, + &value, + count, + pdu_id.into(), + &db.globals, + &db.account_data, + &db.admin, + )?; + + resolved_map.insert(event_id, Ok::<(), String>(())); + continue; + } + + // We have a state event so we need info for state-res let get_state_response = match send_request( &db.globals, body.body.origin.clone(), @@ -461,7 +494,6 @@ pub async fn send_transaction_message_route<'a>( // As an example a possible error // {"errcode":"M_FORBIDDEN","error":"Host not in room."} Err(err) => { - error!("Request failed: {}", err); resolved_map.insert(event_id, Err(err.to_string())); continue; } @@ -472,10 +504,10 @@ pub async fn send_transaction_message_route<'a>( .iter() .chain(get_state_response.auth_chain.iter()) // add auth events .map(|pdu| { - let (event_id, json) = process_incoming_pdu(pdu); + let (event_id, json) = crate::pdu::process_incoming_pdu(pdu); ( event_id.clone(), - std::sync::Arc::new( + Arc::new( // When creating a StateEvent the event_id arg will be used // over any found in the json and it will not use ruma::reference_hash // to generate one @@ -486,65 +518,12 @@ pub async fn send_transaction_message_route<'a>( }) .collect::>(); - if value.get("state_key").is_none() { - if !db.rooms.is_joined(&pdu.sender, room_id)? { - error!("Sender is not joined {}", pdu.kind); - resolved_map.insert(event_id, Err("User is not in this room".into())); - continue; - } - - // If the event is older than the last event in pduid_pdu Tree then find the - // closest ancestor we know of and insert after the known ancestor by - // altering the known events pduid to = same roomID + same count bytes + 0x1 - // pushing a single byte every time a simple append cannot be done. - match db - .rooms - .get_closest_parent(room_id, &pdu.prev_events, &their_current_state)? - { - Some(ClosestParent::Append) => { - let count = db.globals.next_count()?; - let mut pdu_id = room_id.as_bytes().to_vec(); - pdu_id.push(0xff); - pdu_id.extend_from_slice(&count.to_be_bytes()); - - db.rooms.append_pdu( - &pdu, - &value, - count, - pdu_id.into(), - &db.globals, - &db.account_data, - &db.admin, - )?; - } - Some(ClosestParent::Insert(old_count)) => { - println!("INSERT PDU FOUND {}", old_count); - - let count = old_count; - let mut pdu_id = room_id.as_bytes().to_vec(); - pdu_id.push(0xff); - pdu_id.extend_from_slice(&count.to_be_bytes()); - // Create a new count that is after old_count but before - // the pdu appended after - pdu_id.push(1); - - db.rooms.append_pdu( - &pdu, - &value, - count, - pdu_id.into(), - &db.globals, - &db.account_data, - &db.admin, - )?; - } - _ => panic!("Not a sequential event or no parents found"), - }; - resolved_map.insert(event_id, Ok::<(), String>(())); - continue; - } - let our_current_state = db.rooms.room_state_full(room_id)?; + // State resolution takes care of these checks + // 4. Passes authorization rules based on the event's auth events, otherwise it is rejected. + // 5. Passes authorization rules based on the state at the event, otherwise it is rejected. + + // TODO: 6. Passes authorization rules based on the current state of the room, otherwise it is "soft failed". match state_res::StateResolution::resolve( room_id, &ruma::RoomVersionId::Version6, @@ -576,7 +555,7 @@ pub async fn send_transaction_message_route<'a>( // closest ancestor we know of and insert after the known ancestor by // altering the known events pduid to = same roomID + same count bytes + 0x1 // pushing a single byte every time a simple append cannot be done. - match db.rooms.get_closest_parent( + match db.rooms.get_latest_pduid_before( room_id, &pdu.prev_events, &their_current_state, @@ -598,8 +577,6 @@ pub async fn send_transaction_message_route<'a>( )?; } Some(ClosestParent::Insert(old_count)) => { - println!("INSERT STATE PDU FOUND {}", old_count); - let count = old_count; let mut pdu_id = room_id.as_bytes().to_vec(); pdu_id.push(0xff); @@ -618,14 +595,16 @@ pub async fn send_transaction_message_route<'a>( &db.admin, )?; } - _ => panic!("Not a sequential event or no parents found"), + _ => { + error!("Not a sequential event or no parents found"); + continue; + } } resolved_map.insert(event_id, Ok::<(), String>(())); } // If the eventId is not found in the resolved state auth has failed Ok(_) => { - // TODO have state_res give the actual auth error in this case resolved_map.insert( event_id, Err("This event failed authentication, not found in resolved set".into()), @@ -637,7 +616,7 @@ pub async fn send_transaction_message_route<'a>( }; } - Ok(dbg!(send_transaction_message::v1::Response { pdus: resolved_map }).into()) + Ok(send_transaction_message::v1::Response { pdus: resolved_map }.into()) } #[cfg_attr( @@ -748,25 +727,3 @@ pub fn get_user_devices_route<'a>( .into()) } */ - -/// Generates a correct eventId for the incoming pdu. -/// -/// Returns a tuple of the new `EventId` and the PDU with the eventId inserted as a `serde_json::Value`. -fn process_incoming_pdu(pdu: &ruma::Raw) -> (EventId, CanonicalJsonObject) { - let mut value = - serde_json::from_str(pdu.json().get()).expect("A Raw<...> is always valid JSON"); - - let event_id = EventId::try_from(&*format!( - "${}", - ruma::signatures::reference_hash(&value, &RoomVersionId::Version6) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); - - value.insert( - "event_id".to_owned(), - to_canonical_value(&event_id).expect("EventId is a valid CanonicalJsonValue"), - ); - - (event_id, value) -} From 164b1633d8b9bfe0dfba03dca8703adf422d586b Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Fri, 4 Dec 2020 18:16:17 -0500 Subject: [PATCH 16/36] Update ruma to latest, renamed server keys and removed PduStub --- Cargo.lock | 69 +++++++++++++++++---------------- Cargo.toml | 4 +- src/client_server/account.rs | 1 + src/client_server/config.rs | 2 +- src/client_server/directory.rs | 15 +++++-- src/client_server/membership.rs | 6 +-- src/client_server/presence.rs | 2 +- src/client_server/profile.rs | 2 +- src/client_server/room.rs | 15 ++++--- src/client_server/sync.rs | 3 +- src/database/account_data.rs | 3 +- src/database/admin.rs | 5 ++- src/database/rooms.rs | 11 +++--- src/database/rooms/edus.rs | 3 +- src/database/users.rs | 3 +- src/error.rs | 6 +-- src/pdu.rs | 13 ++++--- src/push_rules.rs | 23 +++++++---- src/server_server.rs | 24 +++++++----- 19 files changed, 123 insertions(+), 87 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index afb3abf3..efc34e4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1210,6 +1210,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "paste" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7151b083b0664ed58ed669fcdd92f01c3d2fdbf10af4931a301474950b52bfa9" + [[package]] name = "pear" version = "0.2.0-dev" @@ -1611,7 +1617,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "assign", "js_int", @@ -1629,24 +1635,22 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "http", "percent-encoding", "ruma-api-macros", - "ruma-common", "ruma-identifiers", "ruma-serde", "serde", "serde_json", - "strum", "thiserror", ] [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1657,12 +1661,13 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "ruma-api", "ruma-common", "ruma-events", "ruma-identifiers", + "ruma-serde", "serde", "serde_json", ] @@ -1670,7 +1675,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "assign", "http", @@ -1684,38 +1689,25 @@ dependencies = [ "ruma-serde", "serde", "serde_json", - "strum", ] [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "js_int", - "ruma-common-macros", + "maplit", "ruma-identifiers", "ruma-serde", "serde", "serde_json", - "strum", -] - -[[package]] -name = "ruma-common-macros" -version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", ] [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "js_int", "ruma-common", @@ -1724,13 +1716,12 @@ dependencies = [ "ruma-serde", "serde", "serde_json", - "strum", ] [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1741,7 +1732,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "js_int", "ruma-api", @@ -1756,8 +1747,9 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ + "paste", "rand", "ruma-identifiers-macros", "ruma-identifiers-validation", @@ -1769,7 +1761,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "proc-macro2", "quote", @@ -1780,28 +1772,39 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "serde", - "strum", ] [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "form_urlencoded", "itoa", "js_int", + "ruma-serde-macros", "serde", "serde_json", ] +[[package]] +name = "ruma-serde-macros" +version = "0.2.0" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/ruma/ruma?rev=c15382ca41262058302959eac4029ab4a1ea5889#c15382ca41262058302959eac4029ab4a1ea5889" +source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" dependencies = [ "base64 0.12.3", "ring", @@ -2059,7 +2062,7 @@ checksum = "3015a7d0a5fd5105c91c3710d42f9ccf0abfb287d62206484dcc67f9569a6483" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res?branch=timo-spec-comp#d2a85669cc6056679ce6ca0fde4658a879ad2b08" +source = "git+https://github.com/ruma/state-res?branch=timo-spec-comp#99214e6fa6b9843b0d9e1f6ef0698d7fdb234fb2" dependencies = [ "itertools", "js_int", diff --git a/Cargo.toml b/Cargo.toml index e7b87fa1..4902c7c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,14 +18,14 @@ edition = "2018" rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] } # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks", "unstable-exhaustive-types"], rev = "c15382ca41262058302959eac4029ab4a1ea5889" } +ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks", "unstable-exhaustive-types"], rev = "e8882fe8142d7b55ed4c8ccc6150946945f9e237" } # ruma = { git = "https://github.com/DevinR528/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "unstable-join" } # ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } # Used when doing state resolution # state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] } state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec", "gen-eventid"] } -# state-res = { path = "../../state-res", features = ["unstable-pre-spec"] } +# state-res = { path = "../../state-res", features = ["unstable-pre-spec", "gen-eventid"] } # Used for long polling tokio = "0.2.22" diff --git a/src/client_server/account.rs b/src/client_server/account.rs index ab90de5b..76bbebb5 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -492,6 +492,7 @@ pub async fn register_route( body: "Thanks for trying out Conduit! This software is still in development, so expect many bugs and missing features. If you have federation enabled, you can join the Conduit chat room by typing /join #conduit:matrix.org. Important: Please don't join any other Matrix rooms over federation without permission from the room's admins. Some actions might trigger bugs in other server implementations, breaking the chat for everyone else.".to_owned(), }), relates_to: None, + new_content: None, }, )) .expect("event is valid, we just created it"), diff --git a/src/client_server/config.rs b/src/client_server/config.rs index dd8de640..f1d233a7 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -6,7 +6,7 @@ use ruma::{ r0::config::{get_global_account_data, set_global_account_data}, }, events::{custom::CustomEventContent, BasicEvent}, - Raw, + serde::Raw, }; #[cfg(feature = "conduit_bin")] diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index d8af2e3f..559071a1 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -20,7 +20,8 @@ use ruma::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, }, - Raw, ServerName, + serde::Raw, + ServerName, }; #[cfg(feature = "conduit_bin")] @@ -83,7 +84,13 @@ pub async fn set_room_visibility_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - match body.visibility { + match &body.visibility { + room::Visibility::_Custom(_s) => { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Room visibility type is not supported.", + )); + } room::Visibility::Public => { db.rooms.set_public(&body.room_id, true)?; info!("{} made {} public", sender_user, body.room_id); @@ -294,7 +301,9 @@ pub async fn get_public_rooms_filtered_helper( .url, ) }) - .transpose()?, + .transpose()? + // url is now an Option so we must flatten + .flatten(), }; Ok(chunk) }) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index b5e4042f..6d3a6907 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -18,8 +18,8 @@ use ruma::{ federation, }, events::{pdu::Pdu, room::member, EventType}, - serde::{to_canonical_value, CanonicalJsonObject}, - EventId, Raw, RoomId, RoomVersionId, ServerName, UserId, + serde::{to_canonical_value, CanonicalJsonObject, Raw}, + EventId, RoomId, RoomVersionId, ServerName, UserId, }; use state_res::StateEvent; use std::{ @@ -541,7 +541,7 @@ async fn join_room_by_id_helper( federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, - pdu_stub: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), + pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), }, ) .await?; diff --git a/src/client_server/presence.rs b/src/client_server/presence.rs index e597c698..15c746e4 100644 --- a/src/client_server/presence.rs +++ b/src/client_server/presence.rs @@ -32,7 +32,7 @@ pub async fn set_presence_route( .try_into() .expect("time is valid"), ), - presence: body.presence, + presence: body.presence.clone(), status_msg: body.status_msg.clone(), }, sender: sender_user.clone(), diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 22d13cbd..828d2593 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -8,7 +8,7 @@ use ruma::{ }, }, events::EventType, - Raw, + serde::Raw, }; #[cfg(feature = "conduit_bin")] diff --git a/src/client_server/room.rs b/src/client_server/room.rs index a50f69c8..f92fc8df 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -10,7 +10,8 @@ use ruma::{ room::{guest_access, history_visibility, join_rules, member, name, topic}, EventType, }, - Raw, RoomAliasId, RoomId, RoomVersionId, + serde::Raw, + RoomAliasId, RoomId, RoomVersionId, }; use std::{cmp::max, collections::BTreeMap, convert::TryFrom}; @@ -141,10 +142,14 @@ pub async fn create_room_route( // 4. Events set by preset // Figure out preset. We need it for preset specific events - let preset = body.preset.unwrap_or_else(|| match body.visibility { - room::Visibility::Private => create_room::RoomPreset::PrivateChat, - room::Visibility::Public => create_room::RoomPreset::PublicChat, - }); + let preset = body + .preset + .clone() + .unwrap_or_else(|| match &body.visibility { + room::Visibility::Private => create_room::RoomPreset::PrivateChat, + room::Visibility::Public => create_room::RoomPreset::PublicChat, + room::Visibility::_Custom(s) => create_room::RoomPreset::_Custom(s.into()), + }); // 4.1 Join Rules db.rooms.build_and_append_pdu( diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 360691ab..d7c24dcc 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -3,7 +3,8 @@ use crate::{ConduitResult, Database, Error, Ruma}; use ruma::{ api::client::r0::sync::sync_events, events::{room::member::MembershipState, AnySyncEphemeralRoomEvent, EventType}, - Raw, RoomId, UserId, + serde::Raw, + RoomId, UserId, }; #[cfg(feature = "conduit_bin")] diff --git a/src/database/account_data.rs b/src/database/account_data.rs index 9a6a050f..855ebfeb 100644 --- a/src/database/account_data.rs +++ b/src/database/account_data.rs @@ -2,7 +2,8 @@ use crate::{utils, Error, Result}; use ruma::{ api::client::error::ErrorKind, events::{AnyEvent as EduEvent, EventType}, - Raw, RoomId, UserId, + serde::Raw, + RoomId, UserId, }; use serde::{de::DeserializeOwned, Serialize}; use sled::IVec; diff --git a/src/database/admin.rs b/src/database/admin.rs index 87a60a14..10b62214 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -3,7 +3,10 @@ use std::convert::{TryFrom, TryInto}; use crate::pdu::PduBuilder; use log::warn; use rocket::futures::{channel::mpsc, stream::StreamExt}; -use ruma::{events::room::message, events::EventType, UserId}; +use ruma::{ + events::{room::message, EventType}, + UserId, +}; use tokio::select; pub enum AdminCommand { diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 0618fd6e..fb139a64 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -15,8 +15,8 @@ use ruma::{ }, EventType, }, - serde::{to_canonical_value, CanonicalJsonObject}, - EventId, Raw, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId, + serde::{to_canonical_value, CanonicalJsonObject, Raw}, + EventId, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId, }; use sled::IVec; use state_res::{event_auth, Error as StateError, Requester, StateEvent, StateMap, StateStore}; @@ -102,7 +102,7 @@ impl StateStore for Rooms { .and_then(|pdu: StateEvent| { // conduit's PDU's always contain a room_id but some // of ruma's do not so this must be an Option - if pdu.room_id() == Some(room_id) { + if pdu.room_id() == room_id { Ok(Arc::new(pdu)) } else { Err(StateError::NotFound( @@ -278,7 +278,7 @@ impl Rooms { for ((event_type, state_key), pdu_id) in state { let mut state_id = prefix.clone(); - state_id.extend_from_slice(&event_type.as_str().as_bytes()); + state_id.extend_from_slice(&event_type.as_ref().as_bytes()); state_id.push(0xff); state_id.extend_from_slice(&state_key.as_bytes()); self.stateid_pduid.insert(state_id, pdu_id)?; @@ -592,6 +592,7 @@ impl Rooms { body: format!("Command: {}, Args: {:?}", command, args), formatted: None, relates_to: None, + new_content: None, }, )); } @@ -633,7 +634,7 @@ impl Rooms { if let Some(state_key) = &new_pdu.state_key { let mut new_state = old_state; - let mut pdu_key = new_pdu.kind.as_str().as_bytes().to_vec(); + let mut pdu_key = new_pdu.kind.as_ref().as_bytes().to_vec(); pdu_key.push(0xff); pdu_key.extend_from_slice(state_key.as_bytes()); new_state.insert(pdu_key.into(), new_pdu_id.into()); diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index 29f5407b..bf0cdfc9 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -6,7 +6,8 @@ use ruma::{ AnyEvent as EduEvent, SyncEphemeralRoomEvent, }, presence::PresenceState, - Raw, RoomId, UserId, + serde::Raw, + RoomId, UserId, }; use std::{ collections::HashMap, diff --git a/src/database/users.rs b/src/database/users.rs index 885c0415..0421ae22 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -10,7 +10,8 @@ use ruma::{ }, encryption::DeviceKeys, events::{AnyToDeviceEvent, EventType}, - DeviceId, DeviceKeyAlgorithm, DeviceKeyId, Raw, UserId, + serde::Raw, + DeviceId, DeviceKeyAlgorithm, DeviceKeyId, UserId, }; use std::{collections::BTreeMap, convert::TryFrom, mem, time::SystemTime}; diff --git a/src/error.rs b/src/error.rs index 4c24fd75..316ca742 100644 --- a/src/error.rs +++ b/src/error.rs @@ -143,11 +143,7 @@ impl log::Log for ConduitLogger { } self.db.admin.send(AdminCommand::SendTextMessage( - message::TextMessageEventContent { - body: output, - formatted: None, - relates_to: None, - }, + message::TextMessageEventContent::plain(output), )); } } diff --git a/src/pdu.rs b/src/pdu.rs index e56e81ad..75ef4927 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -5,8 +5,8 @@ use ruma::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, - serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue}, - EventId, Raw, RoomId, RoomVersionId, ServerKeyId, ServerName, UserId, + serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue, Raw}, + EventId, RoomId, RoomVersionId, ServerName, ServerSigningKeyId, UserId, }; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -36,7 +36,7 @@ pub struct PduEvent { #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] pub unsigned: serde_json::Map, pub hashes: EventHash, - pub signatures: BTreeMap, BTreeMap>, + pub signatures: BTreeMap, BTreeMap>, } impl PduEvent { @@ -205,9 +205,10 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } + /// This does not return a full `Pdu` it is only to satisfy ruma's types. pub fn convert_to_outgoing_federation_event( mut pdu_json: CanonicalJsonObject, - ) -> Raw { + ) -> Raw { if let Some(CanonicalJsonValue::Object(unsigned)) = pdu_json.get_mut("unsigned") { unsigned.remove("transaction_id"); } @@ -232,7 +233,7 @@ impl From<&state_res::StateEvent> for PduEvent { fn from(pdu: &state_res::StateEvent) -> Self { Self { event_id: pdu.event_id(), - room_id: pdu.room_id().unwrap().clone(), + room_id: pdu.room_id().clone(), sender: pdu.sender().clone(), origin_server_ts: (pdu .origin_server_ts() @@ -288,7 +289,7 @@ impl PduEvent { /// /// Returns a tuple of the new `EventId` and the PDU with the eventId inserted as a `serde_json::Value`. pub(crate) fn process_incoming_pdu( - pdu: &ruma::Raw, + pdu: &Raw, ) -> (EventId, CanonicalJsonObject) { let mut value = serde_json::from_str(pdu.json().get()).expect("A Raw<...> is always valid JSON"); diff --git a/src/push_rules.rs b/src/push_rules.rs index 32c709e6..76a1a613 100644 --- a/src/push_rules.rs +++ b/src/push_rules.rs @@ -1,15 +1,18 @@ use ruma::{ push::{ - Action, ConditionalPushRule, ConditionalPushRuleInit, PatternedPushRule, - PatternedPushRuleInit, PushCondition, RoomMemberCountIs, Ruleset, Tweak, + Action, ConditionalPushRule, ConditionalPushRuleInit, ContentPushRule, OverridePushRule, + PatternedPushRule, PatternedPushRuleInit, PushCondition, RoomMemberCountIs, Ruleset, Tweak, + UnderridePushRule, }, UserId, }; pub fn default_pushrules(user_id: &UserId) -> Ruleset { let mut rules = Ruleset::default(); - rules.content = vec![contains_user_name_rule(&user_id)]; - rules.override_ = vec![ + + rules.add(ContentPushRule(contains_user_name_rule(&user_id))); + + for rule in vec![ master_rule(), suppress_notices_rule(), invite_for_me_rule(), @@ -17,14 +20,20 @@ pub fn default_pushrules(user_id: &UserId) -> Ruleset { contains_display_name_rule(), tombstone_rule(), roomnotif_rule(), - ]; - rules.underride = vec![ + ] { + rules.add(OverridePushRule(rule)); + } + + for rule in vec![ call_rule(), encrypted_room_one_to_one_rule(), room_one_to_one_rule(), message_rule(), encrypted_rule(), - ]; + ] { + rules.add(UnderridePushRule(rule)); + } + rules } diff --git a/src/server_server.rs b/src/server_server.rs index a7f6391d..fe36e7cd 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -11,7 +11,8 @@ use ruma::{ federation::{ directory::{get_public_rooms, get_public_rooms_filtered}, discovery::{ - get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, + get_server_keys, get_server_version::v1 as get_server_version, ServerSigningKeys, + VerifyKey, }, event::get_missing_events, query::get_profile_information, @@ -20,11 +21,11 @@ use ruma::{ OutgoingRequest, }, directory::{IncomingFilter, IncomingRoomNetwork}, - EventId, RoomId, ServerName, UserId, + EventId, RoomId, ServerName, ServerSigningKeyId, UserId, }; use std::{ collections::BTreeMap, - convert::{TryFrom, TryInto}, + convert::TryFrom, fmt::Debug, sync::Arc, time::{Duration, SystemTime}, @@ -243,16 +244,17 @@ pub fn get_server_keys(db: State<'_, Database>) -> Json { let mut verify_keys = BTreeMap::new(); verify_keys.insert( - format!("ed25519:{}", db.globals.keypair().version()) - .try_into() - .expect("DB stores valid ServerKeyId's"), + ServerSigningKeyId::try_from( + format!("ed25519:{}", db.globals.keypair().version()).as_str(), + ) + .expect("found invalid server signing keys in DB"), VerifyKey { key: base64::encode_config(db.globals.keypair().public_key(), base64::STANDARD_NO_PAD), }, ); let mut response = serde_json::from_slice( http::Response::try_from(get_server_keys::v2::Response { - server_key: ServerKey { + server_key: ServerSigningKeys { server_name: db.globals.server_name().to_owned(), verify_keys, old_verify_keys: BTreeMap::new(), @@ -430,7 +432,7 @@ pub async fn send_transaction_message_route<'a>( // would return a M_BAD_JSON error. let mut resolved_map = BTreeMap::new(); for pdu in &body.pdus { - // Ruma/PduEvent/StateEvent satifies - 1. Is a valid event, otherwise it is dropped. + // Ruma/PduEvent/StateEvent satisfies - 1. Is a valid event, otherwise it is dropped. // state-res checks signatures - 2. Passes signature checks, otherwise event is dropped. @@ -450,7 +452,7 @@ pub async fn send_transaction_message_route<'a>( continue; } - // If it is not a state event, we can skip state-res + // If it is not a state event, we can skip state-res... maybe if value.get("state_key").is_none() { if !db.rooms.is_joined(&pdu.sender, room_id)? { warn!("Sender is not joined {}", pdu.kind); @@ -679,7 +681,9 @@ pub fn get_profile_information_route<'a>( let mut displayname = None; let mut avatar_url = None; - match body.field { + match &body.field { + // TODO: what to do with custom + Some(ProfileField::_Custom(_s)) => {} Some(ProfileField::DisplayName) => displayname = db.users.displayname(&body.user_id)?, Some(ProfileField::AvatarUrl) => avatar_url = db.users.avatar_url(&body.user_id)?, None => { From 45086b54b35de7ec22beb6fbe5cdfed31465bc0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 5 Dec 2020 21:03:43 +0100 Subject: [PATCH 17/36] improvement: upgrade dependencies, fix timeline reload bug --- Cargo.lock | 168 ++++++++++++++++++++++++---------------- Cargo.toml | 28 +++---- src/database.rs | 63 +++++++++------ src/database/admin.rs | 2 - src/database/globals.rs | 37 +++------ src/main.rs | 14 ++-- src/ruma_wrapper.rs | 5 +- src/server_server.rs | 33 ++++++-- 8 files changed, 202 insertions(+), 148 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index efc34e4e..5062b8ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,9 +61,12 @@ dependencies = [ [[package]] name = "atomic" -version = "0.4.6" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f46ca51dca4837f1520754d1c8c36636356b81553d928dc9c177025369a06e" +checksum = "c3410529e8288c463bedb5930f82833bc0c90e5d2fe639a56582a4d09220b281" +dependencies = [ + "autocfg", +] [[package]] name = "atty" @@ -163,9 +166,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cc" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" [[package]] name = "cfg-if" @@ -192,15 +195,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "cloudabi" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" -dependencies = [ - "bitflags", -] - [[package]] name = "color_quant" version = "1.1.0" @@ -211,7 +205,7 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" name = "conduit" version = "0.1.0" dependencies = [ - "base64 0.12.3", + "base64 0.13.0", "directories", "http", "image", @@ -256,9 +250,8 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "cookie" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784ad0fbab4f3e9cef09f20e0aea6000ae08d2cb98ac4c0abc53df18803d702f" +version = "0.15.0-dev" +source = "git+https://github.com/SergioBenitez/cookie-rs.git?rev=1c3ca83#1c3ca838543b60a4448d279dc4b903cc7a2bc22a" dependencies = [ "percent-encoding", "time 0.2.23", @@ -328,7 +321,7 @@ dependencies = [ [[package]] name = "devise" version = "0.3.0" -source = "git+https://github.com/SergioBenitez/Devise.git?rev=1e42a2691#1e42a2691ef9934a446b8ed0ca1c4c8cf283f8bf" +source = "git+https://github.com/SergioBenitez/Devise.git?rev=3648468#3648468a9ede9ca896cd35bc1eb818c7a9fb3047" dependencies = [ "devise_codegen", "devise_core", @@ -337,7 +330,7 @@ dependencies = [ [[package]] name = "devise_codegen" version = "0.3.0" -source = "git+https://github.com/SergioBenitez/Devise.git?rev=1e42a2691#1e42a2691ef9934a446b8ed0ca1c4c8cf283f8bf" +source = "git+https://github.com/SergioBenitez/Devise.git?rev=3648468#3648468a9ede9ca896cd35bc1eb818c7a9fb3047" dependencies = [ "devise_core", "quote", @@ -346,7 +339,7 @@ dependencies = [ [[package]] name = "devise_core" version = "0.3.0" -source = "git+https://github.com/SergioBenitez/Devise.git?rev=1e42a2691#1e42a2691ef9934a446b8ed0ca1c4c8cf283f8bf" +source = "git+https://github.com/SergioBenitez/Devise.git?rev=3648468#3648468a9ede9ca896cd35bc1eb818c7a9fb3047" dependencies = [ "bitflags", "proc-macro2", @@ -408,6 +401,19 @@ dependencies = [ "syn", ] +[[package]] +name = "figment" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b13e2d266426f89e45fc544117ade84fad2a58ff676f34cc34e123fe4391b856" +dependencies = [ + "pear", + "serde", + "toml", + "uncased", + "version_check", +] + [[package]] name = "fnv" version = "1.0.7" @@ -846,9 +852,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" +checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" dependencies = [ "wasm-bindgen", ] @@ -1009,9 +1015,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.6.22" +version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" dependencies = [ "cfg-if 0.1.10", "fuchsia-zircon", @@ -1197,12 +1203,11 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" +checksum = "d7c6d9b8427445284a09c55be860a15855ab580a417ccad9da88f5a06787ced0" dependencies = [ - "cfg-if 0.1.10", - "cloudabi", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall", @@ -1218,8 +1223,9 @@ checksum = "7151b083b0664ed58ed669fcdd92f01c3d2fdbf10af4931a301474950b52bfa9" [[package]] name = "pear" -version = "0.2.0-dev" -source = "git+https://github.com/SergioBenitez/Pear.git?rev=4b68055#4b680556063568a42fcd4328335cdfdf7608be49" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f612cbd0f9dd03f5dd28a191c48e4148c3b027e41207b32eee130373c6c941" dependencies = [ "inlinable_string", "pear_codegen", @@ -1228,8 +1234,9 @@ dependencies = [ [[package]] name = "pear_codegen" -version = "0.2.0-dev" -source = "git+https://github.com/SergioBenitez/Pear.git?rev=4b68055#4b680556063568a42fcd4328335cdfdf7608be49" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602cf1780ee9bbca663ea75769e05643e16fe87d7c8ac9f4f385a2ed8940a75c" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", @@ -1357,8 +1364,9 @@ dependencies = [ [[package]] name = "proc-macro2-diagnostics" -version = "0.1.0" -source = "git+https://github.com/SergioBenitez/proc-macro2-diagnostics.git?rev=13fbb43#13fbb43db72034b6f9660a9b00e338cebd8dcf44" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf29726d67464d49fa6224a1d07936a8c08bb3fba727c7493f6cf1616fdaada" dependencies = [ "proc-macro2", "quote", @@ -1542,9 +1550,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.16.18" +version = "0.16.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70017ed5c555d79ee3538fc63ca09c70ad8f317dcadc1adc2c496b60c22bb24f" +checksum = "024a1e66fea74c66c66624ee5622a7ff0e4b73a13b4f5c326ddb50c708944226" dependencies = [ "cc", "libc", @@ -1558,24 +1566,28 @@ dependencies = [ [[package]] name = "rocket" version = "0.5.0-dev" -source = "git+https://github.com/timokoesters/Rocket.git?branch=empty_parameters#f6d40ecd5d871d97837b3116eb670fb3c06d95b9" +source = "git+https://github.com/SergioBenitez/Rocket.git?rev=1f1f44f336e5a172361fc1860461bb03667b1ed2#1f1f44f336e5a172361fc1860461bb03667b1ed2" dependencies = [ "async-trait", "atomic", "atty", "binascii", + "either", + "figment", "futures", "log", "memchr", "num_cpus", - "pear", + "parking_lot", + "rand", "ref-cast", "rocket_codegen", "rocket_http", + "serde", "state", "time 0.2.23", "tokio", - "toml", + "ubyte", "version_check", "yansi", ] @@ -1583,7 +1595,7 @@ dependencies = [ [[package]] name = "rocket_codegen" version = "0.5.0-dev" -source = "git+https://github.com/timokoesters/Rocket.git?branch=empty_parameters#f6d40ecd5d871d97837b3116eb670fb3c06d95b9" +source = "git+https://github.com/SergioBenitez/Rocket.git?rev=1f1f44f336e5a172361fc1860461bb03667b1ed2#1f1f44f336e5a172361fc1860461bb03667b1ed2" dependencies = [ "devise", "glob", @@ -1595,14 +1607,16 @@ dependencies = [ [[package]] name = "rocket_http" version = "0.5.0-dev" -source = "git+https://github.com/timokoesters/Rocket.git?branch=empty_parameters#f6d40ecd5d871d97837b3116eb670fb3c06d95b9" +source = "git+https://github.com/SergioBenitez/Rocket.git?rev=1f1f44f336e5a172361fc1860461bb03667b1ed2#1f1f44f336e5a172361fc1860461bb03667b1ed2" dependencies = [ "cookie", + "either", "http", "hyper", "indexmap", "log", "mime", + "parking_lot", "pear", "percent-encoding", "ref-cast", @@ -1611,7 +1625,9 @@ dependencies = [ "time 0.2.23", "tokio", "tokio-rustls", + "uncased", "unicode-xid", + "version_check", ] [[package]] @@ -1952,9 +1968,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.59" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" +checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779" dependencies = [ "itoa", "ryu", @@ -2022,9 +2038,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acad6f34eb9e8a259d3283d1e8c1d34d7415943d4895f65cc73813c7396fc85" +checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" [[package]] name = "socket2" @@ -2480,6 +2496,24 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "ubyte" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42756bb9e708855de2f8a98195643dff31a97f0485d90d8467b39dc24be9e8fe" +dependencies = [ + "serde", +] + +[[package]] +name = "uncased" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "369fa7fd7969c5373541d3c9a40dc1b76ce676fc87aba30d87c0ad3b97fad179" +dependencies = [ + "version_check", +] + [[package]] name = "unicase" version = "2.6.0" @@ -2573,11 +2607,11 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "serde", "serde_json", "wasm-bindgen-macro", @@ -2585,9 +2619,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" dependencies = [ "bumpalo", "lazy_static", @@ -2600,11 +2634,11 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" +checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -2612,9 +2646,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2622,9 +2656,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" dependencies = [ "proc-macro2", "quote", @@ -2635,15 +2669,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" [[package]] name = "wasm-bindgen-test" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34d1cdc8b98a557f24733d50a1199c4b0635e465eecba9c45b214544da197f64" +checksum = "0355fa0c1f9b792a09b6dcb6a8be24d51e71e6d74972f9eb4a44c4c004d24a25" dependencies = [ "console_error_panic_hook", "js-sys", @@ -2655,9 +2689,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8fb9c67be7439ee8ab1b7db502a49c05e51e2835b66796c705134d9b8e1a585" +checksum = "27e07b46b98024c2ba2f9e83a10c2ef0515f057f2da299c1762a2017de80438b" dependencies = [ "proc-macro2", "quote", @@ -2665,9 +2699,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" +checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" dependencies = [ "js-sys", "wasm-bindgen", @@ -2675,9 +2709,9 @@ dependencies = [ [[package]] name = "webpki" -version = "0.21.3" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab146130f5f790d45f82aeeb09e55a256573373ec64409fc19a6fb82fb1032ae" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" dependencies = [ "ring", "untrusted", diff --git a/Cargo.toml b/Cargo.toml index 4902c7c9..b1dec17f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,8 @@ edition = "2018" [dependencies] # Used to handle requests # TODO: This can become optional as soon as proper configs are supported -#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", default-features = false, features = ["tls"] } # Used to handle requests -rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] } +rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "1f1f44f336e5a172361fc1860461bb03667b1ed2", features = ["tls"] } # Used to handle requests +#rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] } # Used for matrix spec type definitions and helpers ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks", "unstable-exhaustive-types"], rev = "e8882fe8142d7b55ed4c8ccc6150946945f9e237" } @@ -27,10 +27,10 @@ ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec", "gen-eventid"] } # state-res = { path = "../../state-res", features = ["unstable-pre-spec", "gen-eventid"] } -# Used for long polling -tokio = "0.2.22" +# Used for long polling and federation sender, should be the same as rocket::tokio +tokio = { version = "0.2.23" } # Used for storing data permanently -sled = { version = "0.34.4", default-features = false } +sled = { version = "0.34.6", default-features = false } # Used for emitting log entries log = "0.4.11" # Used for rocket<->ruma conversions @@ -40,25 +40,25 @@ directories = "3.0.1" # Used for number types for ruma js_int = "0.1.9" # Used for ruma wrapper -serde_json = { version = "1.0.57", features = ["raw_value"] } +serde_json = { version = "1.0.60", features = ["raw_value"] } # Used for pdu definition -serde = "1.0.116" +serde = "1.0.117" # Used for secure identifiers rand = "0.7.3" # Used to hash passwords -rust-argon2 = "0.8.2" +rust-argon2 = "0.8.3" # Used to send requests -reqwest = "0.10.8" +reqwest = "0.10.9" # Used for conduit::Error type -thiserror = "1.0.20" +thiserror = "1.0.22" # Used to generate thumbnails for images -image = { version = "0.23.9", default-features = false, features = ["jpeg", "png", "gif"] } +image = { version = "0.23.12", default-features = false, features = ["jpeg", "png", "gif"] } # Used to encode server public key -base64 = "0.12.3" +base64 = "0.13.0" # Used when hashing the state -ring = "0.16.15" +ring = "0.16.19" # Used when querying the SRV record of other servers -trust-dns-resolver = "0.19.5" +trust-dns-resolver = "0.19.6" [features] default = ["conduit_bin"] diff --git a/src/database.rs b/src/database.rs index 576a250e..406ce77c 100644 --- a/src/database.rs +++ b/src/database.rs @@ -13,12 +13,39 @@ use crate::{Error, Result}; use directories::ProjectDirs; use futures::StreamExt; use log::info; -use rocket::{ - futures::{self, channel::mpsc}, - Config, -}; -use ruma::{DeviceId, UserId}; -use std::{convert::TryFrom, fs::remove_dir_all}; +use rocket::futures::{self, channel::mpsc}; +use ruma::{DeviceId, ServerName, UserId}; +use serde::Deserialize; +use std::{convert::TryInto, fs::remove_dir_all}; + +#[derive(Clone, Deserialize)] +pub struct Config { + #[serde(default = "default_server_name")] + server_name: Box, + database_path: Option, + #[serde(default = "default_cache_capacity")] + cache_capacity: u64, + #[serde(default = "default_max_request_size")] + max_request_size: u32, + #[serde(default)] + registration_disabled: bool, + #[serde(default)] + encryption_disabled: bool, + #[serde(default)] + federation_enabled: bool, +} + +fn default_server_name() -> Box { + "localhost".try_into().expect("") +} + +fn default_cache_capacity() -> u64 { + 1024 * 1024 * 1024 +} + +fn default_max_request_size() -> u32 { + 20 * 1024 * 1024 // Default to 20 MB +} #[derive(Clone)] pub struct Database { @@ -49,19 +76,18 @@ impl Database { } /// Load an existing database or create a new one. - pub fn load_or_create(config: &Config) -> Result { - let server_name = config.get_str("server_name").unwrap_or("localhost"); - + pub fn load_or_create(config: Config) -> Result { let path = config - .get_str("database_path") - .map(|x| Ok::<_, Error>(x.to_owned())) - .unwrap_or_else(|_| { + .database_path + .clone() + .map(Ok::<_, Error>) + .unwrap_or_else(|| { let path = ProjectDirs::from("xyz", "koesters", "conduit") .ok_or_else(|| { Error::bad_config("The OS didn't return a valid home directory path.") })? .data_dir() - .join(server_name); + .join(config.server_name.as_str()); Ok(path .to_str() @@ -71,15 +97,8 @@ impl Database { let db = sled::Config::default() .path(&path) - .cache_capacity( - u64::try_from( - config - .get_int("cache_capacity") - .unwrap_or(1024 * 1024 * 1024), - ) - .map_err(|_| Error::bad_config("Cache capacity needs to be a u64."))?, - ) - .print_profile_on_drop(false) + .cache_capacity(config.cache_capacity) + .print_profile_on_drop(true) .open()?; info!("Opened sled database at {}", path); diff --git a/src/database/admin.rs b/src/database/admin.rs index 10b62214..778796f7 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -49,8 +49,6 @@ impl Admin { Some(event) = receiver.next() => { match event { AdminCommand::SendTextMessage(message) => { - println!("{:?}", message); - if let Some(conduit_room) = &conduit_room { db.rooms.build_and_append_pdu( PduBuilder { diff --git a/src/database/globals.rs b/src/database/globals.rs index 359d0643..403fadd0 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -1,7 +1,7 @@ -use crate::{utils, Error, Result}; +use crate::{database::Config, utils, Error, Result}; use log::error; use ruma::ServerName; -use std::{convert::TryInto, sync::Arc}; +use std::sync::Arc; pub const COUNTER: &str = "c"; @@ -10,15 +10,11 @@ pub struct Globals { pub(super) globals: sled::Tree, keypair: Arc, reqwest_client: reqwest::Client, - server_name: Box, - max_request_size: u32, - registration_disabled: bool, - encryption_disabled: bool, - federation_enabled: bool, + config: Config, } impl Globals { - pub fn load(globals: sled::Tree, config: &rocket::Config) -> Result { + pub fn load(globals: sled::Tree, config: Config) -> Result { let bytes = &*globals .update_and_fetch("keypair", utils::generate_keypair)? .expect("utils::generate_keypair always returns Some"); @@ -57,20 +53,7 @@ impl Globals { globals, keypair: Arc::new(keypair), reqwest_client: reqwest::Client::new(), - server_name: config - .get_str("server_name") - .unwrap_or("localhost") - .to_string() - .try_into() - .map_err(|_| Error::bad_config("Invalid server_name."))?, - max_request_size: config - .get_int("max_request_size") - .unwrap_or(20 * 1024 * 1024) // Default to 20 MB - .try_into() - .map_err(|_| Error::bad_config("Invalid max_request_size."))?, - registration_disabled: config.get_bool("registration_disabled").unwrap_or(false), - encryption_disabled: config.get_bool("encryption_disabled").unwrap_or(false), - federation_enabled: config.get_bool("federation_enabled").unwrap_or(false), + config, }) } @@ -102,22 +85,22 @@ impl Globals { } pub fn server_name(&self) -> &ServerName { - self.server_name.as_ref() + self.config.server_name.as_ref() } pub fn max_request_size(&self) -> u32 { - self.max_request_size + self.config.max_request_size } pub fn registration_disabled(&self) -> bool { - self.registration_disabled + self.config.registration_disabled } pub fn encryption_disabled(&self) -> bool { - self.encryption_disabled + self.config.encryption_disabled } pub fn federation_enabled(&self) -> bool { - self.federation_enabled + self.config.federation_enabled } } diff --git a/src/main.rs b/src/main.rs index f2edc137..75b74ccf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,7 +21,7 @@ use rocket::{fairing::AdHoc, routes}; fn setup_rocket() -> rocket::Rocket { // Force log level off, so we can use our own logger - std::env::set_var("ROCKET_LOG", "off"); + std::env::set_var("ROCKET_LOG_LEVEL", "off"); rocket::ignite() .mount( @@ -123,9 +123,9 @@ fn setup_rocket() -> rocket::Rocket { client_server::get_pushers_route, client_server::set_pushers_route, client_server::upgrade_room_route, - server_server::get_server_version, - server_server::get_server_keys, - server_server::get_server_keys_deprecated, + server_server::get_server_version_route, + server_server::get_server_keys_route, + server_server::get_server_keys_deprecated_route, server_server::get_public_rooms_route, server_server::get_public_rooms_filtered_route, server_server::send_transaction_message_route, @@ -133,8 +133,10 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_profile_information_route, ], ) - .attach(AdHoc::on_attach("Config", |mut rocket| async { - let data = Database::load_or_create(rocket.config().await).expect("valid config"); + .attach(AdHoc::on_attach("Config", |rocket| async { + let data = + Database::load_or_create(rocket.figment().extract().expect("config is valid")) + .expect("config is valid"); data.sending.start_handler(&data.globals, &data.rooms); log::set_boxed_logger(Box::new(ConduitLogger { diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 4b3d08d5..9597ac85 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -15,7 +15,8 @@ use { log::warn, rocket::{ data::{ - Data, FromDataFuture, FromTransformedData, Transform, TransformFuture, Transformed, + ByteUnit, Data, FromDataFuture, FromTransformedData, Transform, TransformFuture, + Transformed, }, http::Status, outcome::Outcome::*, @@ -97,7 +98,7 @@ where } let limit = db.globals.max_request_size(); - let mut handle = data.open().take(limit.into()); + let mut handle = data.open(ByteUnit::Byte(limit.into())); let mut body = Vec::new(); handle.read_to_end(&mut body).await.unwrap(); diff --git a/src/server_server.rs b/src/server_server.rs index fe36e7cd..da046d30 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -193,6 +193,8 @@ where } } + let status = reqwest_response.status(); + let body = reqwest_response .bytes() .await @@ -201,17 +203,27 @@ where Vec::new().into() }) // TODO: handle timeout .into_iter() - .collect(); + .collect::>(); + + if status != 200 { + warn!( + "Server returned bad response {} ({}): {} {:?}", + destination, + url, + status, + utils::string_from_bytes(&body) + ); + } let response = T::IncomingResponse::try_from( http_response .body(body) .expect("reqwest body is valid http body"), ); - response.map_err(|e| { + response.map_err(|_| { warn!( - "Server returned bad response {} ({}): {:?}", - destination, url, e + "Server returned invalid response bytes {} ({})", + destination, url ); Error::BadServerResponse("Server returned bad response.") }) @@ -221,7 +233,9 @@ where } #[cfg_attr(feature = "conduit_bin", get("/_matrix/federation/v1/version"))] -pub fn get_server_version(db: State<'_, Database>) -> ConduitResult { +pub fn get_server_version_route( + db: State<'_, Database>, +) -> ConduitResult { if !db.globals.federation_enabled() { return Err(Error::bad_config("Federation is disabled.")); } @@ -236,7 +250,7 @@ pub fn get_server_version(db: State<'_, Database>) -> ConduitResult) -> Json { +pub fn get_server_keys_route(db: State<'_, Database>) -> Json { if !db.globals.federation_enabled() { // TODO: Use proper types return Json("Federation is disabled.".to_owned()); @@ -278,8 +292,8 @@ pub fn get_server_keys(db: State<'_, Database>) -> Json { } #[cfg_attr(feature = "conduit_bin", get("/_matrix/key/v2/server/<_>"))] -pub fn get_server_keys_deprecated(db: State<'_, Database>) -> Json { - get_server_keys(db) +pub fn get_server_keys_deprecated_route(db: State<'_, Database>) -> Json { + get_server_keys_route(db) } #[cfg_attr( @@ -464,6 +478,9 @@ pub async fn send_transaction_message_route<'a>( let mut pdu_id = room_id.as_bytes().to_vec(); pdu_id.push(0xff); pdu_id.extend_from_slice(&count.to_be_bytes()); + + db.rooms.append_to_state(&pdu_id, &pdu)?; + db.rooms.append_pdu( &pdu, &value, From d62f17a91a6647fd7b5bd503ddcd08e6a7e3e951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 6 Dec 2020 11:05:51 +0100 Subject: [PATCH 18/36] improvement: cache actual destination --- src/database.rs | 4 +- src/database/globals.rs | 19 +++++++-- src/main.rs | 1 + src/server_server.rs | 87 ++++++++++++++++++++++++++--------------- 4 files changed, 75 insertions(+), 36 deletions(-) diff --git a/src/database.rs b/src/database.rs index 406ce77c..4905070a 100644 --- a/src/database.rs +++ b/src/database.rs @@ -76,7 +76,7 @@ impl Database { } /// Load an existing database or create a new one. - pub fn load_or_create(config: Config) -> Result { + pub async fn load_or_create(config: Config) -> Result { let path = config .database_path .clone() @@ -106,7 +106,7 @@ impl Database { let (admin_sender, admin_receiver) = mpsc::unbounded(); let db = Self { - globals: globals::Globals::load(db.open_tree("global")?, config)?, + globals: globals::Globals::load(db.open_tree("global")?, config).await?, users: users::Users { userid_password: db.open_tree("userid_password")?, userid_displayname: db.open_tree("userid_displayname")?, diff --git a/src/database/globals.rs b/src/database/globals.rs index 403fadd0..1221609d 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -1,20 +1,25 @@ use crate::{database::Config, utils, Error, Result}; +use trust_dns_resolver::TokioAsyncResolver; +use std::collections::HashMap; use log::error; use ruma::ServerName; use std::sync::Arc; +use std::sync::RwLock; pub const COUNTER: &str = "c"; #[derive(Clone)] pub struct Globals { pub(super) globals: sled::Tree, + config: Config, keypair: Arc, reqwest_client: reqwest::Client, - config: Config, + pub actual_destination_cache: Arc, (String, Option)>>>, // actual_destination, host + dns_resolver: TokioAsyncResolver, } impl Globals { - pub fn load(globals: sled::Tree, config: Config) -> Result { + pub async fn load(globals: sled::Tree, config: Config) -> Result { let bytes = &*globals .update_and_fetch("keypair", utils::generate_keypair)? .expect("utils::generate_keypair always returns Some"); @@ -51,9 +56,13 @@ impl Globals { Ok(Self { globals, + config, keypair: Arc::new(keypair), reqwest_client: reqwest::Client::new(), - config, + dns_resolver: TokioAsyncResolver::tokio_from_system_conf().await.map_err(|_| { + Error::bad_config("Failed to set up trust dns resolver with system config.") + })?, + actual_destination_cache: Arc::new(RwLock::new(HashMap::new())), }) } @@ -103,4 +112,8 @@ impl Globals { pub fn federation_enabled(&self) -> bool { self.config.federation_enabled } + + pub fn dns_resolver(&self) -> &TokioAsyncResolver { + &self.dns_resolver + } } diff --git a/src/main.rs b/src/main.rs index 75b74ccf..58d3427f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -136,6 +136,7 @@ fn setup_rocket() -> rocket::Rocket { .attach(AdHoc::on_attach("Config", |rocket| async { let data = Database::load_or_create(rocket.figment().extract().expect("config is valid")) + .await .expect("config is valid"); data.sending.start_handler(&data.globals, &data.rooms); diff --git a/src/server_server.rs b/src/server_server.rs index da046d30..58dd872e 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -30,7 +30,6 @@ use std::{ sync::Arc, time::{Duration, SystemTime}, }; -use trust_dns_resolver::AsyncResolver; pub async fn request_well_known( globals: &crate::database::globals::Globals, @@ -66,36 +65,26 @@ where return Err(Error::bad_config("Federation is disabled.")); } - let resolver = AsyncResolver::tokio_from_system_conf().await.map_err(|_| { - Error::bad_config("Failed to set up trust dns resolver with system config.") - })?; - - let mut host = None; - - let actual_destination = "https://".to_owned() - + &if let Some(mut delegated_hostname) = - request_well_known(globals, &destination.as_str()).await - { - if let Ok(Some(srv)) = resolver - .srv_lookup(format!("_matrix._tcp.{}", delegated_hostname)) - .await - .map(|srv| srv.iter().next().map(|result| result.target().to_string())) - { - host = Some(delegated_hostname); - srv.trim_end_matches('.').to_owned() - } else { - if delegated_hostname.find(':').is_none() { - delegated_hostname += ":8448"; - } - delegated_hostname - } - } else { - let mut destination = destination.as_str().to_owned(); - if destination.find(':').is_none() { - destination += ":8448"; - } - destination - }; + let maybe_result = globals + .actual_destination_cache + .read() + .unwrap() + .get(&destination) + .cloned(); + + let (actual_destination, host) = if let Some(result) = maybe_result { + println!("Loaded {} -> {:?}", destination, result); + result + } else { + let result = find_actual_destination(globals, &destination).await; + globals + .actual_destination_cache + .write() + .unwrap() + .insert(destination.clone(), result.clone()); + println!("Saving {} -> {:?}", destination, result); + result + }; let mut http_request = request .try_into_http_request(&actual_destination, Some("")) @@ -232,6 +221,42 @@ where } } +/// Returns: actual_destination, host header +async fn find_actual_destination( + globals: &crate::database::globals::Globals, + destination: &Box, +) -> (String, Option) { + let mut host = None; + + let actual_destination = "https://".to_owned() + + &if let Some(mut delegated_hostname) = + request_well_known(globals, destination.as_str()).await + { + if let Ok(Some(srv)) = globals + .dns_resolver() + .srv_lookup(format!("_matrix._tcp.{}", delegated_hostname)) + .await + .map(|srv| srv.iter().next().map(|result| result.target().to_string())) + { + host = Some(delegated_hostname); + srv.trim_end_matches('.').to_owned() + } else { + if delegated_hostname.find(':').is_none() { + delegated_hostname += ":8448"; + } + delegated_hostname + } + } else { + let mut destination = destination.as_str().to_owned(); + if destination.find(':').is_none() { + destination += ":8448"; + } + destination + }; + + (actual_destination, host) +} + #[cfg_attr(feature = "conduit_bin", get("/_matrix/federation/v1/version"))] pub fn get_server_version_route( db: State<'_, Database>, From 6e5b35ea92075cdc9fc62db0f7f946ae6b80d76e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 8 Dec 2020 10:33:44 +0100 Subject: [PATCH 19/36] feat: implement appservices this also reverts some stateres changes --- Cargo.lock | 247 ++++++++------------------------ Cargo.toml | 8 +- src/appservice_server.rs | 104 ++++++++++++++ src/client_server/account.rs | 35 +++-- src/client_server/alias.rs | 41 ++++-- src/client_server/media.rs | 2 +- src/client_server/membership.rs | 6 + src/client_server/message.rs | 3 +- src/client_server/profile.rs | 2 + src/client_server/redact.rs | 1 + src/client_server/room.rs | 15 ++ src/client_server/state.rs | 3 +- src/client_server/to_device.rs | 2 +- src/database.rs | 8 ++ src/database/admin.rs | 61 +++++--- src/database/appservice.rs | 67 +++++++++ src/database/globals.rs | 12 +- src/database/media.rs | 7 +- src/database/rooms.rs | 122 +++++++--------- src/database/sending.rs | 240 +++++++++++++++++++++---------- src/database/transaction_ids.rs | 8 +- src/error.rs | 4 +- src/lib.rs | 1 + src/main.rs | 4 +- src/ruma_wrapper.rs | 90 +++++++++--- src/server_server.rs | 189 +++--------------------- 26 files changed, 697 insertions(+), 585 deletions(-) create mode 100644 src/appservice_server.rs create mode 100644 src/database/appservice.rs diff --git a/Cargo.lock b/Cargo.lock index 5062b8ce..6566b108 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,15 +21,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi 0.3.9", -] - [[package]] name = "arrayref" version = "0.3.6" @@ -182,19 +173,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "chrono" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" -dependencies = [ - "libc", - "num-integer", - "num-traits", - "time 0.1.44", - "winapi 0.3.9", -] - [[package]] name = "color_quant" version = "1.1.0" @@ -219,6 +197,7 @@ dependencies = [ "rust-argon2", "serde", "serde_json", + "serde_yaml", "sled", "state-res", "thiserror", @@ -254,7 +233,7 @@ version = "0.15.0-dev" source = "git+https://github.com/SergioBenitez/cookie-rs.git?rev=1c3ca83#1c3ca838543b60a4448d279dc4b903cc7a2bc22a" dependencies = [ "percent-encoding", - "time 0.2.23", + "time", "version_check", ] @@ -374,6 +353,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "dtoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" + [[package]] name = "either" version = "1.6.1" @@ -575,19 +560,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "generator" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cdc09201b2e8ca1b19290cf7e65de2246b8e91fb6874279722189c4de7b94dc" -dependencies = [ - "cc", - "libc", - "log", - "rustc_version", - "winapi 0.3.9", -] - [[package]] name = "getrandom" version = "0.1.15" @@ -596,7 +568,7 @@ checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ "cfg-if 0.1.10", "libc", - "wasi 0.9.0+wasi-snapshot-preview1", + "wasi", ] [[package]] @@ -886,9 +858,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" [[package]] name = "linked-hash-map" @@ -914,19 +886,6 @@ dependencies = [ "cfg-if 0.1.10", ] -[[package]] -name = "loom" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0e8460f2f2121162705187214720353c517b97bdfb3494c0b1e33d83ebe4bed" -dependencies = [ - "cfg-if 0.1.10", - "generator", - "scoped-tls", - "serde", - "serde_json", -] - [[package]] name = "lru-cache" version = "0.1.2" @@ -948,15 +907,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" -[[package]] -name = "matchers" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" -dependencies = [ - "regex-automata", -] - [[package]] name = "matches" version = "0.1.8" @@ -1468,31 +1418,6 @@ dependencies = [ "syn", ] -[[package]] -name = "regex" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" -dependencies = [ - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" -dependencies = [ - "byteorder", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.6.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" - [[package]] name = "remove_dir_all" version = "0.5.3" @@ -1585,7 +1510,7 @@ dependencies = [ "rocket_http", "serde", "state", - "time 0.2.23", + "time", "tokio", "ubyte", "version_check", @@ -1622,7 +1547,7 @@ dependencies = [ "ref-cast", "smallvec", "state", - "time 0.2.23", + "time", "tokio", "tokio-rustls", "uncased", @@ -1633,7 +1558,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "assign", "js_int", @@ -1651,7 +1576,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "http", "percent-encoding", @@ -1666,7 +1591,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1677,7 +1602,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "ruma-api", "ruma-common", @@ -1691,7 +1616,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "assign", "http", @@ -1710,7 +1635,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "js_int", "maplit", @@ -1723,7 +1648,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "js_int", "ruma-common", @@ -1737,7 +1662,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1748,7 +1673,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "js_int", "ruma-api", @@ -1763,7 +1688,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "paste", "rand", @@ -1777,7 +1702,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "proc-macro2", "quote", @@ -1788,7 +1713,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "serde", ] @@ -1796,7 +1721,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "form_urlencoded", "itoa", @@ -1809,7 +1734,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1820,7 +1745,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/ruma/ruma?rev=e8882fe8142d7b55ed4c8ccc6150946945f9e237#e8882fe8142d7b55ed4c8ccc6150946945f9e237" +source = "git+https://github.com/ruma/ruma?rev=ee814aa84934530d76f5e4b275d739805b49bdef#ee814aa84934530d76f5e4b275d739805b49bdef" dependencies = [ "base64 0.12.3", "ring", @@ -1948,18 +1873,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" +checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" +checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" dependencies = [ "proc-macro2", "quote", @@ -1990,20 +1915,22 @@ dependencies = [ ] [[package]] -name = "sha1" -version = "0.6.0" +name = "serde_yaml" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +checksum = "f7baae0a99f1a324984bcdc5f0718384c1f69775f1c7eec8b859b71b443e3fd7" +dependencies = [ + "dtoa", + "linked-hash-map", + "serde", + "yaml-rust", +] [[package]] -name = "sharded-slab" -version = "0.1.0" +name = "sha1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4921be914e16899a80adefb821f8ddb7974e3f1250223575a44ed994882127" -dependencies = [ - "lazy_static", - "loom", -] +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "signal-hook-registry" @@ -2078,17 +2005,15 @@ checksum = "3015a7d0a5fd5105c91c3710d42f9ccf0abfb287d62206484dcc67f9569a6483" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res?branch=timo-spec-comp#99214e6fa6b9843b0d9e1f6ef0698d7fdb234fb2" +source = "git+https://github.com/ruma/state-res?branch=timo-spec-comp#a1c15253f0777baad251da47c3f2c016cfed6f7e" dependencies = [ "itertools", - "js_int", "maplit", "ruma", "serde", "serde_json", "thiserror", "tracing", - "tracing-subscriber", ] [[package]] @@ -2163,9 +2088,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.53" +version = "1.0.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8833e20724c24de12bbaba5ad230ea61c3eafb05b881c7c9d3cfe8638b187e68" +checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44" dependencies = [ "proc-macro2", "quote", @@ -2206,26 +2131,6 @@ dependencies = [ "syn", ] -[[package]] -name = "thread_local" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "time" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi 0.3.9", -] - [[package]] name = "time" version = "0.2.23" @@ -2407,49 +2312,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tracing-log" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e0f8c7178e13481ff6765bd169b33e8d554c5d2bbede5e32c356194be02b9b9" -dependencies = [ - "lazy_static", - "log", - "tracing-core", -] - -[[package]] -name = "tracing-serde" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" -dependencies = [ - "serde", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1fa8f0c8f4c594e4fc9debc1990deab13238077271ba84dd853d54902ee3401" -dependencies = [ - "ansi_term", - "chrono", - "lazy_static", - "matchers", - "regex", - "serde", - "serde_json", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", - "tracing-serde", -] - [[package]] name = "trust-dns-proto" version = "0.19.6" @@ -2599,12 +2461,6 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasm-bindgen" version = "0.2.69" @@ -2791,6 +2647,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "yaml-rust" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39f0c922f1a334134dc2f7a8b67dc5d25f0735263feec974345ff706bcf20b0d" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "yansi" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index b1dec17f..1e4afe2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,14 +18,14 @@ rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "1f1f44f33 #rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] } # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks", "unstable-exhaustive-types"], rev = "e8882fe8142d7b55ed4c8ccc6150946945f9e237" } +ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks", "unstable-exhaustive-types"], rev = "ee814aa84934530d76f5e4b275d739805b49bdef" } # ruma = { git = "https://github.com/DevinR528/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "unstable-join" } # ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } # Used when doing state resolution -# state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] } +# state-res = { git = "https://github.com/timokoesters/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec"] } state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec", "gen-eventid"] } -# state-res = { path = "../../state-res", features = ["unstable-pre-spec", "gen-eventid"] } +#state-res = { path = "../state-res", features = ["unstable-pre-spec", "gen-eventid"] } # Used for long polling and federation sender, should be the same as rocket::tokio tokio = { version = "0.2.23" } @@ -41,6 +41,8 @@ directories = "3.0.1" js_int = "0.1.9" # Used for ruma wrapper serde_json = { version = "1.0.60", features = ["raw_value"] } +# Used for appservice registration files +serde_yaml = "0.8.14" # Used for pdu definition serde = "1.0.117" # Used for secure identifiers diff --git a/src/appservice_server.rs b/src/appservice_server.rs new file mode 100644 index 00000000..f1436e0a --- /dev/null +++ b/src/appservice_server.rs @@ -0,0 +1,104 @@ +use crate::{utils, Error, Result}; +use http::header::{HeaderValue, CONTENT_TYPE}; +use log::warn; +use ruma::api::OutgoingRequest; +use std::{ + convert::{TryFrom, TryInto}, + fmt::Debug, + time::Duration, +}; + +pub async fn send_request( + globals: &crate::database::globals::Globals, + registration: serde_yaml::Value, + request: T, +) -> Result +where + T: Debug, +{ + let destination = registration.get("url").unwrap().as_str().unwrap(); + let hs_token = registration.get("hs_token").unwrap().as_str().unwrap(); + + let mut http_request = request + .try_into_http_request(&destination, Some("")) + .unwrap(); + + let mut parts = http_request.uri().clone().into_parts(); + let old_path_and_query = parts.path_and_query.unwrap().as_str().to_owned(); + let symbol = if old_path_and_query.contains("?") { + "&" + } else { + "?" + }; + + parts.path_and_query = Some( + (old_path_and_query + symbol + "access_token=" + hs_token) + .parse() + .unwrap(), + ); + *http_request.uri_mut() = parts.try_into().expect("our manipulation is always valid"); + + http_request.headers_mut().insert( + CONTENT_TYPE, + HeaderValue::from_str("application/json").unwrap(), + ); + + let mut reqwest_request = reqwest::Request::try_from(http_request) + .expect("all http requests are valid reqwest requests"); + + *reqwest_request.timeout_mut() = Some(Duration::from_secs(30)); + + let url = reqwest_request.url().clone(); + let reqwest_response = globals.reqwest_client().execute(reqwest_request).await; + + // Because reqwest::Response -> http::Response is complicated: + match reqwest_response { + Ok(mut reqwest_response) => { + let status = reqwest_response.status(); + let mut http_response = http::Response::builder().status(status); + let headers = http_response.headers_mut().unwrap(); + + for (k, v) in reqwest_response.headers_mut().drain() { + if let Some(key) = k { + headers.insert(key, v); + } + } + + let status = reqwest_response.status(); + + let body = reqwest_response + .bytes() + .await + .unwrap_or_else(|e| { + warn!("server error: {}", e); + Vec::new().into() + }) // TODO: handle timeout + .into_iter() + .collect::>(); + + if status != 200 { + warn!( + "Server returned bad response {} ({}): {} {:?}", + destination, + url, + status, + utils::string_from_bytes(&body) + ); + } + + let response = T::IncomingResponse::try_from( + http_response + .body(body) + .expect("reqwest body is valid http body"), + ); + response.map_err(|_| { + warn!( + "Server returned invalid response bytes {} ({})", + destination, url + ); + Error::BadServerResponse("Server returned bad response.") + }) + } + Err(e) => Err(e.into()), + } +} diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 76bbebb5..8fb926e2 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -139,18 +139,20 @@ pub async fn register_route( auth_error: None, }; - if let Some(auth) = &body.auth { - let (worked, uiaainfo) = - db.uiaa - .try_auth(&user_id, "".into(), auth, &uiaainfo, &db.users, &db.globals)?; - if !worked { + if !body.from_appservice { + if let Some(auth) = &body.auth { + let (worked, uiaainfo) = + db.uiaa + .try_auth(&user_id, "".into(), auth, &uiaainfo, &db.users, &db.globals)?; + if !worked { + return Err(Error::Uiaa(uiaainfo)); + } + // Success! + } else { + uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); + db.uiaa.create(&user_id, "".into(), &uiaainfo)?; return Err(Error::Uiaa(uiaainfo)); } - // Success! - } else { - uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - db.uiaa.create(&user_id, "".into(), &uiaainfo)?; - return Err(Error::Uiaa(uiaainfo)); } if missing_username { @@ -241,6 +243,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 2. Make conduit bot join @@ -265,6 +268,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 3. Power levels @@ -302,6 +306,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 4.1 Join Rules @@ -322,6 +327,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 4.2 History Visibility @@ -344,6 +350,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 4.3 Guest Access @@ -364,6 +371,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 6. Events implied by name and topic @@ -386,6 +394,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; db.rooms.build_and_append_pdu( @@ -405,6 +414,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // Room alias @@ -430,6 +440,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?; @@ -456,6 +467,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; db.rooms.build_and_append_pdu( PduBuilder { @@ -478,6 +490,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // Send welcome message @@ -506,6 +519,7 @@ pub async fn register_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; } @@ -681,6 +695,7 @@ pub async fn deactivate_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; } diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 094e70a2..ec73ffc4 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -1,7 +1,8 @@ use super::State; -use crate::{server_server, ConduitResult, Database, Error, Ruma}; +use crate::{appservice_server, server_server, ConduitResult, Database, Error, Ruma}; use ruma::{ api::{ + appservice, client::{ error::ErrorKind, r0::alias::{create_alias, delete_alias, get_alias}, @@ -75,13 +76,37 @@ pub async fn get_alias_helper( return Ok(get_alias::Response::new(response.room_id, response.servers).into()); } - let room_id = db - .rooms - .id_from_alias(&room_alias)? - .ok_or(Error::BadRequest( - ErrorKind::NotFound, - "Room with alias not found.", - ))?; + let mut room_id = None; + match db.rooms.id_from_alias(&room_alias)? { + Some(r) => room_id = Some(r), + None => { + for (_id, registration) in db.appservice.iter_all().filter_map(|r| r.ok()) { + if appservice_server::send_request( + &db.globals, + registration, + appservice::query::query_room_alias::v1::Request { room_alias }, + ) + .await + .is_ok() + { + room_id = Some(db.rooms.id_from_alias(&room_alias)?.ok_or_else(|| { + Error::bad_config("Appservice lied to us. Room does not exist.") + })?); + break; + } + } + } + }; + + let room_id = match room_id { + Some(room_id) => room_id, + None => { + return Err(Error::BadRequest( + ErrorKind::NotFound, + "Room with alias not found.", + )) + } + }; Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_owned()]).into()) } diff --git a/src/client_server/media.rs b/src/client_server/media.rs index e6bd182b..0776c9e5 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -45,7 +45,7 @@ pub async fn create_content_route( db.flush().await?; - Ok(create_content::Response { content_uri: mxc }.into()) + Ok(create_content::Response { content_uri: mxc, blurhash: None }.into()) } #[cfg_attr( diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 6d3a6907..46548d5f 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -128,6 +128,7 @@ pub async fn leave_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; db.flush().await?; @@ -167,6 +168,7 @@ pub async fn invite_user_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; db.flush().await?; @@ -222,6 +224,7 @@ pub async fn kick_user_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; db.flush().await?; @@ -281,6 +284,7 @@ pub async fn ban_user_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; db.flush().await?; @@ -332,6 +336,7 @@ pub async fn unban_user_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; db.flush().await?; @@ -713,6 +718,7 @@ async fn join_room_by_id_helper( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; } diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 327b9ab2..3640730f 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -22,7 +22,7 @@ pub async fn send_message_event_route( body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + let sender_device = body.sender_device.as_deref(); // Check if this is a new transaction id if let Some(response) = @@ -69,6 +69,7 @@ pub async fn send_message_event_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; db.transaction_ids.add_txnid( diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 828d2593..761443d3 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -67,6 +67,7 @@ pub async fn set_displayname_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // Presence update @@ -163,6 +164,7 @@ pub async fn set_avatar_url_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // Presence update diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 6f7728a3..212e751d 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -35,6 +35,7 @@ pub async fn redact_event_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; db.flush().await?; diff --git a/src/client_server/room.rs b/src/client_server/room.rs index f92fc8df..e473e6e1 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -69,6 +69,7 @@ pub async fn create_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 2. Let the room creator join @@ -93,6 +94,7 @@ pub async fn create_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 3. Power levels @@ -137,6 +139,7 @@ pub async fn create_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 4. Events set by preset @@ -176,6 +179,7 @@ pub async fn create_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 4.2 History Visibility @@ -196,6 +200,7 @@ pub async fn create_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 4.3 Guest Access @@ -224,6 +229,7 @@ pub async fn create_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // 5. Events listed in initial_state @@ -246,6 +252,7 @@ pub async fn create_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; } @@ -270,6 +277,7 @@ pub async fn create_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; } @@ -291,6 +299,7 @@ pub async fn create_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; } @@ -317,6 +326,7 @@ pub async fn create_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; } @@ -407,6 +417,7 @@ pub async fn upgrade_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // Get the old room federations status @@ -450,6 +461,7 @@ pub async fn upgrade_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // Join the new room @@ -474,6 +486,7 @@ pub async fn upgrade_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; // Recommended transferable state events list from the specs @@ -510,6 +523,7 @@ pub async fn upgrade_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; } @@ -556,6 +570,7 @@ pub async fn upgrade_room_route( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; db.flush().await?; diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 010b20d3..cecb79d4 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -63,8 +63,8 @@ pub async fn send_state_event_for_empty_key_route( let Ruma { body, sender_user, - sender_device: _, json_body, + .. } = body; let json = serde_json::from_str::( @@ -288,6 +288,7 @@ pub async fn send_state_event_for_key_helper( &db.sending, &db.admin, &db.account_data, + &db.appservice, )?; Ok(event_id) diff --git a/src/client_server/to_device.rs b/src/client_server/to_device.rs index 8cc3e299..5bc001e4 100644 --- a/src/client_server/to_device.rs +++ b/src/client_server/to_device.rs @@ -17,7 +17,7 @@ pub async fn send_event_to_device_route( body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + let sender_device = body.sender_device.as_deref(); // Check if this is a new transaction id if db diff --git a/src/database.rs b/src/database.rs index 4905070a..51505178 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,5 +1,6 @@ pub mod account_data; pub mod admin; +pub mod appservice; pub mod globals; pub mod key_backups; pub mod media; @@ -16,6 +17,8 @@ use log::info; use rocket::futures::{self, channel::mpsc}; use ruma::{DeviceId, ServerName, UserId}; use serde::Deserialize; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; use std::{convert::TryInto, fs::remove_dir_all}; #[derive(Clone, Deserialize)] @@ -59,6 +62,7 @@ pub struct Database { pub transaction_ids: transaction_ids::TransactionIds, pub sending: sending::Sending, pub admin: admin::Admin, + pub appservice: appservice::Appservice, pub _db: sled::Db, } @@ -180,6 +184,10 @@ impl Database { admin: admin::Admin { sender: admin_sender, }, + appservice: appservice::Appservice { + cached_registrations: Arc::new(RwLock::new(HashMap::new())), + id_appserviceregistrations: db.open_tree("id_appserviceregistrations")?, + }, _db: db, }; diff --git a/src/database/admin.rs b/src/database/admin.rs index 778796f7..7de6bf92 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -10,7 +10,9 @@ use ruma::{ use tokio::select; pub enum AdminCommand { - SendTextMessage(message::TextMessageEventContent), + RegisterAppservice(serde_yaml::Value), + ListAppservices, + SendMessage(message::MessageEventContent), } #[derive(Clone)] @@ -44,28 +46,49 @@ impl Admin { warn!("Conduit instance does not have an #admins room. Logging to that room will not work."); } + let send_message = |message: message::MessageEventContent| { + if let Some(conduit_room) = &conduit_room { + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMessage, + content: serde_json::to_value(message) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: None, + }, + &conduit_user, + &conduit_room, + &db.globals, + &db.sending, + &db.admin, + &db.account_data, + &db.appservice, + ) + .unwrap(); + } + }; + loop { select! { Some(event) = receiver.next() => { match event { - AdminCommand::SendTextMessage(message) => { - if let Some(conduit_room) = &conduit_room { - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMessage, - content: serde_json::to_value(message).expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: None, - }, - &conduit_user, - &conduit_room, - &db.globals, - &db.sending, - &db.admin, - &db.account_data, - ).unwrap(); - } + AdminCommand::RegisterAppservice(yaml) => { + db.appservice.register_appservice(yaml).unwrap(); // TODO handle error + } + AdminCommand::ListAppservices => { + let appservices = db.appservice.iter_ids().collect::>(); + let count = appservices.len(); + let output = format!( + "Appservices ({}): {}", + count, + appservices.into_iter().filter_map(|r| r.ok()).collect::>().join(", ") + ); + send_message(message::MessageEventContent::text_plain(output)); + } + AdminCommand::SendMessage(message) => { + send_message(message); } } } diff --git a/src/database/appservice.rs b/src/database/appservice.rs new file mode 100644 index 00000000..26ea5b93 --- /dev/null +++ b/src/database/appservice.rs @@ -0,0 +1,67 @@ +use crate::{utils, Error, Result}; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +#[derive(Clone)] +pub struct Appservice { + pub(super) cached_registrations: Arc>>, + pub(super) id_appserviceregistrations: sled::Tree, +} + +impl Appservice { + pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> { + // TODO: Rumaify + let id = yaml.get("id").unwrap().as_str().unwrap(); + self.id_appserviceregistrations + .insert(id, serde_yaml::to_string(&yaml).unwrap().as_bytes())?; + self.cached_registrations + .write() + .unwrap() + .insert(id.to_owned(), yaml); + + Ok(()) + } + + pub fn get_registration(&self, id: &str) -> Result> { + self.cached_registrations + .read() + .unwrap() + .get(id) + .map_or_else( + || { + Ok(self + .id_appserviceregistrations + .get(id)? + .map(|bytes| { + Ok::<_, Error>(serde_yaml::from_slice(&bytes).map_err(|_| { + Error::bad_database( + "Invalid registration bytes in id_appserviceregistrations.", + ) + })?) + }) + .transpose()?) + }, + |r| Ok(Some(r.clone())), + ) + } + + pub fn iter_ids(&self) -> impl Iterator> { + self.id_appserviceregistrations.iter().keys().map(|id| { + Ok(utils::string_from_bytes(&id?).map_err(|_| { + Error::bad_database("Invalid id bytes in id_appserviceregistrations.") + })?) + }) + } + + pub fn iter_all<'a>( + &'a self, + ) -> impl Iterator> + 'a { + self.iter_ids().filter_map(|id| id.ok()).map(move |id| { + Ok(( + id.clone(), + self.get_registration(&id)? + .expect("iter_ids only returns appservices that exist"), + )) + }) + } +} diff --git a/src/database/globals.rs b/src/database/globals.rs index 1221609d..e913c0f2 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -1,10 +1,10 @@ use crate::{database::Config, utils, Error, Result}; -use trust_dns_resolver::TokioAsyncResolver; -use std::collections::HashMap; use log::error; use ruma::ServerName; +use std::collections::HashMap; use std::sync::Arc; use std::sync::RwLock; +use trust_dns_resolver::TokioAsyncResolver; pub const COUNTER: &str = "c"; @@ -59,9 +59,11 @@ impl Globals { config, keypair: Arc::new(keypair), reqwest_client: reqwest::Client::new(), - dns_resolver: TokioAsyncResolver::tokio_from_system_conf().await.map_err(|_| { - Error::bad_config("Failed to set up trust dns resolver with system config.") - })?, + dns_resolver: TokioAsyncResolver::tokio_from_system_conf() + .await + .map_err(|_| { + Error::bad_config("Failed to set up trust dns resolver with system config.") + })?, actual_destination_cache: Arc::new(RwLock::new(HashMap::new())), }) } diff --git a/src/database/media.rs b/src/database/media.rs index 89d48e16..448d0714 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -290,7 +290,12 @@ impl Media { file: thumbnail_bytes.to_vec(), })) } else { - Ok(None) + // Couldn't parse file to generate thumbnail, send original + Ok(Some(FileMeta { + filename, + content_type, + file: file.to_vec(), + })) } } else { Ok(None) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index fb139a64..3e2a17f7 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -36,16 +36,6 @@ use super::admin::AdminCommand; /// hashing the entire state. pub type StateHashId = IVec; -/// An enum that represents the two valid states when searching -/// for an events "parent". -/// -/// An events parent is any event we are aware of that is part of -/// the events `prev_events` array. -pub(crate) enum ClosestParent { - Append, - Insert(u64), -} - #[derive(Clone)] pub struct Rooms { pub edus: edus::RoomEdus, @@ -411,54 +401,6 @@ impl Rooms { } } - /// Recursively search for a PDU from our DB that is also in the - /// `prev_events` field of the incoming PDU. - /// - /// First we check if the last PDU inserted to the given room is a parent - /// if not we recursively check older `prev_events` to insert the incoming - /// event after. - pub(crate) fn get_latest_pduid_before( - &self, - room: &RoomId, - incoming_prev_ids: &[EventId], - their_state: &BTreeMap>, - ) -> Result> { - match self.pduid_pdu.scan_prefix(room.as_bytes()).last() { - Some(Ok(val)) - if incoming_prev_ids.contains( - &serde_json::from_slice::(&val.1) - .map_err(|_| { - Error::bad_database("last DB entry contains invalid PDU bytes") - })? - .event_id, - ) => - { - Ok(Some(ClosestParent::Append)) - } - _ => { - let mut prev_ids = incoming_prev_ids.to_vec(); - while let Some(id) = prev_ids.pop() { - match self.get_pdu_id(&id)? { - Some(pdu_id) => { - return Ok(Some(ClosestParent::Insert(self.pdu_count(&pdu_id)?))); - } - None => { - prev_ids.extend(their_state.get(&id).map_or( - Err(Error::BadServerResponse( - "Failed to find previous event for PDU in state", - )), - // `prev_event_ids` will return an empty Vec instead of failing - // so it works perfect for our use here - |pdu| Ok(pdu.prev_event_ids()), - )?); - } - } - } - Ok(None) - } - } - } - /// Returns the leaf pdus of a room. pub fn get_pdu_leaves(&self, room_id: &RoomId) -> Result> { let mut prefix = room_id.as_bytes().to_vec(); @@ -583,18 +525,59 @@ impl Rooms { .as_ref() == Some(&pdu.room_id) { - let mut parts = body.split_whitespace().skip(1); + let mut lines = body.lines(); + let command_line = lines.next().expect("each string has at least one line"); + let body = lines.collect::>(); + + let mut parts = command_line.split_whitespace().skip(1); if let Some(command) = parts.next() { let args = parts.collect::>(); - admin.send(AdminCommand::SendTextMessage( - message::TextMessageEventContent { - body: format!("Command: {}, Args: {:?}", command, args), - formatted: None, - relates_to: None, - new_content: None, - }, - )); + match command { + "register_appservice" => { + if body.len() > 2 + && body[0].trim() == "```" + && body.last().unwrap().trim() == "```" + { + let appservice_config = body[1..body.len() - 1].join("\n"); + let parsed_config = serde_yaml::from_str::( + &appservice_config, + ); + match parsed_config { + Ok(yaml) => { + admin.send(AdminCommand::RegisterAppservice(yaml)); + } + Err(e) => { + admin.send(AdminCommand::SendMessage( + message::MessageEventContent::text_plain( + format!( + "Could not parse appservice config: {}", + e + ), + ), + )); + } + } + } else { + admin.send(AdminCommand::SendMessage( + message::MessageEventContent::text_plain( + "Expected code block in command body.", + ), + )); + } + } + "list_appservices" => { + admin.send(AdminCommand::ListAppservices); + } + _ => { + admin.send(AdminCommand::SendMessage( + message::MessageEventContent::text_plain(format!( + "Command: {}, Args: {:?}", + command, args + )), + )); + } + } } } } @@ -675,6 +658,7 @@ impl Rooms { sending: &super::sending::Sending, admin: &super::admin::Admin, account_data: &super::account_data::AccountData, + appservice: &super::appservice::Appservice, ) -> Result { let PduBuilder { event_type, @@ -923,6 +907,10 @@ impl Rooms { sending.send_pdu(&server, &pdu_id)?; } + for appservice in appservice.iter_all().filter_map(|r| r.ok()) { + sending.send_pdu_appservice(&appservice.0, &pdu_id)?; + } + Ok(pdu.event_id) } diff --git a/src/database/sending.rs b/src/database/sending.rs index cd88e088..7ce7d633 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -1,26 +1,35 @@ use std::{collections::HashMap, convert::TryFrom, time::SystemTime}; -use crate::{server_server, utils, Error, PduEvent, Result}; +use crate::{appservice_server, server_server, utils, Error, PduEvent, Result}; use federation::transactions::send_transaction_message; -use log::{debug, warn}; +use log::warn; use rocket::futures::stream::{FuturesUnordered, StreamExt}; -use ruma::{api::federation, ServerName}; +use ruma::{ + api::{appservice, federation}, + ServerName, +}; use sled::IVec; use tokio::select; #[derive(Clone)] pub struct Sending { /// The state for a given state hash. - pub(super) servernamepduids: sled::Tree, // ServernamePduId = ServerName + PduId - pub(super) servercurrentpdus: sled::Tree, // ServerCurrentPdus = ServerName + PduId (pduid can be empty for reservation) + pub(super) servernamepduids: sled::Tree, // ServernamePduId = (+)ServerName + PduId + pub(super) servercurrentpdus: sled::Tree, // ServerCurrentPdus = (+)ServerName + PduId (pduid can be empty for reservation) } impl Sending { - pub fn start_handler(&self, globals: &super::globals::Globals, rooms: &super::rooms::Rooms) { + pub fn start_handler( + &self, + globals: &super::globals::Globals, + rooms: &super::rooms::Rooms, + appservice: &super::appservice::Appservice, + ) { let servernamepduids = self.servernamepduids.clone(); let servercurrentpdus = self.servercurrentpdus.clone(); let rooms = rooms.clone(); let globals = globals.clone(); + let appservice = appservice.clone(); tokio::spawn(async move { let mut futures = FuturesUnordered::new(); @@ -28,7 +37,7 @@ impl Sending { // Retry requests we could not finish yet let mut current_transactions = HashMap::new(); - for (server, pdu) in servercurrentpdus + for (server, pdu, is_appservice) in servercurrentpdus .iter() .filter_map(|r| r.ok()) .map(|(key, _)| { @@ -38,45 +47,61 @@ impl Sending { Error::bad_database("Invalid bytes in servercurrentpdus.") })?; + let server = utils::string_from_bytes(&server).map_err(|_| { + Error::bad_database("Invalid server bytes in server_currenttransaction") + })?; + + // Appservices start with a plus + let (server, is_appservice) = if server.starts_with("+") { + (&server[1..], true) + } else { + (&*server, false) + }; + Ok::<_, Error>(( - Box::::try_from(utils::string_from_bytes(&server).map_err( - |_| { - Error::bad_database( - "Invalid server bytes in server_currenttransaction", - ) - }, - )?) - .map_err(|_| { + Box::::try_from(server).map_err(|_| { Error::bad_database( "Invalid server string in server_currenttransaction", ) })?, IVec::from(pdu), + is_appservice, )) }) .filter_map(|r| r.ok()) - .filter(|(_, pdu)| !pdu.is_empty()) // Skip reservation key + .filter(|(_, pdu, _)| !pdu.is_empty()) // Skip reservation key .take(50) // This should not contain more than 50 anyway { current_transactions - .entry(server) + .entry((server, is_appservice)) .or_insert_with(Vec::new) .push(pdu); } - for (server, pdus) in current_transactions { - futures.push(Self::handle_event(server, pdus, &globals, &rooms)); + for ((server, is_appservice), pdus) in current_transactions { + futures.push(Self::handle_event( + server, + is_appservice, + pdus, + &globals, + &rooms, + &appservice, + )); } let mut subscriber = servernamepduids.watch_prefix(b""); loop { select! { - Some(server) = futures.next() => { - debug!("sending response: {:?}", &server); - match server { - Ok((server, _response)) => { - let mut prefix = server.as_bytes().to_vec(); + Some(response) = futures.next() => { + match response { + Ok((server, is_appservice)) => { + let mut prefix = if is_appservice { + "+".as_bytes().to_vec() + } else { + Vec::new() + }; + prefix.extend_from_slice(server.as_bytes()); prefix.push(0xff); for key in servercurrentpdus @@ -109,13 +134,13 @@ impl Sending { servernamepduids.remove(¤t_key).unwrap(); } - futures.push(Self::handle_event(server, new_pdus, &globals, &rooms)); + futures.push(Self::handle_event(server, is_appservice, new_pdus, &globals, &rooms, &appservice)); } else { servercurrentpdus.remove(&prefix).unwrap(); // servercurrentpdus with the prefix should be empty now } } - Err((server, e)) => { + Err((server, _is_appservice, e)) => { warn!("Couldn't send transaction to {}: {}", server, e) // TODO: exponential backoff } @@ -126,24 +151,37 @@ impl Sending { let servernamepduid = key.clone(); let mut parts = servernamepduid.splitn(2, |&b| b == 0xff); - if let Some((server, pdu_id)) = utils::string_from_bytes( + if let Some((server, is_appservice, pdu_id)) = utils::string_from_bytes( parts .next() .expect("splitn will always return 1 or more elements"), ) .map_err(|_| Error::bad_database("ServerName in servernamepduid bytes are invalid.")) - .and_then(|server_str| Box::::try_from(server_str) - .map_err(|_| Error::bad_database("ServerName in servernamepduid is invalid."))) + .map(|server_str| { + // Appservices start with a plus + if server_str.starts_with("+") { + (server_str[1..].to_owned(), true) + } else { + (server_str, false) + } + }) + .and_then(|(server_str, is_appservice)| Box::::try_from(server_str) + .map_err(|_| Error::bad_database("ServerName in servernamepduid is invalid.")).map(|s| (s, is_appservice))) .ok() - .and_then(|server| parts + .and_then(|(server, is_appservice)| parts .next() .ok_or_else(|| Error::bad_database("Invalid servernamepduid in db.")) .ok() - .map(|pdu_id| (server, pdu_id)) + .map(|pdu_id| (server, is_appservice, pdu_id)) ) // TODO: exponential backoff - .filter(|(server, _)| { - let mut prefix = server.to_string().as_bytes().to_vec(); + .filter(|(server, is_appservice, _)| { + let mut prefix = if *is_appservice { + "+".as_bytes().to_vec() + } else { + Vec::new() + }; + prefix.extend_from_slice(server.as_bytes()); prefix.push(0xff); servercurrentpdus @@ -154,7 +192,7 @@ impl Sending { servercurrentpdus.insert(&key, &[]).unwrap(); servernamepduids.remove(&key).unwrap(); - futures.push(Self::handle_event(server, vec![pdu_id.into()], &globals, &rooms)); + futures.push(Self::handle_event(server, is_appservice, vec![pdu_id.into()], &globals, &rooms, &appservice)); } } } @@ -172,56 +210,102 @@ impl Sending { Ok(()) } + pub fn send_pdu_appservice(&self, appservice_id: &str, pdu_id: &[u8]) -> Result<()> { + let mut key = "+".as_bytes().to_vec(); + key.extend_from_slice(appservice_id.as_bytes()); + key.push(0xff); + key.extend_from_slice(pdu_id); + self.servernamepduids.insert(key, b"")?; + + Ok(()) + } + async fn handle_event( server: Box, + is_appservice: bool, pdu_ids: Vec, globals: &super::globals::Globals, rooms: &super::rooms::Rooms, - ) -> std::result::Result< - (Box, send_transaction_message::v1::Response), - (Box, Error), - > { - let pdu_jsons = pdu_ids - .iter() - .map(|pdu_id| { - Ok::<_, (Box, Error)>( - // TODO: check room version and remove event_id if needed - serde_json::from_str( - PduEvent::convert_to_outgoing_federation_event( - rooms - .get_pdu_json_from_id(pdu_id) - .map_err(|e| (server.clone(), e))? - .ok_or_else(|| { - ( - server.clone(), - Error::bad_database( - "Event in servernamepduids not found in db.", - ), - ) - })?, + appservice: &super::appservice::Appservice, + ) -> std::result::Result<(Box, bool), (Box, bool, Error)> { + if is_appservice { + let pdu_jsons = pdu_ids + .iter() + .map(|pdu_id| { + Ok::<_, (Box, Error)>( + rooms + .get_pdu_from_id(pdu_id) + .map_err(|e| (server.clone(), e))? + .ok_or_else(|| { + ( + server.clone(), + Error::bad_database( + "Event in servernamepduids not found in db.", + ), + ) + })? + .to_any_event(), + ) + }) + .filter_map(|r| r.ok()) + .collect::>(); + appservice_server::send_request( + &globals, + appservice + .get_registration(server.as_str()) + .unwrap() + .unwrap(), // TODO: handle error + appservice::event::push_events::v1::Request { + events: &pdu_jsons, + txn_id: &utils::random_string(16), + }, + ) + .await + .map(|_response| (server.clone(), is_appservice)) + .map_err(|e| (server, is_appservice, e)) + } else { + let pdu_jsons = pdu_ids + .iter() + .map(|pdu_id| { + Ok::<_, (Box, Error)>( + // TODO: check room version and remove event_id if needed + serde_json::from_str( + PduEvent::convert_to_outgoing_federation_event( + rooms + .get_pdu_json_from_id(pdu_id) + .map_err(|e| (server.clone(), e))? + .ok_or_else(|| { + ( + server.clone(), + Error::bad_database( + "Event in servernamepduids not found in db.", + ), + ) + })?, + ) + .json() + .get(), ) - .json() - .get(), + .expect("Raw<..> is always valid"), ) - .expect("Raw<..> is always valid"), - ) - }) - .filter_map(|r| r.ok()) - .collect::>(); - - server_server::send_request( - &globals, - server.clone(), - send_transaction_message::v1::Request { - origin: globals.server_name(), - pdus: &pdu_jsons, - edus: &[], - origin_server_ts: SystemTime::now(), - transaction_id: &utils::random_string(16), - }, - ) - .await - .map(|response| (server.clone(), response)) - .map_err(|e| (server, e)) + }) + .filter_map(|r| r.ok()) + .collect::>(); + + server_server::send_request( + &globals, + server.clone(), + send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus: &pdu_jsons, + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &utils::random_string(16), + }, + ) + .await + .map(|_response| (server.clone(), is_appservice)) + .map_err(|e| (server, is_appservice, e)) + } } } diff --git a/src/database/transaction_ids.rs b/src/database/transaction_ids.rs index 7c0eb98b..1f8ba7de 100644 --- a/src/database/transaction_ids.rs +++ b/src/database/transaction_ids.rs @@ -11,13 +11,13 @@ impl TransactionIds { pub fn add_txnid( &self, user_id: &UserId, - device_id: &DeviceId, + device_id: Option<&DeviceId>, txn_id: &str, data: &[u8], ) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(device_id.as_bytes()); + key.extend_from_slice(device_id.map(|d| d.as_bytes()).unwrap_or_default()); key.push(0xff); key.extend_from_slice(txn_id.as_bytes()); @@ -29,12 +29,12 @@ impl TransactionIds { pub fn existing_txnid( &self, user_id: &UserId, - device_id: &DeviceId, + device_id: Option<&DeviceId>, txn_id: &str, ) -> Result> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(device_id.as_bytes()); + key.extend_from_slice(device_id.map(|d| d.as_bytes()).unwrap_or_default()); key.push(0xff); key.extend_from_slice(txn_id.as_bytes()); diff --git a/src/error.rs b/src/error.rs index 316ca742..7d4a7518 100644 --- a/src/error.rs +++ b/src/error.rs @@ -142,8 +142,8 @@ impl log::Log for ConduitLogger { mut_last_logs.insert(output.clone(), Instant::now()); } - self.db.admin.send(AdminCommand::SendTextMessage( - message::TextMessageEventContent::plain(output), + self.db.admin.send(AdminCommand::SendMessage( + message::MessageEventContent::text_plain(output), )); } } diff --git a/src/lib.rs b/src/lib.rs index eea32c75..aed129fd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +pub mod appservice_server; pub mod client_server; mod database; mod error; diff --git a/src/main.rs b/src/main.rs index 58d3427f..95748943 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ #![warn(rust_2018_idioms)] +pub mod appservice_server; pub mod client_server; pub mod server_server; @@ -139,7 +140,8 @@ fn setup_rocket() -> rocket::Rocket { .await .expect("config is valid"); - data.sending.start_handler(&data.globals, &data.rooms); + data.sending + .start_handler(&data.globals, &data.rooms, &data.appservice); log::set_boxed_logger(Box::new(ConduitLogger { db: data.clone(), last_logs: Default::default(), diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 9597ac85..0fdca743 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -12,7 +12,7 @@ use std::{ #[cfg(feature = "conduit_bin")] use { crate::utils, - log::warn, + log::{debug, warn}, rocket::{ data::{ ByteUnit, Data, FromDataFuture, FromTransformedData, Transform, TransformFuture, @@ -34,6 +34,7 @@ pub struct Ruma { pub sender_user: Option, pub sender_device: Option>, pub json_body: Option>, // This is None when body is not a valid string + pub from_appservice: bool, } #[cfg(feature = "conduit_bin")] @@ -66,28 +67,72 @@ where .await .expect("database was loaded"); - let (sender_user, sender_device) = match T::METADATA.authentication { - AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => { - // Get token from header or query value - let token = match request - .headers() - .get_one("Authorization") - .map(|s| s[7..].to_owned()) // Split off "Bearer " - .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok())) - { - // TODO: M_MISSING_TOKEN - None => return Failure((Status::Unauthorized, ())), - Some(token) => token, - }; - - // Check if token is valid - match db.users.find_from_token(&token).unwrap() { - // TODO: M_UNKNOWN_TOKEN - None => return Failure((Status::Unauthorized, ())), - Some((user_id, device_id)) => (Some(user_id), Some(device_id.into())), + // Get token from header or query value + let token = request + .headers() + .get_one("Authorization") + .map(|s| s[7..].to_owned()) // Split off "Bearer " + .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok())); + + let (sender_user, sender_device, from_appservice) = if let Some((_id, registration)) = + db.appservice + .iter_all() + .filter_map(|r| r.ok()) + .find(|(_id, registration)| { + registration + .get("as_token") + .and_then(|as_token| as_token.as_str()) + .map_or(false, |as_token| token.as_deref() == Some(as_token)) + }) { + match T::METADATA.authentication { + AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => { + let user_id = request.get_query_value::("user_id").map_or_else( + || { + UserId::parse_with_server_name( + registration + .get("sender_localpart") + .unwrap() + .as_str() + .unwrap(), + db.globals.server_name(), + ) + .unwrap() + }, + |string| { + UserId::try_from(string.expect("parsing to string always works")) + .unwrap() + }, + ); + + if !db.users.exists(&user_id).unwrap() { + return Failure((Status::Unauthorized, ())); + } + + // TODO: Check if appservice is allowed to be that user + (Some(user_id), None, true) } + AuthScheme::ServerSignatures => (None, None, true), + AuthScheme::None => (None, None, true), + } + } else { + match T::METADATA.authentication { + AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => { + if let Some(token) = token { + match db.users.find_from_token(&token).unwrap() { + // TODO: M_UNKNOWN_TOKEN + None => return Failure((Status::Unauthorized, ())), + Some((user_id, device_id)) => { + (Some(user_id), Some(device_id.into()), false) + } + } + } else { + // TODO: M_MISSING_TOKEN + return Failure((Status::Unauthorized, ())); + } + } + AuthScheme::ServerSignatures => (None, None, false), + AuthScheme::None => (None, None, false), } - _ => (None, None), }; let mut http_request = http::Request::builder() @@ -103,7 +148,7 @@ where handle.read_to_end(&mut body).await.unwrap(); let http_request = http_request.body(body.clone()).unwrap(); - log::debug!("{:?}", http_request); + debug!("{:?}", http_request); match ::Incoming::try_from(http_request) { Ok(t) => Success(Ruma { @@ -114,6 +159,7 @@ where json_body: utils::string_from_bytes(&body) .ok() .and_then(|s| serde_json::value::RawValue::from_string(s).ok()), + from_appservice, }), Err(e) => { warn!("{:?}", e); diff --git a/src/server_server.rs b/src/server_server.rs index 58dd872e..7d12c54c 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,10 +1,7 @@ -use crate::{ - client_server, database::rooms::ClosestParent, utils, ConduitResult, Database, Error, PduEvent, - Result, Ruma, -}; +use crate::{client_server, utils, ConduitResult, Database, Error, PduEvent, Result, Ruma}; use get_profile_information::v1::ProfileField; use http::header::{HeaderValue, AUTHORIZATION, HOST}; -use log::{error, warn}; +use log::warn; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ api::{ @@ -27,7 +24,6 @@ use std::{ collections::BTreeMap, convert::TryFrom, fmt::Debug, - sync::Arc, time::{Duration, SystemTime}, }; @@ -73,7 +69,6 @@ where .cloned(); let (actual_destination, host) = if let Some(result) = maybe_result { - println!("Loaded {} -> {:?}", destination, result); result } else { let result = find_actual_destination(globals, &destination).await; @@ -82,7 +77,6 @@ where .write() .unwrap() .insert(destination.clone(), result.clone()); - println!("Saving {} -> {:?}", destination, result); result }; @@ -491,173 +485,28 @@ pub async fn send_transaction_message_route<'a>( continue; } - // If it is not a state event, we can skip state-res... maybe - if value.get("state_key").is_none() { - if !db.rooms.is_joined(&pdu.sender, room_id)? { - warn!("Sender is not joined {}", pdu.kind); - resolved_map.insert(event_id, Err("User is not in this room".into())); - continue; - } - - let count = db.globals.next_count()?; - let mut pdu_id = room_id.as_bytes().to_vec(); - pdu_id.push(0xff); - pdu_id.extend_from_slice(&count.to_be_bytes()); + let count = db.globals.next_count()?; + let mut pdu_id = room_id.as_bytes().to_vec(); + pdu_id.push(0xff); + pdu_id.extend_from_slice(&count.to_be_bytes()); - db.rooms.append_to_state(&pdu_id, &pdu)?; - - db.rooms.append_pdu( - &pdu, - &value, - count, - pdu_id.into(), - &db.globals, - &db.account_data, - &db.admin, - )?; - - resolved_map.insert(event_id, Ok::<(), String>(())); - continue; - } + db.rooms.append_to_state(&pdu_id, &pdu)?; - // We have a state event so we need info for state-res - let get_state_response = match send_request( + db.rooms.append_pdu( + &pdu, + &value, + count, + pdu_id.clone().into(), &db.globals, - body.body.origin.clone(), - ruma::api::federation::event::get_room_state::v1::Request { - room_id, - event_id: &event_id, - }, - ) - .await - { - Ok(res) => res, - // We can't hard fail because there are some valid errors, just - // keep checking PDU's - // - // As an example a possible error - // {"errcode":"M_FORBIDDEN","error":"Host not in room."} - Err(err) => { - resolved_map.insert(event_id, Err(err.to_string())); - continue; - } - }; + &db.account_data, + &db.admin, + )?; - let their_current_state = get_state_response - .pdus - .iter() - .chain(get_state_response.auth_chain.iter()) // add auth events - .map(|pdu| { - let (event_id, json) = crate::pdu::process_incoming_pdu(pdu); - ( - event_id.clone(), - Arc::new( - // When creating a StateEvent the event_id arg will be used - // over any found in the json and it will not use ruma::reference_hash - // to generate one - state_res::StateEvent::from_id_canon_obj(event_id, json) - .expect("valid pdu json"), - ), - ) - }) - .collect::>(); - - let our_current_state = db.rooms.room_state_full(room_id)?; - // State resolution takes care of these checks - // 4. Passes authorization rules based on the event's auth events, otherwise it is rejected. - // 5. Passes authorization rules based on the state at the event, otherwise it is rejected. - - // TODO: 6. Passes authorization rules based on the current state of the room, otherwise it is "soft failed". - match state_res::StateResolution::resolve( - room_id, - &ruma::RoomVersionId::Version6, - &[ - our_current_state - .iter() - .map(|((ev, sk), v)| ((ev.clone(), sk.to_owned()), v.event_id.clone())) - .collect::>(), - their_current_state - .iter() - .map(|(_id, v)| ((v.kind(), v.state_key()), v.event_id())) - .collect::>(), - ], - Some( - our_current_state - .iter() - .map(|(_k, v)| (v.event_id.clone(), v.convert_for_state_res())) - .chain( - their_current_state - .iter() - .map(|(id, ev)| (id.clone(), ev.clone())), - ) - .collect::>(), - ), - &db.rooms, - ) { - Ok(resolved) if resolved.values().any(|id| &event_id == id) => { - // If the event is older than the last event in pduid_pdu Tree then find the - // closest ancestor we know of and insert after the known ancestor by - // altering the known events pduid to = same roomID + same count bytes + 0x1 - // pushing a single byte every time a simple append cannot be done. - match db.rooms.get_latest_pduid_before( - room_id, - &pdu.prev_events, - &their_current_state, - )? { - Some(ClosestParent::Append) => { - let count = db.globals.next_count()?; - let mut pdu_id = room_id.as_bytes().to_vec(); - pdu_id.push(0xff); - pdu_id.extend_from_slice(&count.to_be_bytes()); - - db.rooms.append_pdu( - &pdu, - &value, - count, - pdu_id.into(), - &db.globals, - &db.account_data, - &db.admin, - )?; - } - Some(ClosestParent::Insert(old_count)) => { - let count = old_count; - let mut pdu_id = room_id.as_bytes().to_vec(); - pdu_id.push(0xff); - pdu_id.extend_from_slice(&count.to_be_bytes()); - // Create a new count that is after old_count but before - // the pdu appended after - pdu_id.push(1); - - db.rooms.append_pdu( - &pdu, - &value, - count, - pdu_id.into(), - &db.globals, - &db.account_data, - &db.admin, - )?; - } - _ => { - error!("Not a sequential event or no parents found"); - continue; - } - } + for appservice in db.appservice.iter_all().filter_map(|r| r.ok()) { + db.sending.send_pdu_appservice(&appservice.0, &pdu_id)?; + } - resolved_map.insert(event_id, Ok::<(), String>(())); - } - // If the eventId is not found in the resolved state auth has failed - Ok(_) => { - resolved_map.insert( - event_id, - Err("This event failed authentication, not found in resolved set".into()), - ); - } - Err(e) => { - resolved_map.insert(event_id, Err(e.to_string())); - } - }; + resolved_map.insert(event_id, Ok::<(), String>(())); } Ok(send_transaction_message::v1::Response { pdus: resolved_map }.into()) From 6606e41dde413af64278e52ee2a376377c8c035e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 19 Dec 2020 16:00:11 +0100 Subject: [PATCH 20/36] feat: improved state store --- src/client_server/alias.rs | 32 +++--- src/client_server/directory.rs | 28 ++--- src/client_server/media.rs | 60 ++++++----- src/client_server/membership.rs | 55 +++++----- src/client_server/read_marker.rs | 19 +++- src/client_server/sync.rs | 111 ++++++++++--------- src/database.rs | 13 ++- src/database/globals.rs | 10 +- src/database/rooms.rs | 178 +++++++++++++++++-------------- src/database/sending.rs | 137 ++++++++++++++++++------ src/error.rs | 2 +- src/main.rs | 9 +- src/server_server.rs | 2 +- 13 files changed, 405 insertions(+), 251 deletions(-) diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index ec73ffc4..498e8827 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -1,5 +1,5 @@ use super::State; -use crate::{appservice_server, server_server, ConduitResult, Database, Error, Ruma}; +use crate::{ConduitResult, Database, Error, Ruma}; use ruma::{ api::{ appservice, @@ -66,12 +66,14 @@ pub async fn get_alias_helper( room_alias: &RoomAliasId, ) -> ConduitResult { if room_alias.server_name() != db.globals.server_name() { - let response = server_server::send_request( - &db.globals, - room_alias.server_name().to_owned(), - federation::query::get_room_information::v1::Request { room_alias }, - ) - .await?; + let response = db + .sending + .send_federation_request( + &db.globals, + room_alias.server_name().to_owned(), + federation::query::get_room_information::v1::Request { room_alias }, + ) + .await?; return Ok(get_alias::Response::new(response.room_id, response.servers).into()); } @@ -81,13 +83,15 @@ pub async fn get_alias_helper( Some(r) => room_id = Some(r), None => { for (_id, registration) in db.appservice.iter_all().filter_map(|r| r.ok()) { - if appservice_server::send_request( - &db.globals, - registration, - appservice::query::query_room_alias::v1::Request { room_alias }, - ) - .await - .is_ok() + if db + .sending + .send_appservice_request( + &db.globals, + registration, + appservice::query::query_room_alias::v1::Request { room_alias }, + ) + .await + .is_ok() { room_id = Some(db.rooms.id_from_alias(&room_alias)?.ok_or_else(|| { Error::bad_config("Appservice lied to us. Room does not exist.") diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 559071a1..fa5db3a6 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -1,5 +1,5 @@ use super::State; -use crate::{server_server, ConduitResult, Database, Error, Result, Ruma}; +use crate::{ConduitResult, Database, Error, Result, Ruma}; use log::info; use ruma::{ api::{ @@ -133,19 +133,21 @@ pub async fn get_public_rooms_filtered_helper( .clone() .filter(|server| *server != db.globals.server_name().as_str()) { - let response = server_server::send_request( - &db.globals, - other_server.to_owned(), - federation::directory::get_public_rooms_filtered::v1::Request { - limit, - since: since.as_deref(), - filter: Filter { - generic_search_term: filter.generic_search_term.as_deref(), + let response = db + .sending + .send_federation_request( + &db.globals, + other_server.to_owned(), + federation::directory::get_public_rooms_filtered::v1::Request { + limit, + since: since.as_deref(), + filter: Filter { + generic_search_term: filter.generic_search_term.as_deref(), + }, + room_network: RoomNetwork::Matrix, }, - room_network: RoomNetwork::Matrix, - }, - ) - .await?; + ) + .await?; return Ok(get_public_rooms_filtered::Response { chunk: response diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 0776c9e5..156040bd 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -1,7 +1,5 @@ use super::State; -use crate::{ - database::media::FileMeta, server_server, utils, ConduitResult, Database, Error, Ruma, -}; +use crate::{database::media::FileMeta, utils, ConduitResult, Database, Error, Ruma}; use ruma::api::client::{ error::ErrorKind, r0::media::{create_content, get_content, get_content_thumbnail, get_media_config}, @@ -45,7 +43,11 @@ pub async fn create_content_route( db.flush().await?; - Ok(create_content::Response { content_uri: mxc, blurhash: None }.into()) + Ok(create_content::Response { + content_uri: mxc, + blurhash: None, + } + .into()) } #[cfg_attr( @@ -71,16 +73,18 @@ pub async fn get_content_route( } .into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { - let get_content_response = server_server::send_request( - &db.globals, - body.server_name.clone(), - get_content::Request { - allow_remote: false, - server_name: &body.server_name, - media_id: &body.media_id, - }, - ) - .await?; + let get_content_response = db + .sending + .send_federation_request( + &db.globals, + body.server_name.clone(), + get_content::Request { + allow_remote: false, + server_name: &body.server_name, + media_id: &body.media_id, + }, + ) + .await?; db.media.create( mxc, @@ -118,19 +122,21 @@ pub async fn get_content_thumbnail_route( )? { Ok(get_content_thumbnail::Response { file, content_type }.into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { - let get_thumbnail_response = server_server::send_request( - &db.globals, - body.server_name.clone(), - get_content_thumbnail::Request { - allow_remote: false, - height: body.height, - width: body.width, - method: body.method, - server_name: &body.server_name, - media_id: &body.media_id, - }, - ) - .await?; + let get_thumbnail_response = db + .sending + .send_federation_request( + &db.globals, + body.server_name.clone(), + get_content_thumbnail::Request { + allow_remote: false, + height: body.height, + width: body.width, + method: body.method, + server_name: &body.server_name, + media_id: &body.media_id, + }, + ) + .await?; db.media.upload_thumbnail( mxc, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 46548d5f..e8d57bc1 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -2,7 +2,7 @@ use super::State; use crate::{ client_server, pdu::{PduBuilder, PduEvent}, - server_server, utils, ConduitResult, Database, Error, Result, Ruma, + utils, ConduitResult, Database, Error, Result, Ruma, }; use log::warn; use ruma::{ @@ -401,9 +401,10 @@ pub async fn get_member_events_route( Ok(get_member_events::Response { chunk: db .rooms - .room_state_type(&body.room_id, &EventType::RoomMember)? - .values() - .map(|pdu| pdu.to_member_event()) + .room_state_full(&body.room_id)? + .iter() + .filter(|(key, _)| key.0 == EventType::RoomMember) + .map(|(_, pdu)| pdu.to_member_event()) .collect(), } .into()) @@ -463,16 +464,18 @@ async fn join_room_by_id_helper( )); for remote_server in servers { - let make_join_response = server_server::send_request( - &db.globals, - remote_server.clone(), - federation::membership::create_join_event_template::v1::Request { - room_id, - user_id: sender_user, - ver: &[RoomVersionId::Version5, RoomVersionId::Version6], - }, - ) - .await; + let make_join_response = db + .sending + .send_federation_request( + &db.globals, + remote_server.clone(), + federation::membership::create_join_event_template::v1::Request { + room_id, + user_id: sender_user, + ver: &[RoomVersionId::Version5, RoomVersionId::Version6], + }, + ) + .await; make_join_response_and_server = make_join_response.map(|r| (r, remote_server)); @@ -540,16 +543,18 @@ async fn join_room_by_id_helper( // It has enough fields to be called a proper event now let join_event = join_event_stub; - let send_join_response = server_server::send_request( - &db.globals, - remote_server.clone(), - federation::membership::create_join_event::v2::Request { - room_id, - event_id: &event_id, - pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), - }, - ) - .await?; + let send_join_response = db + .sending + .send_federation_request( + &db.globals, + remote_server.clone(), + federation::membership::create_join_event::v2::Request { + room_id, + event_id: &event_id, + pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), + }, + ) + .await?; let add_event_id = |pdu: &Raw| -> Result<(EventId, CanonicalJsonObject)> { let mut value = serde_json::from_str(pdu.json().get()) @@ -694,7 +699,7 @@ async fn join_room_by_id_helper( } } - db.rooms.force_state(room_id, state)?; + db.rooms.force_state(room_id, state, &db.globals)?; } else { let event = member::MemberEventContent { membership: member::MembershipState::Join, diff --git a/src/client_server/read_marker.rs b/src/client_server/read_marker.rs index f3e7211d..0c4ec1a4 100644 --- a/src/client_server/read_marker.rs +++ b/src/client_server/read_marker.rs @@ -1,7 +1,9 @@ use super::State; use crate::{ConduitResult, Database, Error, Ruma}; use ruma::{ - api::client::{error::ErrorKind, r0::read_marker::set_read_marker}, + api::client::{ + error::ErrorKind, r0::capabilities::get_capabilities, r0::read_marker::set_read_marker, + }, events::{AnyEphemeralRoomEvent, AnyEvent, EventType}, }; @@ -76,3 +78,18 @@ pub async fn set_read_marker_route( Ok(set_read_marker::Response.into()) } + +#[cfg_attr( + feature = "conduit_bin", + post("/_matrix/client/r0/rooms/<_>/receipt/<_>/<_>", data = "") +)] +pub async fn set_receipt_route( + db: State<'_, Database>, + body: Ruma, +) -> ConduitResult { + let _sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + db.flush().await?; + + Ok(set_read_marker::Response.into()) +} diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index d7c24dcc..82136511 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -102,9 +102,15 @@ pub async fn sync_events_route( } // Database queries: - let encrypted_room = db - .rooms - .room_state_get(&room_id, &EventType::RoomEncryption, "")? + + let current_state = db.rooms.room_state_full(&room_id)?; + let current_members = current_state + .iter() + .filter(|(key, _)| key.0 == EventType::RoomMember) + .map(|(key, value)| (&key.1, value)) // Only keep state key + .collect::>(); + let encrypted_room = current_state + .get(&(EventType::RoomEncryption, "".to_owned())) .is_some(); // These type is Option>. The outer Option is None when there is no event between @@ -117,45 +123,45 @@ pub async fn sync_events_route( .as_ref() .map(|pdu| db.rooms.pdu_state_hash(&pdu.as_ref().ok()?.0).ok()?); - let since_members = since_state_hash.as_ref().map(|state_hash| { - state_hash.as_ref().and_then(|state_hash| { - db.rooms - .state_type(&state_hash, &EventType::RoomMember) - .ok() - }) + let since_state = since_state_hash.as_ref().map(|state_hash| { + state_hash + .as_ref() + .and_then(|state_hash| db.rooms.state_full(&room_id, &state_hash).ok()) }); - let since_encryption = since_state_hash.as_ref().map(|state_hash| { - state_hash.as_ref().and_then(|state_hash| { - db.rooms - .state_get(&state_hash, &EventType::RoomEncryption, "") - .ok() - }) + let since_encryption = since_state.as_ref().map(|state| { + state + .as_ref() + .map(|state| state.get(&(EventType::RoomEncryption, "".to_owned()))) }); - let current_members = db.rooms.room_state_type(&room_id, &EventType::RoomMember)?; - // Calculations: let new_encrypted_room = encrypted_room && since_encryption.map_or(false, |encryption| encryption.is_none()); - let send_member_count = since_members.as_ref().map_or(false, |since_members| { - since_members.as_ref().map_or(true, |since_members| { - current_members.len() != since_members.len() + let send_member_count = since_state.as_ref().map_or(false, |since_state| { + since_state.as_ref().map_or(true, |since_state| { + current_members.len() + != since_state + .iter() + .filter(|(key, _)| key.0 == EventType::RoomMember) + .count() }) }); - let since_sender_member = since_members.as_ref().map(|since_members| { - since_members.as_ref().and_then(|members| { - members.get(sender_user.as_str()).and_then(|pdu| { - serde_json::from_value::>( - pdu.content.clone(), - ) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database.")) - .ok() - }) + let since_sender_member = since_state.as_ref().map(|since_state| { + since_state.as_ref().and_then(|state| { + state + .get(&(EventType::RoomMember, sender_user.as_str().to_owned())) + .and_then(|pdu| { + serde_json::from_value::< + Raw, + >(pdu.content.clone()) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }) }) }); @@ -170,30 +176,32 @@ pub async fn sync_events_route( .membership; let since_membership = - since_members + since_state .as_ref() - .map_or(MembershipState::Join, |members| { - members + .map_or(MembershipState::Join, |since_state| { + since_state .as_ref() - .and_then(|members| { - members.get(&user_id).and_then(|since_member| { - serde_json::from_value::< - Raw, - >( - since_member.content.clone() - ) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| { - Error::bad_database("Invalid PDU in database.") + .and_then(|since_state| { + since_state + .get(&(EventType::RoomMember, user_id.clone())) + .and_then(|since_member| { + serde_json::from_value::< + Raw, + >( + since_member.content.clone() + ) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| { + Error::bad_database("Invalid PDU in database.") + }) + .ok() }) - .ok() - }) }) .map_or(MembershipState::Leave, |member| member.membership) }); - let user_id = UserId::try_from(user_id) + let user_id = UserId::try_from(user_id.clone()) .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; match (since_membership, current_membership) { @@ -456,7 +464,12 @@ pub async fn sync_events_route( }) .and_then(|state_hash| { db.rooms - .state_get(&state_hash, &EventType::RoomMember, sender_user.as_str()) + .state_get( + &room_id, + &state_hash, + &EventType::RoomMember, + sender_user.as_str(), + ) .ok()? .ok_or_else(|| Error::bad_database("State hash in db doesn't have a state.")) .ok() diff --git a/src/database.rs b/src/database.rs index 51505178..99bba832 100644 --- a/src/database.rs +++ b/src/database.rs @@ -20,6 +20,7 @@ use serde::Deserialize; use std::collections::HashMap; use std::sync::{Arc, RwLock}; use std::{convert::TryInto, fs::remove_dir_all}; +use tokio::sync::Semaphore; #[derive(Clone, Deserialize)] pub struct Config { @@ -30,6 +31,8 @@ pub struct Config { cache_capacity: u64, #[serde(default = "default_max_request_size")] max_request_size: u32, + #[serde(default = "default_max_concurrent_requests")] + max_concurrent_requests: u16, #[serde(default)] registration_disabled: bool, #[serde(default)] @@ -39,7 +42,9 @@ pub struct Config { } fn default_server_name() -> Box { - "localhost".try_into().expect("") + "localhost" + .try_into() + .expect("localhost is valid servername") } fn default_cache_capacity() -> u64 { @@ -50,6 +55,10 @@ fn default_max_request_size() -> u32 { 20 * 1024 * 1024 // Default to 20 MB } +fn default_max_concurrent_requests() -> u16 { + 4 +} + #[derive(Clone)] pub struct Database { pub globals: globals::Globals, @@ -159,6 +168,7 @@ impl Database { roomuserid_invited: db.open_tree("roomuserid_invited")?, userroomid_left: db.open_tree("userroomid_left")?, + statekey_short: db.open_tree("statekey_short")?, stateid_pduid: db.open_tree("stateid_pduid")?, pduid_statehash: db.open_tree("pduid_statehash")?, roomid_statehash: db.open_tree("roomid_statehash")?, @@ -180,6 +190,7 @@ impl Database { sending: sending::Sending { servernamepduids: db.open_tree("servernamepduids")?, servercurrentpdus: db.open_tree("servercurrentpdus")?, + maximum_requests: Arc::new(Semaphore::new(10)), }, admin: admin::Admin { sender: admin_sender, diff --git a/src/database/globals.rs b/src/database/globals.rs index e913c0f2..485650f2 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -4,6 +4,7 @@ use ruma::ServerName; use std::collections::HashMap; use std::sync::Arc; use std::sync::RwLock; +use std::time::Duration; use trust_dns_resolver::TokioAsyncResolver; pub const COUNTER: &str = "c"; @@ -54,11 +55,18 @@ impl Globals { } }; + let reqwest_client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(30)) + .timeout(Duration::from_secs(60 * 3)) + .pool_max_idle_per_host(1) + .build() + .unwrap(); + Ok(Self { globals, config, keypair: Arc::new(keypair), - reqwest_client: reqwest::Client::new(), + reqwest_client, dns_resolver: TokioAsyncResolver::tokio_from_system_conf() .await .map_err(|_| { diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 3e2a17f7..3f096a9f 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -62,7 +62,8 @@ pub struct Rooms { /// Remember the state hash at events in the past. pub(super) pduid_statehash: sled::Tree, /// The state for a given state hash. - pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + EventType + StateKey + pub(super) statekey_short: sled::Tree, // StateKey = EventType + StateKey, Short = Count + pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + Short, PduId = Count (without roomid) } impl StateStore for Rooms { @@ -106,21 +107,28 @@ impl StateStore for Rooms { impl Rooms { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. - pub fn state_full(&self, state_hash: &StateHashId) -> Result> { + pub fn state_full( + &self, + room_id: &RoomId, + state_hash: &StateHashId, + ) -> Result> { self.stateid_pduid .scan_prefix(&state_hash) .values() - .map(|pduid| { - self.pduid_pdu.get(&pduid?)?.map_or_else( - || Err(Error::bad_database("Failed to find StateMap.")), + .map(|pduid_short| { + let mut pduid = room_id.as_bytes().to_vec(); + pduid.push(0xff); + pduid.extend_from_slice(&pduid_short?); + self.pduid_pdu.get(&pduid)?.map_or_else( + || Err(Error::bad_database("Failed to find PDU in state snapshot.")), |b| { serde_json::from_slice::(&b) .map_err(|_| Error::bad_database("Invalid PDU in db.")) }, ) }) + .filter_map(|r| r.ok()) .map(|pdu| { - let pdu = pdu?; Ok(( ( pdu.kind.clone(), @@ -135,64 +143,45 @@ impl Rooms { .collect::>>() } - /// Returns all state entries for this type. - pub fn state_type( - &self, - state_hash: &StateHashId, - event_type: &EventType, - ) -> Result> { - let mut prefix = state_hash.to_vec(); - prefix.push(0xff); - prefix.extend_from_slice(&event_type.to_string().as_bytes()); - prefix.push(0xff); - - let mut hashmap = HashMap::new(); - for pdu in self - .stateid_pduid - .scan_prefix(&prefix) - .values() - .map(|pdu_id| { - Ok::<_, Error>( - serde_json::from_slice::(&self.pduid_pdu.get(pdu_id?)?.ok_or_else( - || Error::bad_database("PDU in state not found in database."), - )?) - .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?, - ) - }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert(state_key, pdu); - } - Ok(hashmap) - } - /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`). pub fn state_get( &self, + room_id: &RoomId, state_hash: &StateHashId, event_type: &EventType, state_key: &str, ) -> Result> { - let mut key = state_hash.to_vec(); - key.push(0xff); - key.extend_from_slice(&event_type.to_string().as_bytes()); + let mut key = event_type.to_string().as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(&state_key.as_bytes()); - self.stateid_pduid.get(&key)?.map_or(Ok(None), |pdu_id| { - Ok::<_, Error>(Some(( - pdu_id.clone(), - serde_json::from_slice::( - &self.pduid_pdu.get(&pdu_id)?.ok_or_else(|| { - Error::bad_database("PDU in state not found in database.") - })?, - ) - .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?, - ))) - }) + let short = self.statekey_short.get(&key)?; + + if let Some(short) = short { + let mut stateid = state_hash.to_vec(); + stateid.push(0xff); + stateid.extend_from_slice(&short); + + self.stateid_pduid + .get(&stateid)? + .map_or(Ok(None), |pdu_id_short| { + let mut pdu_id = room_id.as_bytes().to_vec(); + pdu_id.push(0xff); + pdu_id.extend_from_slice(&pdu_id_short); + + Ok::<_, Error>(Some(( + pdu_id.clone().into(), + serde_json::from_slice::( + &self.pduid_pdu.get(&pdu_id)?.ok_or_else(|| { + Error::bad_database("PDU in state not found in database.") + })?, + ) + .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?, + ))) + }) + } else { + return Ok(None); + } } /// Returns the last state hash key added to the db. @@ -260,6 +249,7 @@ impl Rooms { &self, room_id: &RoomId, state: HashMap<(EventType, String), Vec>, + globals: &super::globals::Globals, ) -> Result<()> { let state_hash = self.calculate_hash(&state.values().map(|pdu_id| &**pdu_id).collect::>())?; @@ -267,11 +257,29 @@ impl Rooms { prefix.push(0xff); for ((event_type, state_key), pdu_id) in state { + let mut statekey = event_type.as_ref().as_bytes().to_vec(); + statekey.push(0xff); + statekey.extend_from_slice(&state_key.as_bytes()); + + let short = match self.statekey_short.get(&statekey)? { + Some(short) => utils::u64_from_bytes(&short) + .map_err(|_| Error::bad_database("Invalid short bytes in statekey_short."))?, + None => { + let short = globals.next_count()?; + self.statekey_short + .insert(&statekey, &short.to_be_bytes())?; + short + } + }; + + let pdu_id_short = pdu_id + .splitn(2, |&b| b == 0xff) + .nth(1) + .ok_or_else(|| Error::bad_database("Invalid pduid in state."))?; + let mut state_id = prefix.clone(); - state_id.extend_from_slice(&event_type.as_ref().as_bytes()); - state_id.push(0xff); - state_id.extend_from_slice(&state_key.as_bytes()); - self.stateid_pduid.insert(state_id, pdu_id)?; + state_id.extend_from_slice(&short.to_be_bytes()); + self.stateid_pduid.insert(state_id, pdu_id_short)?; } self.roomid_statehash @@ -283,25 +291,12 @@ impl Rooms { /// Returns the full room state. pub fn room_state_full(&self, room_id: &RoomId) -> Result> { if let Some(current_state_hash) = self.current_state_hash(room_id)? { - self.state_full(¤t_state_hash) + self.state_full(&room_id, ¤t_state_hash) } else { Ok(BTreeMap::new()) } } - /// Returns all state entries for this type. - pub fn room_state_type( - &self, - room_id: &RoomId, - event_type: &EventType, - ) -> Result> { - if let Some(current_state_hash) = self.current_state_hash(room_id)? { - self.state_type(¤t_state_hash, event_type) - } else { - Ok(HashMap::new()) - } - } - /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`). pub fn room_state_get( &self, @@ -310,7 +305,7 @@ impl Rooms { state_key: &str, ) -> Result> { if let Some(current_state_hash) = self.current_state_hash(room_id)? { - self.state_get(¤t_state_hash, event_type, state_key) + self.state_get(&room_id, ¤t_state_hash, event_type, state_key) } else { Ok(None) } @@ -593,7 +588,12 @@ impl Rooms { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. /// The incoming event is the `pdu_id` passed to this method. - pub fn append_to_state(&self, new_pdu_id: &[u8], new_pdu: &PduEvent) -> Result { + pub fn append_to_state( + &self, + new_pdu_id: &[u8], + new_pdu: &PduEvent, + globals: &super::globals::Globals, + ) -> Result { let old_state = if let Some(old_state_hash) = self.roomid_statehash.get(new_pdu.room_id.as_bytes())? { // Store state for event. The state does not include the event itself. @@ -608,7 +608,7 @@ impl Rooms { self.stateid_pduid .scan_prefix(&prefix) .filter_map(|pdu| pdu.map_err(|e| error!("{}", e)).ok()) - // Chop the old state_hash out leaving behind the (EventType, StateKey) + // Chop the old state_hash out leaving behind the short key (u64) .map(|(k, v)| (k.subslice(prefix.len(), k.len() - prefix.len()), v)) .collect::>() } else { @@ -620,7 +620,23 @@ impl Rooms { let mut pdu_key = new_pdu.kind.as_ref().as_bytes().to_vec(); pdu_key.push(0xff); pdu_key.extend_from_slice(state_key.as_bytes()); - new_state.insert(pdu_key.into(), new_pdu_id.into()); + + let short = match self.statekey_short.get(&pdu_key)? { + Some(short) => utils::u64_from_bytes(&short) + .map_err(|_| Error::bad_database("Invalid short bytes in statekey_short."))?, + None => { + let short = globals.next_count()?; + self.statekey_short.insert(&pdu_key, &short.to_be_bytes())?; + short + } + }; + + let new_pdu_id_short = new_pdu_id + .splitn(2, |&b| b == 0xff) + .nth(1) + .ok_or_else(|| Error::bad_database("Invalid pduid in state."))?; + + new_state.insert((&short.to_be_bytes()).into(), new_pdu_id_short.into()); let new_state_hash = self.calculate_hash(&new_state.values().map(|b| &**b).collect::>())?; @@ -628,12 +644,10 @@ impl Rooms { let mut key = new_state_hash.to_vec(); key.push(0xff); - // TODO: we could avoid writing to the DB on every state event by keeping - // track of the delta and write that every so often - for (key_without_prefix, pdu_id) in new_state { + for (short, short_pdu_id) in new_state { let mut state_id = key.clone(); - state_id.extend_from_slice(&key_without_prefix); - self.stateid_pduid.insert(&state_id, &pdu_id)?; + state_id.extend_from_slice(&short); + self.stateid_pduid.insert(&state_id, &short_pdu_id)?; } self.roomid_statehash @@ -887,7 +901,7 @@ impl Rooms { // We append to state before appending the pdu, so we don't have a moment in time with the // pdu without it's state. This is okay because append_pdu can't fail. - self.append_to_state(&pdu_id, &pdu)?; + self.append_to_state(&pdu_id, &pdu, &globals)?; self.append_pdu( &pdu, diff --git a/src/database/sending.rs b/src/database/sending.rs index 7ce7d633..f21b1544 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -1,21 +1,29 @@ -use std::{collections::HashMap, convert::TryFrom, time::SystemTime}; +use std::{ + collections::HashMap, + convert::TryFrom, + fmt::Debug, + sync::Arc, + time::{Duration, Instant, SystemTime}, +}; use crate::{appservice_server, server_server, utils, Error, PduEvent, Result}; use federation::transactions::send_transaction_message; use log::warn; use rocket::futures::stream::{FuturesUnordered, StreamExt}; use ruma::{ - api::{appservice, federation}, + api::{appservice, federation, OutgoingRequest}, ServerName, }; use sled::IVec; use tokio::select; +use tokio::sync::Semaphore; #[derive(Clone)] pub struct Sending { /// The state for a given state hash. pub(super) servernamepduids: sled::Tree, // ServernamePduId = (+)ServerName + PduId pub(super) servercurrentpdus: sled::Tree, // ServerCurrentPdus = (+)ServerName + PduId (pduid can be empty for reservation) + pub(super) maximum_requests: Arc, } impl Sending { @@ -40,35 +48,7 @@ impl Sending { for (server, pdu, is_appservice) in servercurrentpdus .iter() .filter_map(|r| r.ok()) - .map(|(key, _)| { - let mut parts = key.splitn(2, |&b| b == 0xff); - let server = parts.next().expect("splitn always returns one element"); - let pdu = parts.next().ok_or_else(|| { - Error::bad_database("Invalid bytes in servercurrentpdus.") - })?; - - let server = utils::string_from_bytes(&server).map_err(|_| { - Error::bad_database("Invalid server bytes in server_currenttransaction") - })?; - - // Appservices start with a plus - let (server, is_appservice) = if server.starts_with("+") { - (&server[1..], true) - } else { - (&*server, false) - }; - - Ok::<_, Error>(( - Box::::try_from(server).map_err(|_| { - Error::bad_database( - "Invalid server string in server_currenttransaction", - ) - })?, - IVec::from(pdu), - is_appservice, - )) - }) - .filter_map(|r| r.ok()) + .filter_map(|(key, _)| Self::parse_servercurrentpdus(key).ok()) .filter(|(_, pdu, _)| !pdu.is_empty()) // Skip reservation key .take(50) // This should not contain more than 50 anyway @@ -90,6 +70,8 @@ impl Sending { )); } + let mut last_failed_try: HashMap, (u32, Instant)> = HashMap::new(); + let mut subscriber = servernamepduids.watch_prefix(b""); loop { select! { @@ -140,9 +122,24 @@ impl Sending { // servercurrentpdus with the prefix should be empty now } } - Err((server, _is_appservice, e)) => { - warn!("Couldn't send transaction to {}: {}", server, e) - // TODO: exponential backoff + Err((server, is_appservice, e)) => { + warn!("Couldn't send transaction to {}: {}", server, e); + let mut prefix = if is_appservice { + "+".as_bytes().to_vec() + } else { + Vec::new() + }; + prefix.extend_from_slice(server.as_bytes()); + prefix.push(0xff); + last_failed_try.insert(server.clone(), match last_failed_try.get(&server) { + Some(last_failed) => { + (last_failed.0+1, Instant::now()) + }, + None => { + (1, Instant::now()) + } + }); + servercurrentpdus.remove(&prefix).unwrap(); } }; }, @@ -174,8 +171,19 @@ impl Sending { .ok() .map(|pdu_id| (server, is_appservice, pdu_id)) ) - // TODO: exponential backoff .filter(|(server, is_appservice, _)| { + if last_failed_try.get(server).map_or(false, |(tries, instant)| { + // Fail if a request has failed recently (exponential backoff) + let mut min_elapsed_duration = Duration::from_secs(60) * *tries * *tries; + if min_elapsed_duration > Duration::from_secs(60*60*24) { + min_elapsed_duration = Duration::from_secs(60*60*24); + } + + instant.elapsed() < min_elapsed_duration + }) { + return false; + } + let mut prefix = if *is_appservice { "+".as_bytes().to_vec() } else { @@ -308,4 +316,63 @@ impl Sending { .map_err(|e| (server, is_appservice, e)) } } + + fn parse_servercurrentpdus(key: IVec) -> Result<(Box, IVec, bool)> { + let mut parts = key.splitn(2, |&b| b == 0xff); + let server = parts.next().expect("splitn always returns one element"); + let pdu = parts + .next() + .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?; + + let server = utils::string_from_bytes(&server).map_err(|_| { + Error::bad_database("Invalid server bytes in server_currenttransaction") + })?; + + // Appservices start with a plus + let (server, is_appservice) = if server.starts_with("+") { + (&server[1..], true) + } else { + (&*server, false) + }; + + Ok::<_, Error>(( + Box::::try_from(server).map_err(|_| { + Error::bad_database("Invalid server string in server_currenttransaction") + })?, + IVec::from(pdu), + is_appservice, + )) + } + + pub async fn send_federation_request( + &self, + globals: &crate::database::globals::Globals, + destination: Box, + request: T, + ) -> Result + where + T: Debug, + { + let permit = self.maximum_requests.acquire().await; + let response = server_server::send_request(globals, destination, request).await; + drop(permit); + + response + } + + pub async fn send_appservice_request( + &self, + globals: &crate::database::globals::Globals, + registration: serde_yaml::Value, + request: T, + ) -> Result + where + T: Debug, + { + let permit = self.maximum_requests.acquire().await; + let response = appservice_server::send_request(globals, registration, request).await; + drop(permit); + + response + } } diff --git a/src/error.rs b/src/error.rs index 7d4a7518..d8e9d023 100644 --- a/src/error.rs +++ b/src/error.rs @@ -121,7 +121,7 @@ impl log::Log for ConduitLogger { fn log(&self, record: &log::Record<'_>) { let output = format!("{} - {}", record.level(), record.args()); - println!("{}", output); + eprintln!("{}", output); if self.enabled(record.metadata()) && record diff --git a/src/main.rs b/src/main.rs index 95748943..38a2ec91 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,7 @@ pub use pdu::PduEvent; pub use rocket::State; pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse}; -use rocket::{fairing::AdHoc, routes}; +use rocket::{catch, catchers, fairing::AdHoc, routes, Request}; fn setup_rocket() -> rocket::Rocket { // Force log level off, so we can use our own logger @@ -70,6 +70,7 @@ fn setup_rocket() -> rocket::Rocket { client_server::get_backup_key_sessions_route, client_server::get_backup_keys_route, client_server::set_read_marker_route, + client_server::set_receipt_route, client_server::create_typing_event_route, client_server::create_room_route, client_server::redact_event_route, @@ -134,6 +135,7 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_profile_information_route, ], ) + .register(catchers![not_found_catcher]) .attach(AdHoc::on_attach("Config", |rocket| async { let data = Database::load_or_create(rocket.figment().extract().expect("config is valid")) @@ -157,3 +159,8 @@ fn setup_rocket() -> rocket::Rocket { async fn main() { setup_rocket().launch().await.unwrap(); } + +#[catch(404)] +fn not_found_catcher(_req: &'_ Request<'_>) -> String { + "404 Not Found".to_owned() +} diff --git a/src/server_server.rs b/src/server_server.rs index 7d12c54c..06539593 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -490,7 +490,7 @@ pub async fn send_transaction_message_route<'a>( pdu_id.push(0xff); pdu_id.extend_from_slice(&count.to_be_bytes()); - db.rooms.append_to_state(&pdu_id, &pdu)?; + db.rooms.append_to_state(&pdu_id, &pdu, &db.globals)?; db.rooms.append_pdu( &pdu, From f12fbca3c559ad9f441c497f9659fdfcce6208db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 22 Dec 2020 20:08:20 +0100 Subject: [PATCH 21/36] fix: send state in /sync, element displays wrong membership changes --- src/client_server/membership.rs | 2 +- src/client_server/sync.rs | 34 +++++++++++++++++++++----------- src/database/rooms.rs | 35 +++++++++++++++++++++++++++++---- src/server_server.rs | 2 +- 4 files changed, 55 insertions(+), 18 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index e8d57bc1..b459d37d 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -686,7 +686,7 @@ async fn join_room_by_id_helper( pdu_id.extend_from_slice(&count.to_be_bytes()); db.rooms.append_pdu( &PduEvent::from(&**pdu), - &utils::to_canonical_object(&**pdu).expect("Pdu is valid canonical object"), + utils::to_canonical_object(&**pdu).expect("Pdu is valid canonical object"), count, pdu_id.clone().into(), &db.globals, diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 82136511..31361164 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -91,15 +91,7 @@ pub async fn sync_events_route( // They /sync response doesn't always return all messages, so we say the output is // limited unless there are events in non_timeline_pdus - let mut limited = false; - - let mut state_pdus = Vec::new(); - for (_, pdu) in non_timeline_pdus { - if pdu.state_key.is_some() { - state_pdus.push(pdu); - } - limited = true; - } + let limited = non_timeline_pdus.next().is_some(); // Database queries: @@ -342,7 +334,7 @@ pub async fn sync_events_route( })?; let room_events = timeline_pdus - .into_iter() + .iter() .map(|(_, pdu)| pdu.to_sync_room_event()) .collect::>(); @@ -392,7 +384,6 @@ pub async fn sync_events_route( prev_batch, events: room_events, }, - // TODO: state before timeline state: sync_events::State { events: if joined_since_last_sync { db.rooms @@ -401,7 +392,26 @@ pub async fn sync_events_route( .map(|(_, pdu)| pdu.to_sync_state_event()) .collect() } else { - Vec::new() + match since_state { + None => Vec::new(), + Some(Some(since_state)) => current_state + .iter() + .filter(|(key, value)| { + since_state.get(key).map(|e| &e.event_id) != Some(&value.event_id) + }) + .filter(|(_, value)| { + !timeline_pdus.iter().any(|(_, timeline_pdu)| { + timeline_pdu.kind == value.kind + && timeline_pdu.state_key == value.state_key + }) + }) + .map(|(_, pdu)| pdu.to_sync_state_event()) + .collect(), + Some(None) => current_state + .iter() + .map(|(_, pdu)| pdu.to_sync_state_event()) + .collect(), + } }, }, ephemeral: sync_events::Ephemeral { events: edus }, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 3f096a9f..e59c77f4 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -15,7 +15,7 @@ use ruma::{ }, EventType, }, - serde::{to_canonical_value, CanonicalJsonObject, Raw}, + serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue, Raw}, EventId, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId, }; use sled::IVec; @@ -444,13 +444,40 @@ impl Rooms { pub fn append_pdu( &self, pdu: &PduEvent, - pdu_json: &CanonicalJsonObject, + mut pdu_json: CanonicalJsonObject, count: u64, pdu_id: IVec, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, admin: &super::admin::Admin, ) -> Result<()> { + // Make unsigned fields correct. This is not properly documented in the spec, but state + // events need to have previous content in the unsigned field, so clients can easily + // interpret things like membership changes + if let Some(state_key) = &pdu.state_key { + if let CanonicalJsonValue::Object(unsigned) = pdu_json + .entry("unsigned".to_owned()) + .or_insert_with(|| CanonicalJsonValue::Object(Default::default())) + { + if let Some(prev_state_hash) = self.pdu_state_hash(&pdu_id).unwrap() { + if let Some(prev_state) = self + .state_get(&pdu.room_id, &prev_state_hash, &pdu.kind, &state_key) + .unwrap() + { + unsigned.insert( + "prev_content".to_owned(), + CanonicalJsonValue::Object( + utils::to_canonical_object(prev_state.1.content) + .expect("event is valid, we just created it"), + ), + ); + } + } + } else { + error!("Invalid unsigned type in pdu."); + } + } + self.replace_pdu_leaves(&pdu.room_id, &pdu.event_id)?; // Mark as read first so the sending client doesn't get a notification even if appending @@ -460,7 +487,7 @@ impl Rooms { self.pduid_pdu.insert( &pdu_id, - &*serde_json::to_string(pdu_json) + &*serde_json::to_string(&pdu_json) .expect("CanonicalJsonObject is always a valid String"), )?; @@ -905,7 +932,7 @@ impl Rooms { self.append_pdu( &pdu, - &pdu_json, + pdu_json, count, pdu_id.clone().into(), globals, diff --git a/src/server_server.rs b/src/server_server.rs index 06539593..7abce5a0 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -494,7 +494,7 @@ pub async fn send_transaction_message_route<'a>( db.rooms.append_pdu( &pdu, - &value, + value, count, pdu_id.clone().into(), &db.globals, From 8dcc1dfe56f1ac9ce3b0b98e8876478898ebb3ef Mon Sep 17 00:00:00 2001 From: Leonhard Kuboschek Date: Tue, 8 Dec 2020 12:34:46 +0100 Subject: [PATCH 22/36] improvement: always use port from SRV lookups Also query SRV records when well-known is not found, fixes #29 --- src/server_server.rs | 199 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 157 insertions(+), 42 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index 7abce5a0..c47afab1 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -24,30 +24,12 @@ use std::{ collections::BTreeMap, convert::TryFrom, fmt::Debug, + net::{IpAddr, SocketAddr}, time::{Duration, SystemTime}, }; -pub async fn request_well_known( - globals: &crate::database::globals::Globals, - destination: &str, -) -> Option { - let body: serde_json::Value = serde_json::from_str( - &globals - .reqwest_client() - .get(&format!( - "https://{}/.well-known/matrix/server", - destination - )) - .send() - .await - .ok()? - .text() - .await - .ok()?, - ) - .ok()?; - Some(body.get("m.server")?.as_str()?.to_owned()) -} + + pub async fn send_request( globals: &crate::database::globals::Globals, @@ -215,42 +197,130 @@ where } } +fn get_ip_with_port(destination_str: String) -> Option { + if destination_str.parse::().is_ok() { + Some(destination_str) + } else if let Ok(ip_addr) = destination_str.parse::() { + Some(SocketAddr::new(ip_addr, 8448).to_string()) + } else { + None + } +} + +fn add_port_to_hostname(destination_str: String) -> String { + match destination_str.find(':') { + None => destination_str.to_owned() + ":8448", + Some(_) => destination_str.to_string(), + } +} + /// Returns: actual_destination, host header +/// Implemented according to the specification at https://matrix.org/docs/spec/server_server/r0.1.4#resolving-server-names +/// Numbers in comments below refer to bullet points in linked section of specification async fn find_actual_destination( globals: &crate::database::globals::Globals, destination: &Box, ) -> (String, Option) { let mut host = None; + let destination_str = destination.as_str().to_owned(); let actual_destination = "https://".to_owned() - + &if let Some(mut delegated_hostname) = - request_well_known(globals, destination.as_str()).await - { - if let Ok(Some(srv)) = globals - .dns_resolver() - .srv_lookup(format!("_matrix._tcp.{}", delegated_hostname)) - .await - .map(|srv| srv.iter().next().map(|result| result.target().to_string())) - { - host = Some(delegated_hostname); - srv.trim_end_matches('.').to_owned() - } else { - if delegated_hostname.find(':').is_none() { - delegated_hostname += ":8448"; - } - delegated_hostname + + &match get_ip_with_port(destination_str.clone()) { + Some(host_port) => { + // 1: IP literal with provided or default port + host_port } - } else { - let mut destination = destination.as_str().to_owned(); - if destination.find(':').is_none() { - destination += ":8448"; + None => { + if destination_str.find(':').is_some() { + // 2: Hostname with included port + destination_str + } else { + match request_well_known(globals, &destination.as_str()).await { + // 3: A .well-known file is available + Some(delegated_hostname) => { + match get_ip_with_port(delegated_hostname.clone()) { + Some(host_and_port) => host_and_port, // 3.1: IP literal in .well-known file + None => { + if destination_str.find(':').is_some() { + // 3.2: Hostname with port in .well-known file + destination_str + } else { + match query_srv_record(globals, &delegated_hostname).await { + // 3.3: SRV lookup successful + Some(hostname) => hostname, + // 3.4: No SRV records, just use the hostname from .well-known + None => add_port_to_hostname(delegated_hostname), + } + } + } + } + } + // 4: No .well-known or an error occured + None => { + match query_srv_record(globals, &destination_str).await { + // 4: SRV record found + Some(hostname) => { + host = Some(destination_str.to_owned()); + hostname + } + // 5: No SRV record found + None => add_port_to_hostname(destination_str.to_string()), + } + } + } + } } - destination }; (actual_destination, host) } +async fn query_srv_record<'a>( + globals: &crate::database::globals::Globals, + hostname: &'a str, +) -> Option { + if let Ok(Some(host_port)) = globals + .dns_resolver() + .srv_lookup(format!("_matrix._tcp.{}", hostname)) + .await + .map(|srv| { + srv.iter().next().map(|result| { + format!( + "{}:{}", + result.target().to_string().trim_end_matches('.'), + result.port().to_string() + ) + }) + }) + { + Some(host_port) + } else { + None + } +} + +pub async fn request_well_known( + globals: &crate::database::globals::Globals, + destination: &str, +) -> Option { + let body: serde_json::Value = serde_json::from_str( + &globals + .reqwest_client() + .get(&format!( + "https://{}/.well-known/matrix/server", + destination + )) + .send() + .await + .ok()? + .text() + .await + .ok()?, + ) + .ok()?; + Some(body.get("m.server")?.as_str()?.to_owned()) +} + #[cfg_attr(feature = "conduit_bin", get("/_matrix/federation/v1/version"))] pub fn get_server_version_route( db: State<'_, Database>, @@ -622,3 +692,48 @@ pub fn get_user_devices_route<'a>( .into()) } */ + +#[cfg(test)] +mod tests { + use super::{add_port_to_hostname, get_ip_with_port}; + + #[test] + fn ips_get_default_ports() { + assert_eq!( + get_ip_with_port(String::from("1.1.1.1")), + Some(String::from("1.1.1.1:8448")) + ); + assert_eq!( + get_ip_with_port(String::from("dead:beef::")), + Some(String::from("[dead:beef::]:8448")) + ); + } + + #[test] + fn ips_keep_custom_ports() { + assert_eq!( + get_ip_with_port(String::from("1.1.1.1:1234")), + Some(String::from("1.1.1.1:1234")) + ); + assert_eq!( + get_ip_with_port(String::from("[dead::beef]:8933")), + Some(String::from("[dead::beef]:8933")) + ); + } + + #[test] + fn hostnames_get_default_ports() { + assert_eq!( + add_port_to_hostname(String::from("example.com")), + "example.com:8448" + ) + } + + #[test] + fn hostnames_keep_custom_ports() { + assert_eq!( + add_port_to_hostname(String::from("example.com:1337")), + "example.com:1337" + ) + } +} From fb9bd34696d78f02e1d8cf65e53f52ce2782ff0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 23 Dec 2020 15:53:41 +0100 Subject: [PATCH 23/36] improvement: better warnings when server is unreachable --- src/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/error.rs b/src/error.rs index d8e9d023..a5f2be42 100644 --- a/src/error.rs +++ b/src/error.rs @@ -34,7 +34,7 @@ pub enum Error { #[from] source: image::error::ImageError, }, - #[error("Could not connect to server.")] + #[error("Could not connect to server: {source}")] ReqwestError { #[from] source: reqwest::Error, From 2cf6fd57b7da57b116ca782130d97027a8eacb79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 23 Dec 2020 19:41:54 +0100 Subject: [PATCH 24/36] improvement: don't send pdus to appservices if it isn't interested TODO: we need to send pdus if a user of the appservice is in the room but not the appservice user itself --- Cargo.lock | 37 +++++++++++++++++++++++++++ Cargo.toml | 2 ++ src/database/rooms.rs | 59 ++++++++++++++++++++++++++++++++++++++++++- src/server_server.rs | 3 --- 4 files changed, 97 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6566b108..b5be6aaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,6 +21,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + [[package]] name = "arrayref" version = "0.3.6" @@ -190,6 +199,7 @@ dependencies = [ "js_int", "log", "rand", + "regex", "reqwest", "ring", "rocket", @@ -1418,6 +1428,24 @@ dependencies = [ "syn", ] +[[package]] +name = "regex" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -2131,6 +2159,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + [[package]] name = "time" version = "0.2.23" diff --git a/Cargo.toml b/Cargo.toml index 1e4afe2a..4b871996 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,8 @@ base64 = "0.13.0" ring = "0.16.19" # Used when querying the SRV record of other servers trust-dns-resolver = "0.19.6" +# Used to find matching events for appservices +regex = "1.4.2" [features] default = ["conduit_bin"] diff --git a/src/database/rooms.rs b/src/database/rooms.rs index e59c77f4..a6d8fea5 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -4,6 +4,7 @@ pub use edus::RoomEdus; use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; use log::error; +use regex::Regex; use ring::digest; use ruma::{ api::client::error::ErrorKind, @@ -949,7 +950,63 @@ impl Rooms { } for appservice in appservice.iter_all().filter_map(|r| r.ok()) { - sending.send_pdu_appservice(&appservice.0, &pdu_id)?; + if let Some(namespaces) = appservice.1.get("namespaces") { + let users = namespaces + .get("users") + .and_then(|users| users.as_sequence()) + .map_or_else( + || Vec::new(), + |users| { + users + .iter() + .map(|users| { + users + .get("regex") + .and_then(|regex| regex.as_str()) + .and_then(|regex| Regex::new(regex).ok()) + }) + .filter_map(|o| o) + .collect::>() + }, + ); + let aliases = namespaces + .get("aliases") + .and_then(|users| users.get("regex")) + .and_then(|regex| regex.as_str()) + .and_then(|regex| Regex::new(regex).ok()); + let rooms = namespaces + .get("rooms") + .and_then(|rooms| rooms.as_sequence()); + + let room_aliases = self.room_aliases(&room_id); + + let bridge_user_id = appservice + .1 + .get("sender_localpart") + .and_then(|string| string.as_str()) + .and_then(|string| { + UserId::parse_with_server_name(string, globals.server_name()).ok() + }); + + if bridge_user_id.map_or(false, |bridge_user_id| { + self.is_joined(&bridge_user_id, room_id).unwrap_or(false) + }) || users.iter().any(|users| { + dbg!( + users.is_match(pdu.sender.as_str()) + || pdu.kind == EventType::RoomMember + && pdu.state_key.as_ref().map_or(false, |state_key| dbg!( + users.is_match(dbg!(&state_key)) + )) + ) + }) || aliases.map_or(false, |aliases| { + room_aliases + .filter_map(|r| r.ok()) + .any(|room_alias| aliases.is_match(room_alias.as_str())) + }) || rooms.map_or(false, |rooms| rooms.contains(&room_id.as_str().into())) + { + sending.send_pdu_appservice(&appservice.0, &pdu_id)?; + } + } } Ok(pdu.event_id) diff --git a/src/server_server.rs b/src/server_server.rs index c47afab1..eb6b2373 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -28,9 +28,6 @@ use std::{ time::{Duration, SystemTime}, }; - - - pub async fn send_request( globals: &crate::database::globals::Globals, destination: Box, From df16b2ba9817bf56d7b125db33c9539cfacf56d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 31 Dec 2020 14:52:08 +0100 Subject: [PATCH 25/36] fix: rare state races --- src/database/rooms.rs | 29 ++++++++++++++++++----------- src/server_server.rs | 2 +- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index a6d8fea5..ab3dd3f3 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -678,9 +678,6 @@ impl Rooms { self.stateid_pduid.insert(&state_id, &short_pdu_id)?; } - self.roomid_statehash - .insert(new_pdu.room_id.as_bytes(), &*new_state_hash)?; - Ok(new_state_hash) } else { Err(Error::bad_database( @@ -689,6 +686,13 @@ impl Rooms { } } + pub fn set_room_state(&self, room_id: &RoomId, state_hash: &StateHashId) -> Result<()> { + self.roomid_statehash + .insert(room_id.as_bytes(), state_hash)?; + + Ok(()) + } + /// Creates a new persisted data unit and adds it to a room. #[allow(clippy::too_many_arguments)] pub fn build_and_append_pdu( @@ -929,7 +933,7 @@ impl Rooms { // We append to state before appending the pdu, so we don't have a moment in time with the // pdu without it's state. This is okay because append_pdu can't fail. - self.append_to_state(&pdu_id, &pdu, &globals)?; + let statehashid = self.append_to_state(&pdu_id, &pdu, &globals)?; self.append_pdu( &pdu, @@ -941,6 +945,10 @@ impl Rooms { admin, )?; + // We set the room state after inserting the pdu, so that we never have a moment in time + // where events in the current room state do not exist + self.set_room_state(&room_id, &statehashid)?; + for server in self .room_servers(room_id) .filter_map(|r| r.ok()) @@ -991,13 +999,12 @@ impl Rooms { if bridge_user_id.map_or(false, |bridge_user_id| { self.is_joined(&bridge_user_id, room_id).unwrap_or(false) }) || users.iter().any(|users| { - dbg!( - users.is_match(pdu.sender.as_str()) - || pdu.kind == EventType::RoomMember - && pdu.state_key.as_ref().map_or(false, |state_key| dbg!( - users.is_match(dbg!(&state_key)) - )) - ) + users.is_match(pdu.sender.as_str()) + || pdu.kind == EventType::RoomMember + && pdu + .state_key + .as_ref() + .map_or(false, |state_key| users.is_match(&state_key)) }) || aliases.map_or(false, |aliases| { room_aliases .filter_map(|r| r.ok()) diff --git a/src/server_server.rs b/src/server_server.rs index eb6b2373..7b980e30 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -95,7 +95,7 @@ where ruma::signatures::sign_json( globals.server_name().as_str(), globals.keypair(), - &mut request_json, + dbg!(&mut request_json), ) .expect("our request json is what ruma expects"); From edfd3c1f34aae8ec29461cf8a5f529a54291c714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 31 Dec 2020 21:07:05 +0100 Subject: [PATCH 26/36] improvement: better config, better logs --- src/appservice_server.rs | 6 +++--- src/client_server/search.rs | 4 ++-- src/database.rs | 35 +++++------------------------------ src/database/admin.rs | 2 +- src/database/sending.rs | 4 ++-- src/error.rs | 30 +++++++++++++++++++++++------- src/main.rs | 31 ++++++++++++++++++++++++------- src/server_server.rs | 16 ++++++++-------- 8 files changed, 68 insertions(+), 60 deletions(-) diff --git a/src/appservice_server.rs b/src/appservice_server.rs index f1436e0a..986909b6 100644 --- a/src/appservice_server.rs +++ b/src/appservice_server.rs @@ -78,10 +78,10 @@ where if status != 200 { warn!( - "Server returned bad response {} ({}): {} {:?}", + "Appservice returned bad response {} {}\n{}\n{:?}", destination, - url, status, + url, utils::string_from_bytes(&body) ); } @@ -93,7 +93,7 @@ where ); response.map_err(|_| { warn!( - "Server returned invalid response bytes {} ({})", + "Appservice returned invalid response bytes {}\n{}", destination, url ); Error::BadServerResponse("Server returned bad response.") diff --git a/src/client_server/search.rs b/src/client_server/search.rs index 0950b25a..5fb87f01 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -77,8 +77,8 @@ pub async fn search_events_route( Ok(search_events::Response::new(ResultCategories { room_events: ResultRoomEvents { - count: None, // TODO? maybe not - groups: BTreeMap::new(), // TODO + count: Some((results.len() as u32).into()), // TODO: set this to none. Element shouldn't depend on it + groups: BTreeMap::new(), // TODO next_batch, results, state: BTreeMap::new(), // TODO diff --git a/src/database.rs b/src/database.rs index 99bba832..84be5785 100644 --- a/src/database.rs +++ b/src/database.rs @@ -18,15 +18,14 @@ use rocket::futures::{self, channel::mpsc}; use ruma::{DeviceId, ServerName, UserId}; use serde::Deserialize; use std::collections::HashMap; +use std::fs::remove_dir_all; use std::sync::{Arc, RwLock}; -use std::{convert::TryInto, fs::remove_dir_all}; use tokio::sync::Semaphore; #[derive(Clone, Deserialize)] pub struct Config { - #[serde(default = "default_server_name")] server_name: Box, - database_path: Option, + database_path: String, #[serde(default = "default_cache_capacity")] cache_capacity: u64, #[serde(default = "default_max_request_size")] @@ -41,12 +40,6 @@ pub struct Config { federation_enabled: bool, } -fn default_server_name() -> Box { - "localhost" - .try_into() - .expect("localhost is valid servername") -} - fn default_cache_capacity() -> u64 { 1024 * 1024 * 1024 } @@ -90,31 +83,13 @@ impl Database { /// Load an existing database or create a new one. pub async fn load_or_create(config: Config) -> Result { - let path = config - .database_path - .clone() - .map(Ok::<_, Error>) - .unwrap_or_else(|| { - let path = ProjectDirs::from("xyz", "koesters", "conduit") - .ok_or_else(|| { - Error::bad_config("The OS didn't return a valid home directory path.") - })? - .data_dir() - .join(config.server_name.as_str()); - - Ok(path - .to_str() - .ok_or_else(|| Error::bad_config("Database path contains invalid unicode."))? - .to_owned()) - })?; - let db = sled::Config::default() - .path(&path) + .path(&config.database_path) .cache_capacity(config.cache_capacity) - .print_profile_on_drop(true) + .print_profile_on_drop(false) .open()?; - info!("Opened sled database at {}", path); + info!("Opened sled database at {}", config.database_path); let (admin_sender, admin_receiver) = mpsc::unbounded(); diff --git a/src/database/admin.rs b/src/database/admin.rs index 7de6bf92..1fb19836 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -43,7 +43,7 @@ impl Admin { .unwrap(); if conduit_room.is_none() { - warn!("Conduit instance does not have an #admins room. Logging to that room will not work."); + warn!("Conduit instance does not have an #admins room. Logging to that room will not work. Restart Conduit after creating a user to fix this."); } let send_message = |message: message::MessageEventContent| { diff --git a/src/database/sending.rs b/src/database/sending.rs index f21b1544..74aad326 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -8,7 +8,7 @@ use std::{ use crate::{appservice_server, server_server, utils, Error, PduEvent, Result}; use federation::transactions::send_transaction_message; -use log::warn; +use log::info; use rocket::futures::stream::{FuturesUnordered, StreamExt}; use ruma::{ api::{appservice, federation, OutgoingRequest}, @@ -123,7 +123,7 @@ impl Sending { } } Err((server, is_appservice, e)) => { - warn!("Couldn't send transaction to {}: {}", server, e); + info!("Couldn't send transaction to {}\n{}", server, e); let mut prefix = if is_appservice { "+".as_bytes().to_vec() } else { diff --git a/src/error.rs b/src/error.rs index a5f2be42..c57843cc 100644 --- a/src/error.rs +++ b/src/error.rs @@ -121,29 +121,45 @@ impl log::Log for ConduitLogger { fn log(&self, record: &log::Record<'_>) { let output = format!("{} - {}", record.level(), record.args()); - eprintln!("{}", output); - if self.enabled(record.metadata()) - && record + && (record .module_path() .map_or(false, |path| path.starts_with("conduit::")) + || record + .module_path() + .map_or(true, |path| !path.starts_with("rocket::")) // Rockets logs are annoying + && record.metadata().level() <= log::Level::Warn) { + let first_line = output + .lines() + .next() + .expect("lines always returns one item"); + + eprintln!("{}", output); + + let mute_duration = match record.metadata().level() { + log::Level::Error => Duration::from_secs(60 * 5), // 5 minutes + log::Level::Warn => Duration::from_secs(60 * 60 * 24), // A day + _ => Duration::from_secs(60 * 60 * 24 * 7), // A week + }; + if self .last_logs .read() .unwrap() - .get(&output) - .map_or(false, |i| i.elapsed() < Duration::from_secs(60 * 30)) + .get(first_line) + .map_or(false, |i| i.elapsed() < mute_duration) + // Don't post this log again for some time { return; } if let Ok(mut_last_logs) = &mut self.last_logs.try_write() { - mut_last_logs.insert(output.clone(), Instant::now()); + mut_last_logs.insert(first_line.to_owned(), Instant::now()); } self.db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain(output), + message::MessageEventContent::notice_plain(output), )); } } diff --git a/src/main.rs b/src/main.rs index 38a2ec91..9c0eab65 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,18 +13,32 @@ mod utils; pub use database::Database; pub use error::{ConduitLogger, Error, Result}; -use log::LevelFilter; pub use pdu::PduEvent; pub use rocket::State; pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse}; +use log::LevelFilter; +use rocket::figment::{ + providers::{Env, Format, Toml}, + Figment, +}; use rocket::{catch, catchers, fairing::AdHoc, routes, Request}; fn setup_rocket() -> rocket::Rocket { // Force log level off, so we can use our own logger - std::env::set_var("ROCKET_LOG_LEVEL", "off"); + std::env::set_var("CONDUIT_LOG_LEVEL", "off"); + + let config = + Figment::from(rocket::Config::release_default()) + .merge( + Toml::file(Env::var("CONDUIT_CONFIG").expect( + "The CONDUIT_CONFIG env var needs to be set. Example: /etc/conduit.toml", + )) + .nested(), + ) + .merge(Env::prefixed("CONDUIT_").global()); - rocket::ignite() + rocket::custom(config) .mount( "/", routes![ @@ -137,10 +151,13 @@ fn setup_rocket() -> rocket::Rocket { ) .register(catchers![not_found_catcher]) .attach(AdHoc::on_attach("Config", |rocket| async { - let data = - Database::load_or_create(rocket.figment().extract().expect("config is valid")) - .await - .expect("config is valid"); + let config = rocket + .figment() + .extract() + .expect("It looks like your config is invalid. Please take a look at the error"); + let data = Database::load_or_create(config) + .await + .expect("config is valid"); data.sending .start_handler(&data.globals, &data.rooms, &data.appservice); diff --git a/src/server_server.rs b/src/server_server.rs index 7b980e30..87e986d5 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,7 +1,7 @@ use crate::{client_server, utils, ConduitResult, Database, Error, PduEvent, Result, Ruma}; use get_profile_information::v1::ProfileField; use http::header::{HeaderValue, AUTHORIZATION, HOST}; -use log::warn; +use log::{info, warn}; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ api::{ @@ -95,7 +95,7 @@ where ruma::signatures::sign_json( globals.server_name().as_str(), globals.keypair(), - dbg!(&mut request_json), + &mut request_json, ) .expect("our request json is what ruma expects"); @@ -161,18 +161,18 @@ where .bytes() .await .unwrap_or_else(|e| { - warn!("server error: {}", e); + warn!("server error {}", e); Vec::new().into() }) // TODO: handle timeout .into_iter() .collect::>(); if status != 200 { - warn!( - "Server returned bad response {} ({}): {} {:?}", + info!( + "Server returned bad response {} {}\n{}\n{:?}", destination, - url, status, + url, utils::string_from_bytes(&body) ); } @@ -183,8 +183,8 @@ where .expect("reqwest body is valid http body"), ); response.map_err(|_| { - warn!( - "Server returned invalid response bytes {} ({})", + info!( + "Server returned invalid response bytes {}\n{}", destination, url ); Error::BadServerResponse("Server returned bad response.") From 85364a9c27b950a8de821726bc5406bde04713c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 1 Jan 2021 10:52:43 +0100 Subject: [PATCH 27/36] improvement: change federation_enabled to federation_disabled This enables federation by default --- src/database.rs | 2 +- src/database/globals.rs | 4 ++-- src/server_server.rs | 18 +++++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/database.rs b/src/database.rs index 84be5785..99ea9f5e 100644 --- a/src/database.rs +++ b/src/database.rs @@ -37,7 +37,7 @@ pub struct Config { #[serde(default)] encryption_disabled: bool, #[serde(default)] - federation_enabled: bool, + federation_disabled: bool, } fn default_cache_capacity() -> u64 { diff --git a/src/database/globals.rs b/src/database/globals.rs index 485650f2..5444d6ec 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -119,8 +119,8 @@ impl Globals { self.config.encryption_disabled } - pub fn federation_enabled(&self) -> bool { - self.config.federation_enabled + pub fn federation_disabled(&self) -> bool { + self.config.federation_disabled } pub fn dns_resolver(&self) -> &TokioAsyncResolver { diff --git a/src/server_server.rs b/src/server_server.rs index 87e986d5..88b6a01c 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -36,7 +36,7 @@ pub async fn send_request( where T: Debug, { - if !globals.federation_enabled() { + if globals.federation_disabled() { return Err(Error::bad_config("Federation is disabled.")); } @@ -322,7 +322,7 @@ pub async fn request_well_known( pub fn get_server_version_route( db: State<'_, Database>, ) -> ConduitResult { - if !db.globals.federation_enabled() { + if db.globals.federation_disabled() { return Err(Error::bad_config("Federation is disabled.")); } @@ -337,7 +337,7 @@ pub fn get_server_version_route( #[cfg_attr(feature = "conduit_bin", get("/_matrix/key/v2/server"))] pub fn get_server_keys_route(db: State<'_, Database>) -> Json { - if !db.globals.federation_enabled() { + if db.globals.federation_disabled() { // TODO: Use proper types return Json("Federation is disabled.".to_owned()); } @@ -390,7 +390,7 @@ pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - if !db.globals.federation_enabled() { + if db.globals.federation_disabled() { return Err(Error::bad_config("Federation is disabled.")); } @@ -437,7 +437,7 @@ pub async fn get_public_rooms_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - if !db.globals.federation_enabled() { + if db.globals.federation_disabled() { return Err(Error::bad_config("Federation is disabled.")); } @@ -484,7 +484,7 @@ pub async fn send_transaction_message_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { - if !db.globals.federation_enabled() { + if db.globals.federation_disabled() { return Err(Error::bad_config("Federation is disabled.")); } @@ -587,7 +587,7 @@ pub fn get_missing_events_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { - if !db.globals.federation_enabled() { + if db.globals.federation_disabled() { return Err(Error::bad_config("Federation is disabled.")); } @@ -632,7 +632,7 @@ pub fn get_profile_information_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { - if !db.globals.federation_enabled() { + if db.globals.federation_disabled() { return Err(Error::bad_config("Federation is disabled.")); } @@ -666,7 +666,7 @@ pub fn get_user_devices_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { - if !db.globals.federation_enabled() { + if db.globals.federation_disabled() { return Err(Error::bad_config("Federation is disabled.")); } From 3bdaf6e79e5ba0f893055c4744d0c107fbfbef77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 1 Jan 2021 11:24:09 +0100 Subject: [PATCH 28/36] improvement: better default config --- .gitignore | 1 + Rocket-example.toml | 31 ------------------------------- conduit-example.toml | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 31 deletions(-) delete mode 100644 Rocket-example.toml create mode 100644 conduit-example.toml diff --git a/.gitignore b/.gitignore index ee48b114..efbc1d9e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ **/*.rs.bk Rocket.toml +conduit.toml diff --git a/Rocket-example.toml b/Rocket-example.toml deleted file mode 100644 index 8eb48e95..00000000 --- a/Rocket-example.toml +++ /dev/null @@ -1,31 +0,0 @@ -[global] -# The name of this server -# Note: If server name != hostname, you need a .well-known file for federation -# to work -server_name = "your.server.name" - -port = 14004 - -# Max size for uploads -#max_request_size = 20_000_000 # in bytes, ~20 MB - -# Disable registration. No new users will be able to register on this server -#registration_disabled = true - -# Disable encryption, so no new encrypted rooms can be created -# Note: existing rooms will continue to work -#encryption_disabled = true - -#federation_enabled = true - -# Default path is in this user's data -#database_path = "/home/timo/MyConduitServer" - -# You should probably leave this at 0.0.0.0 -address = "0.0.0.0" - -# TLS support -# Note: Not necessary when using a reverse proxy: -#[global.tls] -#certs = "/etc/letsencrypt/live/your.server.name/fullchain.pem" -#key = "/etc/letsencrypt/live/your.server.name/privkey.pem" diff --git a/conduit-example.toml b/conduit-example.toml new file mode 100644 index 00000000..70d3ce47 --- /dev/null +++ b/conduit-example.toml @@ -0,0 +1,37 @@ +[global] +# The server_name is the name of this server. It is used as a suffix for user +# and room ids. Examples: matrix.org, conduit.rs +# The Conduit server needs to be reachable at https://your.server.name/ on port +# 443 (client-server) and 8448 (federation) OR you can create /.well-known +# files to redirect requests. See +# https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client +# and https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server +# for more information + +# YOU NEED TO EDIT THIS +#server_name = "your.server.name" + +# This is the only directly where Conduit will save its data +database_path = "/var/lib/conduit/conduit.db" + +# The port Conduit will be running on. You need to set up a reverse proxy in +# your web server (e.g. apache or nginx), so all requests to /_matrix on port +# 443 and 8448 will be forwarded to the Conduit instance running on this port +port = 6167 + +# Max size for uploads +max_request_size = 20_000_000 # in bytes + +# Disable registration. No new users will be able to register on this server +#registration_disabled = false + +# Disable encryption, so no new encrypted rooms can be created +# Note: existing rooms will continue to work +#encryption_disabled = false +#federation_disabled = false + +#cache_capacity = 1073741824 # in bytes, 1024 * 1024 * 1024 +#max_concurrent_requests = 4 # How many requests Conduit sends to other servers at the same time +#workers = 4 # default: cpu core count * 2 + +address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy From d7e56dbfa0b9695458864983f07fe2aa1c36f11f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 20 Oct 2020 14:18:20 +0200 Subject: [PATCH 29/36] docs: recommend using binaries --- DEPLOY_FROM_SOURCE.md => DEPLOY.md | 19 ++++++++++++++----- README.md | 9 ++++----- 2 files changed, 18 insertions(+), 10 deletions(-) rename DEPLOY_FROM_SOURCE.md => DEPLOY.md (72%) diff --git a/DEPLOY_FROM_SOURCE.md b/DEPLOY.md similarity index 72% rename from DEPLOY_FROM_SOURCE.md rename to DEPLOY.md index 456fe6ea..443fac83 100644 --- a/DEPLOY_FROM_SOURCE.md +++ b/DEPLOY.md @@ -7,15 +7,24 @@ Make sure you have `libssl-dev` and `pkg-config` installed and the [rust toolcha ## Install Conduit +You have to download the binary that fits your machine. Run `uname -m` to see what you need: +- x84_64: `https://conduit.rs/master/x86_64/conduit-bin` +- armv7: `https://conduit.rs/master/armv7/conduit-bin` +- armv8: `https://conduit.rs/master/armv8/conduit-bin` +- arm: `https://conduit.rs/master/arm/conduit-bin` + ```bash $ sudo useradd -m conduit -$ sudo -u conduit cargo install --git "https://git.koesters.xyz/timo/conduit.git" +$ sudo -u conduit wget -O /home/conduit/conduit-bin && chmod +x /home/conduit/conduit-bin ``` ## Setup systemd service -In this guide, we set up a systemd service for Conduit, so it's easy to start, stop Conduit and set it to autostart when your server reboots. Paste the default systemd service below and configure it to fit your setup (in /etc/systemd/system/conduit.service). +In this guide, we set up a systemd service for Conduit, so it's easy to +start/stop Conduit and set it to autostart when your server reboots. Paste the +default systemd service you can find below into +`/etc/systemd/system/conduit.service` and configure it to fit your setup. ```systemd [Unit] @@ -38,7 +47,7 @@ User=conduit Group=conduit Type=simple Restart=always -ExecStart=/home/conduit/.cargo/bin/conduit +ExecStart=/home/conduit/conduit-bin [Install] WantedBy=multi-user.target @@ -92,12 +101,12 @@ $ sudo certbot -d conduit.koesters.xyz ## You're done! -Now you can start Conduit with +Now you can start Conduit with: ```bash $ sudo systemctl start conduit ``` -and set it to start automatically when your system boots with +Set it to start automatically when your system boots with: ```bash $ sudo systemctl enable conduit ``` diff --git a/README.md b/README.md index c8397758..8e4b1b70 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,12 @@ example) and register on the `https://conduit.koesters.xyz` homeserver. #### How can I deploy my own? -##### From source +##### Deploy -Clone the repo, build it with `cargo build --release` and call the binary -(target/release/conduit) from somewhere like a systemd script. [Read -more](DEPLOY_FROM_SOURCE.md) +Download or compile a conduit binary and call it from somewhere like a systemd script. [Read +more](DEPLOY.md) -##### Using Docker +##### Deploy using Docker Pull and run the docker image with From b4818716b880c147b34fdef078f0f0a9610d858d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 1 Jan 2021 13:47:53 +0100 Subject: [PATCH 30/36] improvement: better deploy guide --- DEPLOY.md | 142 ++++++++++++++++++++++++----------- src/client_server/account.rs | 2 +- src/client_server/room.rs | 2 +- src/database.rs | 18 +++-- src/database/globals.rs | 12 +-- src/database/rooms.rs | 4 +- src/server_server.rs | 18 ++--- 7 files changed, 129 insertions(+), 69 deletions(-) diff --git a/DEPLOY.md b/DEPLOY.md index 443fac83..f101539e 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -1,53 +1,42 @@ -# Deploy from source +# Deploying Conduit -## Prerequisites +## Getting help -Make sure you have `libssl-dev` and `pkg-config` installed and the [rust toolchain](https://rustup.rs) is available on at least on user. +If you run into any problems while setting up Conduit, write an email to `support@conduit.rs`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new). +## Installing Conduit -## Install Conduit - -You have to download the binary that fits your machine. Run `uname -m` to see what you need: +You have to download the binary that fits your machine. Run `uname -m` to see +what you need. Now copy the right url: - x84_64: `https://conduit.rs/master/x86_64/conduit-bin` - armv7: `https://conduit.rs/master/armv7/conduit-bin` - armv8: `https://conduit.rs/master/armv8/conduit-bin` - arm: `https://conduit.rs/master/arm/conduit-bin` ```bash -$ sudo useradd -m conduit -$ sudo -u conduit wget -O /home/conduit/conduit-bin && chmod +x /home/conduit/conduit-bin +$ sudo wget -O /usr/local/bin/conduit +$ sudo chmod +x /usr/local/bin/conduit ``` -## Setup systemd service +## Setting up a systemd service -In this guide, we set up a systemd service for Conduit, so it's easy to -start/stop Conduit and set it to autostart when your server reboots. Paste the +Now we'll set up a systemd service for Conduit, so it's easy to start/stop +Conduit and set it to autostart when your server reboots. Simply paste the default systemd service you can find below into -`/etc/systemd/system/conduit.service` and configure it to fit your setup. +`/etc/systemd/system/conduit.service`. ```systemd [Unit] -Description=Conduit +Description=Conduit Matrix Server After=network.target [Service] -Environment="ROCKET_SERVER_NAME=YOURSERVERNAME.HERE" # EDIT THIS - -Environment="ROCKET_PORT=14004" # Reverse proxy port - -#Environment="ROCKET_MAX_REQUEST_SIZE=20000000" # in bytes -#Environment="ROCKET_REGISTRATION_DISABLED=true" -#Environment="ROCKET_ENCRYPTION_DISABLED=true" -#Environment="ROCKET_FEDERATION_ENABLED=true" -#Environment="ROCKET_LOG=normal" # Detailed logging - -Environment="ROCKET_ENV=production" -User=conduit -Group=conduit -Type=simple +Environment="CONDUIT_CONFIG=/etc/matrix-conduit/conduit.toml" +User=root +Group=root Restart=always -ExecStart=/home/conduit/conduit-bin +ExecStart=/usr/local/bin/matrix-conduit [Install] WantedBy=multi-user.target @@ -59,43 +48,106 @@ $ sudo systemctl daemon-reload ``` -## Setup Reverse Proxy +## Creating the Conduit configuration file + +Now we need to create the Conduit's config file in `/etc/matrix-conduit/conduit.toml`. Paste this in **and take a moment to read it. You need to change at least the server name.** +```toml +[global] +# The server_name is the name of this server. It is used as a suffix for user +# and room ids. Examples: matrix.org, conduit.rs +# The Conduit server needs to be reachable at https://your.server.name/ on port +# 443 (client-server) and 8448 (federation) OR you can create /.well-known +# files to redirect requests. See +# https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client +# and https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server +# for more information + +# YOU NEED TO EDIT THIS +#server_name = "your.server.name" + +# This is the only directory where Conduit will save its data +database_path = "/var/lib/matrix-conduit/conduit_db" -This depends on whether you use Apache, Nginx or something else. For Apache it looks like this (in /etc/apache2/sites-enabled/050-conduit.conf): +# The port Conduit will be running on. You need to set up a reverse proxy in +# your web server (e.g. apache or nginx), so all requests to /_matrix on port +# 443 and 8448 will be forwarded to the Conduit instance running on this port +port = 6167 + +# Max size for uploads +max_request_size = 20_000_000 # in bytes + +# Disabling registration means no new users will be able to register on this server +allow_registration = false + +# Disable encryption, so no new encrypted rooms can be created +# Note: existing rooms will continue to work +allow_encryption = true +allow_federation = true + +#cache_capacity = 1073741824 # in bytes, 1024 * 1024 * 1024 +#max_concurrent_requests = 4 # How many requests Conduit sends to other servers at the same time +#workers = 4 # default: cpu core count * 2 + +address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy ``` - -ServerName conduit.koesters.xyz # EDIT THIS -AllowEncodedSlashes NoDecode +## Setting up the Reverse Proxy -ServerAlias conduit.koesters.xyz # EDIT THIS +This depends on whether you use Apache, Nginx or another web server. + +### Apache + +Create `/etc/apache2/sites-enabled/050-conduit.conf` and copy-and-paste this: +``` +Listen 8448 + + + +ServerName your.server.name # EDIT THIS -ProxyPreserveHost On -ProxyRequests off AllowEncodedSlashes NoDecode -ProxyPass / http://localhost:14004/ nocanon -ProxyPassReverse / http://localhost:14004/ nocanon +ProxyPass /_matrix/ http://localhost:6167/ +ProxyPassReverse /_matrix/ http://localhost:6167/ Include /etc/letsencrypt/options-ssl-apache.conf - -# EDIT THESE: -SSLCertificateFile /etc/letsencrypt/live/conduit.koesters.xyz/fullchain.pem -SSLCertificateKeyFile /etc/letsencrypt/live/conduit.koesters.xyz/privkey.pem +SSLCertificateFile /etc/letsencrypt/live/your.server.name/fullchain.pem # EDIT THIS +SSLCertificateKeyFile /etc/letsencrypt/live/your.server.name/privkey.pem # EDIT THIS ``` -Then run +**You need to make some edits again.** When you are done, run ```bash $ sudo systemctl reload apache2 ``` +### Nginx + +If you use Nginx and not Apache, add the following server section inside the +http section of `/etc/nginx/nginx.conf` +``` +server { + listen 443; + listen 8448; + server_name your.server.name; # EDIT THIS + + location /_matrix/ { + proxy_pass http://localhost:6167/_matrix/; + } +} +``` +**You need to make some edits again.** When you are done, run +```bash +$ sudo systemctl reload nginx +``` + + ## SSL Certificate -The easiest way to get an SSL certificate for the domain is to install `certbot` and run this: +The easiest way to get an SSL certificate, if you don't have one already, is to install `certbot` and run this: ```bash -$ sudo certbot -d conduit.koesters.xyz +$ sudo certbot -d your.server.name ``` diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 8fb926e2..12c7f7e2 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -86,7 +86,7 @@ pub async fn register_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - if db.globals.registration_disabled() { + if !db.globals.allow_registration() { return Err(Error::BadRequest( ErrorKind::Forbidden, "Registration has been disabled.", diff --git a/src/client_server/room.rs b/src/client_server/room.rs index e473e6e1..092e0830 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -240,7 +240,7 @@ pub async fn create_room_route( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; // Silently skip encryption events if they are not allowed - if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() { + if pdu_builder.event_type == EventType::RoomEncryption && !db.globals.allow_encryption() { continue; } diff --git a/src/database.rs b/src/database.rs index 99ea9f5e..607e1be2 100644 --- a/src/database.rs +++ b/src/database.rs @@ -33,11 +33,19 @@ pub struct Config { #[serde(default = "default_max_concurrent_requests")] max_concurrent_requests: u16, #[serde(default)] - registration_disabled: bool, - #[serde(default)] - encryption_disabled: bool, - #[serde(default)] - federation_disabled: bool, + allow_registration: bool, + #[serde(default = "true_fn")] + allow_encryption: bool, + #[serde(default = "false_fn")] + allow_federation: bool, +} + +fn false_fn() -> bool { + false +} + +fn true_fn() -> bool { + true } fn default_cache_capacity() -> u64 { diff --git a/src/database/globals.rs b/src/database/globals.rs index 5444d6ec..3e24d82f 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -111,16 +111,16 @@ impl Globals { self.config.max_request_size } - pub fn registration_disabled(&self) -> bool { - self.config.registration_disabled + pub fn allow_registration(&self) -> bool { + self.config.allow_registration } - pub fn encryption_disabled(&self) -> bool { - self.config.encryption_disabled + pub fn allow_encryption(&self) -> bool { + self.config.allow_encryption } - pub fn federation_disabled(&self) -> bool { - self.config.federation_disabled + pub fn allow_federation(&self) -> bool { + self.config.allow_federation } pub fn dns_resolver(&self) -> &TokioAsyncResolver { diff --git a/src/database/rooms.rs b/src/database/rooms.rs index ab3dd3f3..40819440 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -786,8 +786,8 @@ impl Rooms { #[allow(clippy::blocks_in_if_conditions)] if !match event_type { EventType::RoomEncryption => { - // Don't allow encryption events when it's disabled - !globals.encryption_disabled() + // Only allow encryption events if it's allowed in the config + globals.allow_encryption() } EventType::RoomMember => { let prev_event = self diff --git a/src/server_server.rs b/src/server_server.rs index 88b6a01c..7ff9e3f0 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -36,7 +36,7 @@ pub async fn send_request( where T: Debug, { - if globals.federation_disabled() { + if !globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); } @@ -322,7 +322,7 @@ pub async fn request_well_known( pub fn get_server_version_route( db: State<'_, Database>, ) -> ConduitResult { - if db.globals.federation_disabled() { + if !db.globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); } @@ -337,7 +337,7 @@ pub fn get_server_version_route( #[cfg_attr(feature = "conduit_bin", get("/_matrix/key/v2/server"))] pub fn get_server_keys_route(db: State<'_, Database>) -> Json { - if db.globals.federation_disabled() { + if !db.globals.allow_federation() { // TODO: Use proper types return Json("Federation is disabled.".to_owned()); } @@ -390,7 +390,7 @@ pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - if db.globals.federation_disabled() { + if !db.globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); } @@ -437,7 +437,7 @@ pub async fn get_public_rooms_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - if db.globals.federation_disabled() { + if !db.globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); } @@ -484,7 +484,7 @@ pub async fn send_transaction_message_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { - if db.globals.federation_disabled() { + if !db.globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); } @@ -587,7 +587,7 @@ pub fn get_missing_events_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { - if db.globals.federation_disabled() { + if !db.globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); } @@ -632,7 +632,7 @@ pub fn get_profile_information_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { - if db.globals.federation_disabled() { + if !db.globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); } @@ -666,7 +666,7 @@ pub fn get_user_devices_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { - if db.globals.federation_disabled() { + if !db.globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); } From d45d033bf1a676cbb3ef7900a415003fdde2a66a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 10 Jan 2021 17:12:16 +0100 Subject: [PATCH 31/36] fix: send presence updates when going offline remaining bug: conduit sends presence updates every 5 minutes even if the user is already offline --- src/database/rooms/edus.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index bf0cdfc9..29edc2a6 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -386,8 +386,6 @@ impl RoomEdus { .take_while(|(_, timestamp)| current_timestamp - timestamp > 5 * 60_000) // 5 Minutes { - self.userid_lastpresenceupdate.remove(&user_id_bytes)?; - // Send new presence events to set the user offline let count = globals.next_count()?.to_be_bytes(); let user_id = utils::string_from_bytes(&user_id_bytes) @@ -421,6 +419,11 @@ impl RoomEdus { .expect("PresenceEvent can be serialized"), )?; } + + self.userid_lastpresenceupdate.insert( + &user_id.to_string().as_bytes(), + &utils::millis_since_unix_epoch().to_be_bytes(), + )?; } Ok(()) From ad7b3f184b340f5b23bc2612192e1550fcc43169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 11 Jan 2021 17:26:47 +0100 Subject: [PATCH 32/36] improvement: send 200 response for turn server info We didn't implement it, but this will stop clients from retrying the endpoint every minute --- src/client_server/voip.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/client_server/voip.rs b/src/client_server/voip.rs index a8db62af..9216f1a6 100644 --- a/src/client_server/voip.rs +++ b/src/client_server/voip.rs @@ -1,13 +1,17 @@ -use crate::{ConduitResult, Error}; -use ruma::api::client::{error::ErrorKind, r0::message::send_message_event}; +use crate::ConduitResult; +use ruma::api::client::r0::voip::get_turn_server_info; +use std::time::Duration; #[cfg(feature = "conduit_bin")] use rocket::get; #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/voip/turnServer"))] -pub async fn turn_server_route() -> ConduitResult { - Err(Error::BadRequest( - ErrorKind::NotFound, - "There is no turn server yet.", - )) +pub async fn turn_server_route() -> ConduitResult { + Ok(get_turn_server_info::Response { + username: "".to_owned(), + password: "".to_owned(), + uris: Vec::new(), + ttl: Duration::from_secs(60 * 60 * 24), + } + .into()) } From c263e130008b815ba7a752b3ea58841e2927e311 Mon Sep 17 00:00:00 2001 From: Paul van Tilburg Date: Mon, 11 Jan 2021 20:28:47 +0100 Subject: [PATCH 33/36] fix: update binary file name in docs for consistency --- DEPLOY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DEPLOY.md b/DEPLOY.md index f101539e..b547b644 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -14,8 +14,8 @@ what you need. Now copy the right url: - arm: `https://conduit.rs/master/arm/conduit-bin` ```bash -$ sudo wget -O /usr/local/bin/conduit -$ sudo chmod +x /usr/local/bin/conduit +$ sudo wget -O /usr/local/bin/matrix-conduit +$ sudo chmod +x /usr/local/bin/matrix-conduit ``` From 265fab843a42d6eaef7a777104a72d101a2e91f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 24 Jan 2021 16:05:52 +0100 Subject: [PATCH 34/36] feature: push rule settings --- src/client_server/push.rs | 624 +++++++++++++++++++++++++++++++++++++- src/main.rs | 5 + 2 files changed, 616 insertions(+), 13 deletions(-) diff --git a/src/client_server/push.rs b/src/client_server/push.rs index 05ba8d06..667d6677 100644 --- a/src/client_server/push.rs +++ b/src/client_server/push.rs @@ -1,16 +1,22 @@ use super::State; use crate::{ConduitResult, Database, Error, Ruma}; -use log::warn; use ruma::{ api::client::{ error::ErrorKind, - r0::push::{get_pushers, get_pushrules_all, set_pushrule, set_pushrule_enabled}, + r0::push::{ + delete_pushrule, get_pushers, get_pushrule, get_pushrule_actions, get_pushrule_enabled, + get_pushrules_all, set_pushrule, set_pushrule_actions, set_pushrule_enabled, RuleKind, + }, }, events::EventType, + push::{ + ConditionalPushRuleInit, ContentPushRule, OverridePushRule, PatternedPushRuleInit, + RoomPushRule, SenderPushRule, SimplePushRuleInit, UnderridePushRule, + }, }; #[cfg(feature = "conduit_bin")] -use rocket::{get, post, put}; +use rocket::{delete, get, post, put}; #[cfg_attr( feature = "conduit_bin", @@ -36,16 +42,201 @@ pub async fn get_pushrules_all_route( .into()) } -#[cfg_attr(feature = "conduit_bin", put( - "/_matrix/client/r0/pushrules/<_>/<_>/<_>", - //data = "" -))] +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "") +)] +pub async fn get_pushrule_route( + db: State<'_, Database>, + body: Ruma>, +) -> ConduitResult { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + let event = db + .account_data + .get::(None, &sender_user, EventType::PushRules)? + .ok_or(Error::BadRequest( + ErrorKind::NotFound, + "PushRules event not found.", + ))?; + + let global = event.content.global; + let rule = match body.kind { + RuleKind::Override => global + .override_ + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map(|rule| rule.0.clone().into()), + RuleKind::Underride => global + .underride + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map(|rule| rule.0.clone().into()), + RuleKind::Sender => global + .sender + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map(|rule| rule.0.clone().into()), + RuleKind::Room => global + .room + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map(|rule| rule.0.clone().into()), + RuleKind::Content => global + .content + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map(|rule| rule.0.clone().into()), + RuleKind::_Custom(_) => None, + }; + + if let Some(rule) = rule { + Ok(get_pushrule::Response { rule }.into()) + } else { + Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.").into()) + } +} + +#[cfg_attr( + feature = "conduit_bin", + put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "") +)] pub async fn set_pushrule_route( db: State<'_, Database>, - //body: Ruma, + body: Ruma>, ) -> ConduitResult { - // TODO - warn!("TODO: set_pushrule_route"); + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if body.scope != "global" { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Scopes other than 'global' are not supported.", + )); + } + + let mut event = db + .account_data + .get::(None, &sender_user, EventType::PushRules)? + .ok_or(Error::BadRequest( + ErrorKind::NotFound, + "PushRules event not found.", + ))?; + + let global = &mut event.content.global; + match body.kind { + RuleKind::Override => { + if let Some(rule) = global + .override_ + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.override_.remove(&rule); + } + + global.override_.insert(OverridePushRule( + ConditionalPushRuleInit { + actions: body.actions.clone(), + default: false, + enabled: true, + rule_id: body.rule_id.clone(), + conditions: body.conditions.clone(), + } + .into(), + )); + } + RuleKind::Underride => { + if let Some(rule) = global + .underride + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.underride.remove(&rule); + } + + global.underride.insert(UnderridePushRule( + ConditionalPushRuleInit { + actions: body.actions.clone(), + default: false, + enabled: true, + rule_id: body.rule_id.clone(), + conditions: body.conditions.clone(), + } + .into(), + )); + } + RuleKind::Sender => { + if let Some(rule) = global + .sender + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.sender.remove(&rule); + } + + global.sender.insert(SenderPushRule( + SimplePushRuleInit { + actions: body.actions.clone(), + default: false, + enabled: true, + rule_id: body.rule_id.clone(), + } + .into(), + )); + } + RuleKind::Room => { + if let Some(rule) = global + .room + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.room.remove(&rule); + } + + global.room.insert(RoomPushRule( + SimplePushRuleInit { + actions: body.actions.clone(), + default: false, + enabled: true, + rule_id: body.rule_id.clone(), + } + .into(), + )); + } + RuleKind::Content => { + if let Some(rule) = global + .content + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.content.remove(&rule); + } + + global.content.insert(ContentPushRule( + PatternedPushRuleInit { + actions: body.actions.clone(), + default: false, + enabled: true, + rule_id: body.rule_id.clone(), + pattern: body.pattern.clone().unwrap_or_default(), + } + .into(), + )); + } + RuleKind::_Custom(_) => {} + } + + db.account_data.update( + None, + &sender_user, + EventType::PushRules, + &event, + &db.globals, + )?; db.flush().await?; @@ -54,19 +245,426 @@ pub async fn set_pushrule_route( #[cfg_attr( feature = "conduit_bin", - put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled") + get("/_matrix/client/r0/pushrules/<_>/<_>/<_>/actions", data = "") +)] +pub async fn get_pushrule_actions_route( + db: State<'_, Database>, + body: Ruma>, +) -> ConduitResult { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if body.scope != "global" { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Scopes other than 'global' are not supported.", + )); + } + + let mut event = db + .account_data + .get::(None, &sender_user, EventType::PushRules)? + .ok_or(Error::BadRequest( + ErrorKind::NotFound, + "PushRules event not found.", + ))?; + + let global = &mut event.content.global; + let actions = match body.kind { + RuleKind::Override => global + .override_ + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map(|rule| rule.0.actions.clone()), + RuleKind::Underride => global + .underride + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map(|rule| rule.0.actions.clone()), + RuleKind::Sender => global + .sender + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map(|rule| rule.0.actions.clone()), + RuleKind::Room => global + .room + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map(|rule| rule.0.actions.clone()), + RuleKind::Content => global + .content + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map(|rule| rule.0.actions.clone()), + RuleKind::_Custom(_) => None, + }; + + db.flush().await?; + + Ok(get_pushrule_actions::Response { + actions: actions.unwrap_or_default(), + } + .into()) +} + +#[cfg_attr( + feature = "conduit_bin", + put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/actions", data = "") +)] +pub async fn set_pushrule_actions_route( + db: State<'_, Database>, + body: Ruma>, +) -> ConduitResult { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if body.scope != "global" { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Scopes other than 'global' are not supported.", + )); + } + + let mut event = db + .account_data + .get::(None, &sender_user, EventType::PushRules)? + .ok_or(Error::BadRequest( + ErrorKind::NotFound, + "PushRules event not found.", + ))?; + + let global = &mut event.content.global; + match body.kind { + RuleKind::Override => { + if let Some(mut rule) = global + .override_ + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.override_.remove(&rule); + rule.0.actions = body.actions.clone(); + global.override_.insert(rule); + } + } + RuleKind::Underride => { + if let Some(mut rule) = global + .underride + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.underride.remove(&rule); + rule.0.actions = body.actions.clone(); + global.underride.insert(rule); + } + } + RuleKind::Sender => { + if let Some(mut rule) = global + .sender + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.sender.remove(&rule); + rule.0.actions = body.actions.clone(); + global.sender.insert(rule); + } + } + RuleKind::Room => { + if let Some(mut rule) = global + .room + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.room.remove(&rule); + rule.0.actions = body.actions.clone(); + global.room.insert(rule); + } + } + RuleKind::Content => { + if let Some(mut rule) = global + .content + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.content.remove(&rule); + rule.0.actions = body.actions.clone(); + global.content.insert(rule); + } + } + RuleKind::_Custom(_) => {} + }; + + db.account_data.update( + None, + &sender_user, + EventType::PushRules, + &event, + &db.globals, + )?; + + db.flush().await?; + + Ok(set_pushrule_actions::Response.into()) +} + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled", data = "") +)] +pub async fn get_pushrule_enabled_route( + db: State<'_, Database>, + body: Ruma>, +) -> ConduitResult { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if body.scope != "global" { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Scopes other than 'global' are not supported.", + )); + } + + let mut event = db + .account_data + .get::(None, &sender_user, EventType::PushRules)? + .ok_or(Error::BadRequest( + ErrorKind::NotFound, + "PushRules event not found.", + ))?; + + let global = &mut event.content.global; + let enabled = match body.kind { + RuleKind::Override => global + .override_ + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map_or(false, |rule| rule.0.enabled), + RuleKind::Underride => global + .underride + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map_or(false, |rule| rule.0.enabled), + RuleKind::Sender => global + .sender + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map_or(false, |rule| rule.0.enabled), + RuleKind::Room => global + .room + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map_or(false, |rule| rule.0.enabled), + RuleKind::Content => global + .content + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .map_or(false, |rule| rule.0.enabled), + RuleKind::_Custom(_) => false, + }; + + db.flush().await?; + + Ok(get_pushrule_enabled::Response { enabled }.into()) +} + +#[cfg_attr( + feature = "conduit_bin", + put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled", data = "") )] pub async fn set_pushrule_enabled_route( db: State<'_, Database>, + body: Ruma>, ) -> ConduitResult { - // TODO - warn!("TODO: set_pushrule_enabled_route"); + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if body.scope != "global" { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Scopes other than 'global' are not supported.", + )); + } + + let mut event = db + .account_data + .get::(None, &sender_user, EventType::PushRules)? + .ok_or(Error::BadRequest( + ErrorKind::NotFound, + "PushRules event not found.", + ))?; + + let global = &mut event.content.global; + match body.kind { + RuleKind::Override => { + if let Some(mut rule) = global + .override_ + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.override_.remove(&rule); + rule.0.enabled = body.enabled; + global.override_.insert(rule); + } + } + RuleKind::Underride => { + if let Some(mut rule) = global + .underride + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.underride.remove(&rule); + rule.0.enabled = body.enabled; + global.underride.insert(rule); + } + } + RuleKind::Sender => { + if let Some(mut rule) = global + .sender + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.sender.remove(&rule); + rule.0.enabled = body.enabled; + global.sender.insert(rule); + } + } + RuleKind::Room => { + if let Some(mut rule) = global + .room + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.room.remove(&rule); + rule.0.enabled = body.enabled; + global.room.insert(rule); + } + } + RuleKind::Content => { + if let Some(mut rule) = global + .content + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.content.remove(&rule); + rule.0.enabled = body.enabled; + global.content.insert(rule); + } + } + RuleKind::_Custom(_) => {} + } + + db.account_data.update( + None, + &sender_user, + EventType::PushRules, + &event, + &db.globals, + )?; db.flush().await?; Ok(set_pushrule_enabled::Response.into()) } +#[cfg_attr( + feature = "conduit_bin", + delete("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "") +)] +pub async fn delete_pushrule_route( + db: State<'_, Database>, + body: Ruma>, +) -> ConduitResult { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if body.scope != "global" { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Scopes other than 'global' are not supported.", + )); + } + + let mut event = db + .account_data + .get::(None, &sender_user, EventType::PushRules)? + .ok_or(Error::BadRequest( + ErrorKind::NotFound, + "PushRules event not found.", + ))?; + + let global = &mut event.content.global; + match body.kind { + RuleKind::Override => { + if let Some(rule) = global + .override_ + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.override_.remove(&rule); + } + } + RuleKind::Underride => { + if let Some(rule) = global + .underride + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.underride.remove(&rule); + } + } + RuleKind::Sender => { + if let Some(rule) = global + .sender + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.sender.remove(&rule); + } + } + RuleKind::Room => { + if let Some(rule) = global + .room + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.room.remove(&rule); + } + } + RuleKind::Content => { + if let Some(rule) = global + .content + .iter() + .find(|rule| rule.0.rule_id == body.rule_id) + .cloned() + { + global.content.remove(&rule); + } + } + RuleKind::_Custom(_) => {} + } + + db.account_data.update( + None, + &sender_user, + EventType::PushRules, + &event, + &db.globals, + )?; + + db.flush().await?; + + Ok(delete_pushrule::Response.into()) +} + #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/pushers"))] pub async fn get_pushers_route() -> ConduitResult { Ok(get_pushers::Response { diff --git a/src/main.rs b/src/main.rs index 9c0eab65..93ab5605 100644 --- a/src/main.rs +++ b/src/main.rs @@ -55,7 +55,12 @@ fn setup_rocket() -> rocket::Rocket { client_server::get_capabilities_route, client_server::get_pushrules_all_route, client_server::set_pushrule_route, + client_server::get_pushrule_route, client_server::set_pushrule_enabled_route, + client_server::get_pushrule_enabled_route, + client_server::get_pushrule_actions_route, + client_server::set_pushrule_actions_route, + client_server::delete_pushrule_route, client_server::get_room_event_route, client_server::get_filter_route, client_server::create_filter_route, From 6924dfc8ea56d8e8347b78364480ea2fce5a5905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 6 Feb 2021 15:27:43 +0100 Subject: [PATCH 35/36] improvement: better appservice compatibility and optimizations --- src/appservice_server.rs | 2 +- src/client_server/state.rs | 2 +- src/client_server/sync.rs | 392 ++++++++++++++++++++----------------- src/database.rs | 7 +- src/database/rooms.rs | 4 + src/database/sending.rs | 3 +- src/main.rs | 32 ++- src/ruma_wrapper.rs | 19 +- src/server_server.rs | 4 +- 9 files changed, 264 insertions(+), 201 deletions(-) diff --git a/src/appservice_server.rs b/src/appservice_server.rs index 986909b6..ec504b57 100644 --- a/src/appservice_server.rs +++ b/src/appservice_server.rs @@ -1,6 +1,6 @@ use crate::{utils, Error, Result}; use http::header::{HeaderValue, CONTENT_TYPE}; -use log::warn; +use log::{info, warn}; use ruma::api::OutgoingRequest; use std::{ convert::{TryFrom, TryInto}, diff --git a/src/client_server/state.rs b/src/client_server/state.rs index cecb79d4..faa415de 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -234,7 +234,7 @@ pub async fn get_state_events_for_empty_key_route( .1; Ok(get_state_events_for_empty_key::Response { - content: serde_json::value::to_raw_value(&event) + content: serde_json::value::to_raw_value(&event.content) .map_err(|_| Error::bad_database("Invalid event content in database"))?, } .into()) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 31361164..494c773c 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -95,15 +95,7 @@ pub async fn sync_events_route( // Database queries: - let current_state = db.rooms.room_state_full(&room_id)?; - let current_members = current_state - .iter() - .filter(|(key, _)| key.0 == EventType::RoomMember) - .map(|(key, value)| (&key.1, value)) // Only keep state key - .collect::>(); - let encrypted_room = current_state - .get(&(EventType::RoomEncryption, "".to_owned())) - .is_some(); + let current_state_hash = db.rooms.current_state_hash(&room_id)?; // These type is Option>. The outer Option is None when there is no event between // since and the current room state, meaning there should be no updates. @@ -115,69 +107,85 @@ pub async fn sync_events_route( .as_ref() .map(|pdu| db.rooms.pdu_state_hash(&pdu.as_ref().ok()?.0).ok()?); - let since_state = since_state_hash.as_ref().map(|state_hash| { - state_hash - .as_ref() - .and_then(|state_hash| db.rooms.state_full(&room_id, &state_hash).ok()) - }); - - let since_encryption = since_state.as_ref().map(|state| { - state - .as_ref() - .map(|state| state.get(&(EventType::RoomEncryption, "".to_owned()))) - }); - - // Calculations: - let new_encrypted_room = - encrypted_room && since_encryption.map_or(false, |encryption| encryption.is_none()); - - let send_member_count = since_state.as_ref().map_or(false, |since_state| { - since_state.as_ref().map_or(true, |since_state| { - current_members.len() - != since_state - .iter() - .filter(|(key, _)| key.0 == EventType::RoomMember) - .count() - }) - }); - - let since_sender_member = since_state.as_ref().map(|since_state| { - since_state.as_ref().and_then(|state| { + let ( + heroes, + joined_member_count, + invited_member_count, + joined_since_last_sync, + state_events, + ) = if since_state_hash != None && Some(¤t_state_hash) != since_state_hash.as_ref() { + let current_state = db.rooms.room_state_full(&room_id)?; + let current_members = current_state + .iter() + .filter(|(key, _)| key.0 == EventType::RoomMember) + .map(|(key, value)| (&key.1, value)) // Only keep state key + .collect::>(); + let encrypted_room = current_state + .get(&(EventType::RoomEncryption, "".to_owned())) + .is_some(); + let since_state = since_state_hash.as_ref().map(|state_hash| { + state_hash + .as_ref() + .and_then(|state_hash| db.rooms.state_full(&room_id, &state_hash).ok()) + }); + + let since_encryption = since_state.as_ref().map(|state| { state - .get(&(EventType::RoomMember, sender_user.as_str().to_owned())) - .and_then(|pdu| { - serde_json::from_value::< + .as_ref() + .map(|state| state.get(&(EventType::RoomEncryption, "".to_owned()))) + }); + + // Calculations: + let new_encrypted_room = + encrypted_room && since_encryption.map_or(false, |encryption| encryption.is_none()); + + let send_member_count = since_state.as_ref().map_or(false, |since_state| { + since_state.as_ref().map_or(true, |since_state| { + current_members.len() + != since_state + .iter() + .filter(|(key, _)| key.0 == EventType::RoomMember) + .count() + }) + }); + + let since_sender_member = since_state.as_ref().map(|since_state| { + since_state.as_ref().and_then(|state| { + state + .get(&(EventType::RoomMember, sender_user.as_str().to_owned())) + .and_then(|pdu| { + serde_json::from_value::< Raw, >(pdu.content.clone()) .expect("Raw::from_value always works") .deserialize() .map_err(|_| Error::bad_database("Invalid PDU in database.")) .ok() - }) - }) - }); - - if encrypted_room { - for (user_id, current_member) in current_members { - let current_membership = serde_json::from_value::< - Raw, - >(current_member.content.clone()) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database."))? - .membership; - - let since_membership = - since_state - .as_ref() - .map_or(MembershipState::Join, |since_state| { - since_state - .as_ref() - .and_then(|since_state| { - since_state - .get(&(EventType::RoomMember, user_id.clone())) - .and_then(|since_member| { - serde_json::from_value::< + }) + }) + }); + + if encrypted_room { + for (user_id, current_member) in current_members { + let current_membership = serde_json::from_value::< + Raw, + >(current_member.content.clone()) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| Error::bad_database("Invalid PDU in database."))? + .membership; + + let since_membership = + since_state + .as_ref() + .map_or(MembershipState::Join, |since_state| { + since_state + .as_ref() + .and_then(|since_state| { + since_state + .get(&(EventType::RoomMember, user_id.clone())) + .and_then(|since_member| { + serde_json::from_value::< Raw, >( since_member.content.clone() @@ -188,123 +196,166 @@ pub async fn sync_events_route( Error::bad_database("Invalid PDU in database.") }) .ok() - }) - }) - .map_or(MembershipState::Leave, |member| member.membership) - }); - - let user_id = UserId::try_from(user_id.clone()) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; - - match (since_membership, current_membership) { - (MembershipState::Leave, MembershipState::Join) => { - // A new user joined an encrypted room - if !share_encrypted_room(&db, &sender_user, &user_id, &room_id) { - device_list_updates.insert(user_id); + }) + }) + .map_or(MembershipState::Leave, |member| member.membership) + }); + + let user_id = UserId::try_from(user_id.clone()) + .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; + + match (since_membership, current_membership) { + (MembershipState::Leave, MembershipState::Join) => { + // A new user joined an encrypted room + if !share_encrypted_room(&db, &sender_user, &user_id, &room_id) { + device_list_updates.insert(user_id); + } } + (MembershipState::Join, MembershipState::Leave) => { + // Write down users that have left encrypted rooms we are in + left_encrypted_users.insert(user_id); + } + _ => {} } - (MembershipState::Join, MembershipState::Leave) => { - // Write down users that have left encrypted rooms we are in - left_encrypted_users.insert(user_id); - } - _ => {} } } - } - let joined_since_last_sync = since_sender_member.map_or(false, |member| { - member.map_or(true, |member| member.membership != MembershipState::Join) - }); + let joined_since_last_sync = since_sender_member.map_or(false, |member| { + member.map_or(true, |member| member.membership != MembershipState::Join) + }); - if joined_since_last_sync && encrypted_room || new_encrypted_room { - // If the user is in a new encrypted room, give them all joined users - device_list_updates.extend( - db.rooms - .room_members(&room_id) - .filter_map(|user_id| Some(user_id.ok()?)) - .filter(|user_id| { - // Don't send key updates from the sender to the sender - sender_user != user_id - }) - .filter(|user_id| { - // Only send keys if the sender doesn't share an encrypted room with the target already - !share_encrypted_room(&db, sender_user, user_id, &room_id) - }), - ); - } - - // Look for device list updates in this room - device_list_updates.extend( - db.users - .keys_changed(&room_id.to_string(), since, None) - .filter_map(|r| r.ok()), - ); + if joined_since_last_sync && encrypted_room || new_encrypted_room { + // If the user is in a new encrypted room, give them all joined users + device_list_updates.extend( + db.rooms + .room_members(&room_id) + .filter_map(|user_id| Some(user_id.ok()?)) + .filter(|user_id| { + // Don't send key updates from the sender to the sender + sender_user != user_id + }) + .filter(|user_id| { + // Only send keys if the sender doesn't share an encrypted room with the target already + !share_encrypted_room(&db, sender_user, user_id, &room_id) + }), + ); + } - let (joined_member_count, invited_member_count, heroes) = if send_member_count { - let joined_member_count = db.rooms.room_members(&room_id).count(); - let invited_member_count = db.rooms.room_members_invited(&room_id).count(); + let (joined_member_count, invited_member_count, heroes) = if send_member_count { + let joined_member_count = db.rooms.room_members(&room_id).count(); + let invited_member_count = db.rooms.room_members_invited(&room_id).count(); - // Recalculate heroes (first 5 members) - let mut heroes = Vec::new(); + // Recalculate heroes (first 5 members) + let mut heroes = Vec::new(); - if joined_member_count + invited_member_count <= 5 { - // Go through all PDUs and for each member event, check if the user is still joined or - // invited until we have 5 or we reach the end + if joined_member_count + invited_member_count <= 5 { + // Go through all PDUs and for each member event, check if the user is still joined or + // invited until we have 5 or we reach the end - for hero in db - .rooms - .all_pdus(&sender_user, &room_id)? - .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus - .filter(|(_, pdu)| pdu.kind == EventType::RoomMember) - .map(|(_, pdu)| { - let content = serde_json::from_value::< - Raw, - >(pdu.content.clone()) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid member event in database."))?; - - if let Some(state_key) = &pdu.state_key { - let user_id = UserId::try_from(state_key.clone()).map_err(|_| { - Error::bad_database("Invalid UserId in member PDU.") + for hero in db + .rooms + .all_pdus(&sender_user, &room_id)? + .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus + .filter(|(_, pdu)| pdu.kind == EventType::RoomMember) + .map(|(_, pdu)| { + let content = serde_json::from_value::< + Raw, + >(pdu.content.clone()) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| { + Error::bad_database("Invalid member event in database.") })?; - // The membership was and still is invite or join - if matches!( - content.membership, - MembershipState::Join | MembershipState::Invite - ) && (db.rooms.is_joined(&user_id, &room_id)? - || db.rooms.is_invited(&user_id, &room_id)?) - { - Ok::<_, Error>(Some(state_key.clone())) + if let Some(state_key) = &pdu.state_key { + let user_id = + UserId::try_from(state_key.clone()).map_err(|_| { + Error::bad_database("Invalid UserId in member PDU.") + })?; + + // The membership was and still is invite or join + if matches!( + content.membership, + MembershipState::Join | MembershipState::Invite + ) && (db.rooms.is_joined(&user_id, &room_id)? + || db.rooms.is_invited(&user_id, &room_id)?) + { + Ok::<_, Error>(Some(state_key.clone())) + } else { + Ok(None) + } } else { Ok(None) } - } else { - Ok(None) + }) + .filter_map(|u| u.ok()) // Filter out buggy users + // Filter for possible heroes + .filter_map(|u| u) + { + if heroes.contains(&hero) || hero == sender_user.as_str() { + continue; } - }) - .filter_map(|u| u.ok()) // Filter out buggy users - // Filter for possible heroes - .filter_map(|u| u) - { - if heroes.contains(&hero) || hero == sender_user.as_str() { - continue; + + heroes.push(hero); } + } - heroes.push(hero); + ( + Some(joined_member_count), + Some(invited_member_count), + heroes, + ) + } else { + (None, None, Vec::new()) + }; + + let state_events = if joined_since_last_sync { + db.rooms + .room_state_full(&room_id)? + .into_iter() + .map(|(_, pdu)| pdu.to_sync_state_event()) + .collect() + } else { + match since_state { + None => Vec::new(), + Some(Some(since_state)) => current_state + .iter() + .filter(|(key, value)| { + since_state.get(key).map(|e| &e.event_id) != Some(&value.event_id) + }) + .filter(|(_, value)| { + !timeline_pdus.iter().any(|(_, timeline_pdu)| { + timeline_pdu.kind == value.kind + && timeline_pdu.state_key == value.state_key + }) + }) + .map(|(_, pdu)| pdu.to_sync_state_event()) + .collect(), + Some(None) => current_state + .iter() + .map(|(_, pdu)| pdu.to_sync_state_event()) + .collect(), } - } + }; ( - Some(joined_member_count), - Some(invited_member_count), heroes, + joined_member_count, + invited_member_count, + joined_since_last_sync, + state_events, ) } else { - (None, None, Vec::new()) + (Vec::new(), None, None, false, Vec::new()) }; + // Look for device list updates in this room + device_list_updates.extend( + db.users + .keys_changed(&room_id.to_string(), since, None) + .filter_map(|r| r.ok()), + ); + let notification_count = if send_notification_counts { if let Some(last_read) = db.rooms.edus.private_read_get(&room_id, &sender_user)? { Some( @@ -385,34 +436,7 @@ pub async fn sync_events_route( events: room_events, }, state: sync_events::State { - events: if joined_since_last_sync { - db.rooms - .room_state_full(&room_id)? - .into_iter() - .map(|(_, pdu)| pdu.to_sync_state_event()) - .collect() - } else { - match since_state { - None => Vec::new(), - Some(Some(since_state)) => current_state - .iter() - .filter(|(key, value)| { - since_state.get(key).map(|e| &e.event_id) != Some(&value.event_id) - }) - .filter(|(_, value)| { - !timeline_pdus.iter().any(|(_, timeline_pdu)| { - timeline_pdu.kind == value.kind - && timeline_pdu.state_key == value.state_key - }) - }) - .map(|(_, pdu)| pdu.to_sync_state_event()) - .collect(), - Some(None) => current_state - .iter() - .map(|(_, pdu)| pdu.to_sync_state_event()) - .collect(), - } - }, + events: state_events, }, ephemeral: sync_events::Ephemeral { events: edus }, }; diff --git a/src/database.rs b/src/database.rs index 607e1be2..afcd58f7 100644 --- a/src/database.rs +++ b/src/database.rs @@ -27,7 +27,7 @@ pub struct Config { server_name: Box, database_path: String, #[serde(default = "default_cache_capacity")] - cache_capacity: u64, + cache_capacity: u32, #[serde(default = "default_max_request_size")] max_request_size: u32, #[serde(default = "default_max_concurrent_requests")] @@ -48,7 +48,7 @@ fn true_fn() -> bool { true } -fn default_cache_capacity() -> u64 { +fn default_cache_capacity() -> u32 { 1024 * 1024 * 1024 } @@ -93,8 +93,7 @@ impl Database { pub async fn load_or_create(config: Config) -> Result { let db = sled::Config::default() .path(&config.database_path) - .cache_capacity(config.cache_capacity) - .print_profile_on_drop(false) + .cache_capacity(config.cache_capacity as u64) .open()?; info!("Opened sled database at {}", config.database_path); diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 40819440..b35d0065 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1010,6 +1010,10 @@ impl Rooms { .filter_map(|r| r.ok()) .any(|room_alias| aliases.is_match(room_alias.as_str())) }) || rooms.map_or(false, |rooms| rooms.contains(&room_id.as_str().into())) + || self + .room_members(&room_id) + .filter_map(|r| r.ok()) + .any(|member| users.iter().any(|regex| regex.is_match(member.as_str()))) { sending.send_pdu_appservice(&appservice.0, &pdu_id)?; } diff --git a/src/database/sending.rs b/src/database/sending.rs index 74aad326..0a66f73d 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -8,7 +8,7 @@ use std::{ use crate::{appservice_server, server_server, utils, Error, PduEvent, Result}; use federation::transactions::send_transaction_message; -use log::info; +use log::{error, info}; use rocket::futures::stream::{FuturesUnordered, StreamExt}; use ruma::{ api::{appservice, federation, OutgoingRequest}, @@ -131,6 +131,7 @@ impl Sending { }; prefix.extend_from_slice(server.as_bytes()); prefix.push(0xff); + last_failed_try.insert(server.clone(), match last_failed_try.get(&server) { Some(last_failed) => { (last_failed.0+1, Instant::now()) diff --git a/src/main.rs b/src/main.rs index 93ab5605..65434a5d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ pub use database::Database; pub use error::{ConduitLogger, Error, Result}; pub use pdu::PduEvent; pub use rocket::State; +use ruma::api::client::error::ErrorKind; pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse}; use log::LevelFilter; @@ -154,7 +155,13 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_profile_information_route, ], ) - .register(catchers![not_found_catcher]) + .register(catchers![ + not_found_catcher, + forbidden_catcher, + unknown_token_catcher, + missing_token_catcher, + bad_json_catcher + ]) .attach(AdHoc::on_attach("Config", |rocket| async { let config = rocket .figment() @@ -186,3 +193,26 @@ async fn main() { fn not_found_catcher(_req: &'_ Request<'_>) -> String { "404 Not Found".to_owned() } + +#[catch(580)] +fn forbidden_catcher() -> Result<()> { + Err(Error::BadRequest(ErrorKind::Forbidden, "Forbidden.")) +} + +#[catch(581)] +fn unknown_token_catcher() -> Result<()> { + Err(Error::BadRequest( + ErrorKind::UnknownToken { soft_logout: false }, + "Unknown token.", + )) +} + +#[catch(582)] +fn missing_token_catcher() -> Result<()> { + Err(Error::BadRequest(ErrorKind::MissingToken, "Missing token.")) +} + +#[catch(583)] +fn bad_json_catcher() -> Result<()> { + Err(Error::BadRequest(ErrorKind::BadJson, "Bad json.")) +} diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 0fdca743..45fcc7fb 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -45,7 +45,7 @@ where http::request::Request>, >>::Error: std::fmt::Debug, { - type Error = (); // TODO: Better error handling + type Error = (); type Owned = Data; type Borrowed = Self::Owned; @@ -82,7 +82,9 @@ where registration .get("as_token") .and_then(|as_token| as_token.as_str()) - .map_or(false, |as_token| token.as_deref() == Some(as_token)) + .map_or(false, |as_token| { + dbg!(token.as_deref()) == dbg!(Some(as_token)) + }) }) { match T::METADATA.authentication { AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => { @@ -105,7 +107,8 @@ where ); if !db.users.exists(&user_id).unwrap() { - return Failure((Status::Unauthorized, ())); + // Forbidden + return Failure((Status::raw(580), ())); } // TODO: Check if appservice is allowed to be that user @@ -119,15 +122,15 @@ where AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => { if let Some(token) = token { match db.users.find_from_token(&token).unwrap() { - // TODO: M_UNKNOWN_TOKEN - None => return Failure((Status::Unauthorized, ())), + // Unknown Token + None => return Failure((Status::raw(581), ())), Some((user_id, device_id)) => { (Some(user_id), Some(device_id.into()), false) } } } else { - // TODO: M_MISSING_TOKEN - return Failure((Status::Unauthorized, ())); + // Missing Token + return Failure((Status::raw(582), ())); } } AuthScheme::ServerSignatures => (None, None, false), @@ -163,7 +166,7 @@ where }), Err(e) => { warn!("{:?}", e); - Failure((Status::BadRequest, ())) + Failure((Status::raw(583), ())) } } }) diff --git a/src/server_server.rs b/src/server_server.rs index 7ff9e3f0..3fea4da4 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -557,7 +557,7 @@ pub async fn send_transaction_message_route<'a>( pdu_id.push(0xff); pdu_id.extend_from_slice(&count.to_be_bytes()); - db.rooms.append_to_state(&pdu_id, &pdu, &db.globals)?; + let next_room_state = db.rooms.append_to_state(&pdu_id, &pdu, &db.globals)?; db.rooms.append_pdu( &pdu, @@ -569,6 +569,8 @@ pub async fn send_transaction_message_route<'a>( &db.admin, )?; + db.rooms.set_room_state(&room_id, &next_room_state)?; + for appservice in db.appservice.iter_all().filter_map(|r| r.ok()) { db.sending.send_pdu_appservice(&appservice.0, &pdu_id)?; } From ea1e4625d1987183f0bb669cb20bbabd08c71d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 7 Feb 2021 13:20:00 +0100 Subject: [PATCH 36/36] fix: default config options --- conduit-example.toml | 6 +++--- src/database.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conduit-example.toml b/conduit-example.toml index 70d3ce47..b82da2c1 100644 --- a/conduit-example.toml +++ b/conduit-example.toml @@ -23,12 +23,12 @@ port = 6167 max_request_size = 20_000_000 # in bytes # Disable registration. No new users will be able to register on this server -#registration_disabled = false +#allow_registration = true # Disable encryption, so no new encrypted rooms can be created # Note: existing rooms will continue to work -#encryption_disabled = false -#federation_disabled = false +#allow_encryption = true +#allow_federation = false #cache_capacity = 1073741824 # in bytes, 1024 * 1024 * 1024 #max_concurrent_requests = 4 # How many requests Conduit sends to other servers at the same time diff --git a/src/database.rs b/src/database.rs index afcd58f7..9fce2938 100644 --- a/src/database.rs +++ b/src/database.rs @@ -32,7 +32,7 @@ pub struct Config { max_request_size: u32, #[serde(default = "default_max_concurrent_requests")] max_concurrent_requests: u16, - #[serde(default)] + #[serde(default = "true_fn")] allow_registration: bool, #[serde(default = "true_fn")] allow_encryption: bool,