diff --git a/src/client_server/user_directory.rs b/src/client_server/user_directory.rs
index 7c0bcc16..349c1399 100644
--- a/src/client_server/user_directory.rs
+++ b/src/client_server/user_directory.rs
@@ -1,15 +1,23 @@
 use crate::{database::DatabaseGuard, Result, Ruma};
-use ruma::api::client::user_directory::search_users;
+use ruma::{
+    api::client::user_directory::search_users,
+    events::{
+        room::join_rules::{JoinRule, RoomJoinRulesEventContent},
+        StateEventType,
+    },
+};
 
 /// # `POST /_matrix/client/r0/user_directory/search`
 ///
 /// Searches all known users for a match.
 ///
-/// - TODO: Hide users that are not in any public rooms?
+/// - Hides any local users that aren't in any public rooms (i.e. those that have the join rule set to public)
+/// and don't share a room with the sender
 pub async fn search_users_route(
     db: DatabaseGuard,
     body: Ruma<search_users::v3::IncomingRequest>,
 ) -> Result<search_users::v3::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
     let limit = u64::from(body.limit) as usize;
 
     let mut users = db.users.iter().filter_map(|user_id| {
@@ -41,7 +49,39 @@ pub async fn search_users_route(
             return None;
         }
 
-        Some(user)
+        let user_is_in_public_rooms =
+            db.rooms
+                .rooms_joined(&user_id)
+                .filter_map(|r| r.ok())
+                .any(|room| {
+                    db.rooms
+                        .room_state_get(&room, &StateEventType::RoomJoinRules, "")
+                        .map_or(false, |event| {
+                            event.map_or(false, |event| {
+                                serde_json::from_str(event.content.get())
+                                    .map_or(false, |r: RoomJoinRulesEventContent| {
+                                        r.join_rule == JoinRule::Public
+                                    })
+                            })
+                        })
+                });
+
+        if user_is_in_public_rooms {
+            return Some(user);
+        }
+
+        let user_is_in_shared_rooms = db
+            .rooms
+            .get_shared_rooms(vec![sender_user.clone(), user_id.clone()])
+            .ok()?
+            .next()
+            .is_some();
+
+        if user_is_in_shared_rooms {
+            return Some(user);
+        }
+
+        None
     });
 
     let results = users.by_ref().take(limit).collect();
diff --git a/tests/sytest/sytest-whitelist b/tests/sytest/sytest-whitelist
index 5afc3fd9..1c969dba 100644
--- a/tests/sytest/sytest-whitelist
+++ b/tests/sytest/sytest-whitelist
@@ -445,6 +445,9 @@ Typing notifications don't leak
 Uninvited users cannot join the room
 Unprivileged users can set m.room.topic if it only needs level 0
 User appears in user directory
+User in private room doesn't appear in user directory
+User joining then leaving public room appears and dissappears from directory 
+User in shared private room does appear in user directory until leave 
 User can create and send/receive messages in a room with version 1
 User can create and send/receive messages in a room with version 2
 User can create and send/receive messages in a room with version 3