diff --git a/src/client/config.rs b/src/client/config.rs
index a6d5a6a..5f8c9b1 100644
--- a/src/client/config.rs
+++ b/src/client/config.rs
@@ -15,6 +15,10 @@ pub struct ServerConfig {
 }
 
 impl ServerConfig {
+    pub fn contains_key(&self, port: u16) -> bool {
+        self.ports.contains_key(&port)
+    }
+
     pub fn get(&self, port: u16) -> PortConfig {
         match self.ports.get(&port) {
             None => PortConfig { enabled: self.auto, description: None },
diff --git a/src/client/ui.rs b/src/client/ui.rs
index 8d979d6..49b3560 100644
--- a/src/client/ui.rs
+++ b/src/client/ui.rs
@@ -457,14 +457,11 @@ impl UI {
                 }
 
                 for port in leftover_ports {
-                    let mut enabled = false;
                     if let Some(listener) = self.ports.get_mut(&port) {
-                        enabled = listener.enabled;
                         listener.disconnect();
                     }
 
-                    if !enabled {
-                        // Just... forget it? Or leave it around forever?
+                    if !self.config.contains_key(port) {
                         self.ports.remove(&port);
                     }
                 }