diff --git a/imapserver/server.go b/imapserver/server.go
index fd6eff1..627baf3 100644
--- a/imapserver/server.go
+++ b/imapserver/server.go
@@ -107,11 +107,11 @@ func (s *Server) logger() Logger {
 }
 
 // Serve accepts incoming connections on the listener ln.
-func (s *Server) Serve(ln net.Listener) error {
+func (s *Server) Serve(ln net.Listener, addr string) error {
 	s.mutex.Lock()
 	ok := !s.closed
 	if ok {
-		s.listeners[ln] = struct{}{}
+		s.listeners[addr] = struct{}{}
 	}
 	s.mutex.Unlock()
 	if !ok {
@@ -164,7 +164,7 @@ func (s *Server) ListenAndServe(addr string) error {
 	if err != nil {
 		return err
 	}
-	return s.Serve(ln)
+	return s.Serve(ln, addr)
 }
 
 // ListenAndServeTLS listens on the TCP network address addr and then calls
@@ -180,7 +180,7 @@ func (s *Server) ListenAndServeTLS(addr string) error {
 	if err != nil {
 		return err
 	}
-	return s.Serve(ln)
+	return s.Serve(ln, addr)
 }
 
 // Close immediately closes all active listeners and connections.