diff --git a/proxy/socks/protocol.go b/proxy/socks/protocol.go
index 5d4cfd9d9b34..829d9a571616 100644
--- a/proxy/socks/protocol.go
+++ b/proxy/socks/protocol.go
@@ -214,7 +214,9 @@ func (s *ServerSession) handshake5(nMethod byte, reader io.Reader, writer net.Co
 			expectedRemote.IP = writer.RemoteAddr().(*net.TCPAddr).IP // unix?
 		} else {
 			expectedRemote.IP = request.Address.IP()
-			expectedRemote.Port = int(request.Port) // 0 is allowed
+		}
+		if request.Port != 0 {
+			expectedRemote.Port = int(request.Port)
 		}
 		tempUDPConn = NewTempUDPConn(udpHub, writer, expectedRemote)
 	}
diff --git a/proxy/socks/temp_udp_listen.go b/proxy/socks/temp_udp_listen.go
index 5e0dab03d977..d1459c67bc9c 100644
--- a/proxy/socks/temp_udp_listen.go
+++ b/proxy/socks/temp_udp_listen.go
@@ -37,15 +37,9 @@ func (c *TempUDPConn) Read(b []byte) (n int, err error) {
 		remote := remote.(*net.UDPAddr)
 		expected := c.ExpectedRemote.Load()
 		if remote.IP.Equal(expected.IP) {
-			if remote.Port == expected.Port {
-				c.Timer.Update()
-				return
-			}
-			if expected.Port == 0 {
-				c.ExpectedRemote.Store(remote)
-				c.Timer.Update()
-				return
-			}
+			c.ExpectedRemote.Store(remote)
+			c.Timer.Update()
+			return
 		}
 	}
 }