From d96e4da1b6b27f424cdd4ed0a0d15ae016a1d848 Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Mon, 24 Aug 2020 07:57:10 +0000
Subject: [PATCH 01/15] UCP/WIREUP: Implement common copying of lanes function
 for futher re-use

---
 src/ucp/wireup/wireup.c    |  6 ++--
 src/ucp/wireup/wireup.h    |  2 ++
 src/ucp/wireup/wireup_cm.c | 72 +++++++++++++++++++++++++-------------
 src/ucp/wireup/wireup_ep.c |  4 +--
 src/ucp/wireup/wireup_ep.h |  2 +-
 5 files changed, 56 insertions(+), 30 deletions(-)

diff --git a/src/ucp/wireup/wireup.c b/src/ucp/wireup/wireup.c
index a1fb9138837..50d4df54fc0 100644
--- a/src/ucp/wireup/wireup.c
+++ b/src/ucp/wireup/wireup.c
@@ -701,12 +701,12 @@ void ucp_wireup_assign_lane(ucp_ep_h ep, ucp_lane_index_t lane, uct_ep_h uct_ep,
         ucs_assert(ucp_wireup_ep_test(ep->uct_eps[lane]));
         ucs_trace("ep %p: wireup uct_ep[%d]=%p next set to %p%s", ep, lane,
                   ep->uct_eps[lane], uct_ep, info);
-        ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep);
+        ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1);
         ucp_wireup_ep_remote_connected(ep->uct_eps[lane]);
     }
 }
 
-static uct_ep_h ucp_wireup_extract_lane(ucp_ep_h ep, ucp_lane_index_t lane)
+uct_ep_h ucp_wireup_extract_lane(ucp_ep_h ep, ucp_lane_index_t lane)
 {
     uct_ep_h uct_ep = ep->uct_eps[lane];
 
@@ -1223,7 +1223,7 @@ ucs_status_t ucp_wireup_connect_remote(ucp_ep_h ep, ucp_lane_index_t lane)
     uct_ep_pending_purge(uct_ep, ucp_wireup_connect_remote_purge_cb, &tmp_q);
 
     /* the wireup ep should use the existing [am_lane] as next_ep */
-    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep);
+    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1);
 
     if (!(ep->flags & UCP_EP_FLAG_CONNECT_REQ_QUEUED)) {
         status = ucp_wireup_send_request(ep);
diff --git a/src/ucp/wireup/wireup.h b/src/ucp/wireup/wireup.h
index 6b57608064d..4aec3bf7dec 100644
--- a/src/ucp/wireup/wireup.h
+++ b/src/ucp/wireup/wireup.h
@@ -132,6 +132,8 @@ ucs_status_t ucp_signaling_ep_create(ucp_ep_h ucp_ep, uct_ep_h uct_ep,
 void ucp_wireup_assign_lane(ucp_ep_h ep, ucp_lane_index_t lane, uct_ep_h uct_ep,
                             const char *info);
 
+uct_ep_h ucp_wireup_extract_lane(ucp_ep_h ep, ucp_lane_index_t lane);
+
 ucs_status_t
 ucp_wireup_connect_lane(ucp_ep_h ep, unsigned ep_init_flags,
                         ucp_lane_index_t lane, unsigned path_index,
diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index a417e2bf8ff..0f398953eb3 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -195,7 +195,7 @@ static ssize_t ucp_cm_client_priv_pack_cb(void *arg,
                 goto out;
             }
 
-            ucp_wireup_ep_set_next_ep(tmp_ep->uct_eps[lane_idx], tl_ep);
+            ucp_wireup_ep_set_next_ep(tmp_ep->uct_eps[lane_idx], tl_ep, 1);
         } else {
             ucs_assert(ucp_worker_is_tl_2iface(worker, rsc_idx));
         }
@@ -251,24 +251,48 @@ ucp_cm_client_connect_prog_arg_free(ucp_cm_client_connect_progress_arg_t *arg)
     ucs_free(arg);
 }
 
-static void ucp_cm_client_restore_ep(ucp_wireup_ep_t *wireup_cm_ep,
-                                     ucp_ep_h ucp_ep)
+static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
+                                 int change_ownership)
 {
-    ucp_ep_h tmp_ep = wireup_cm_ep->tmp_ep;
-    ucp_wireup_ep_t *w_ep;
+    int to_is_owner, from_is_owner;
     ucp_lane_index_t lane_idx;
+    ucs_status_t status;
+    uct_ep_h uct_ep;
 
-    for (lane_idx = 0; lane_idx < ucp_ep_num_lanes(tmp_ep); ++lane_idx) {
-        if (tmp_ep->uct_eps[lane_idx] != NULL) {
-            ucs_assert(ucp_ep->uct_eps[lane_idx] == NULL);
-            ucp_ep->uct_eps[lane_idx] = tmp_ep->uct_eps[lane_idx];
-            w_ep = ucs_derived_of(ucp_ep->uct_eps[lane_idx], ucp_wireup_ep_t);
-            w_ep->super.ucp_ep = ucp_ep;
-        }
+    if (change_ownership) {
+        to_is_owner   = 1;
+        from_is_owner = 0;
+    } else {
+        to_is_owner   = 0;
+        from_is_owner = 1;
     }
 
-    ucp_ep_destroy_base(tmp_ep); /* not needed anymore */
-    wireup_cm_ep->tmp_ep = NULL;
+    for (lane_idx = 0; lane_idx < ucp_ep_num_lanes(from_ep); ++lane_idx) {
+        if ((lane_idx == ucp_ep_get_cm_lane(from_ep)) ||
+            (from_ep->uct_eps[lane_idx] == NULL)) {
+            continue;
+        }
+
+        ucs_assert(to_ep->uct_eps[lane_idx] == NULL);
+        status = ucp_wireup_ep_create(to_ep, &to_ep->uct_eps[lane_idx]);
+        if (status != UCS_OK) {
+            /* coverity[leaked_storage] */
+            continue;
+        }
+
+        uct_ep = ucp_wireup_extract_lane(from_ep, lane_idx);
+        ucp_wireup_ep_set_next_ep(to_ep->uct_eps[lane_idx],
+                                  uct_ep, to_is_owner);
+
+        if (from_ep->uct_eps[lane_idx] == NULL) {
+            /* from_ep must be the owner EP in this case */
+            ucs_assert(from_is_owner);
+            from_ep->uct_eps[lane_idx] = uct_ep;
+        } else {
+            ucp_wireup_ep_set_next_ep(from_ep->uct_eps[lane_idx],
+                                      uct_ep, from_is_owner);
+        }
+    }
 }
 
 /*
@@ -281,7 +305,7 @@ static unsigned ucp_cm_client_connect_progress(void *arg)
     ucp_worker_h worker                                = ucp_ep->worker;
     ucp_context_h context                              = worker->context;
     uct_ep_h uct_cm_ep                                 = ucp_ep_get_cm_uct_ep(ucp_ep);
-    ucp_wireup_ep_t *wireup_ep;
+    ucp_wireup_ep_t *cm_wireup_ep;
     ucp_unpacked_address_t addr;
     uint64_t tl_bitmap;
     ucp_rsc_index_t dev_index;
@@ -292,9 +316,9 @@ static unsigned ucp_cm_client_connect_progress(void *arg)
 
     UCS_ASYNC_BLOCK(&worker->async);
 
-    wireup_ep = ucp_ep_get_cm_wireup_ep(ucp_ep);
-    ucs_assert(wireup_ep != NULL);
-    ucs_assert(wireup_ep->ep_init_flags & UCP_EP_INIT_CM_WIREUP_CLIENT);
+    cm_wireup_ep = ucp_ep_get_cm_wireup_ep(ucp_ep);
+    ucs_assert(cm_wireup_ep != NULL);
+    ucs_assert(cm_wireup_ep->ep_init_flags & UCP_EP_INIT_CM_WIREUP_CLIENT);
 
     status = ucp_address_unpack(worker, progress_arg->sa_data + 1,
                                 UCP_ADDRESS_PACK_FLAG_IFACE_ADDR |
@@ -317,14 +341,14 @@ static unsigned ucp_cm_client_connect_progress(void *arg)
     ucp_ep_update_remote_id(ucp_ep, progress_arg->sa_data->ep_id);
 
     /* Get tl bitmap from tmp_ep, because it contains initial configuration. */
-    tl_bitmap = ucp_ep_get_tl_bitmap(wireup_ep->tmp_ep);
+    tl_bitmap = ucp_ep_get_tl_bitmap(cm_wireup_ep->tmp_ep);
     ucs_assert(tl_bitmap != 0);
     rsc_index = ucs_ffs64(tl_bitmap);
     dev_index = context->tl_rscs[rsc_index].dev_index;
 
     /* Restore initial configuration from tmp_ep created for packing local
      * addresses. */
-    ucp_cm_client_restore_ep(wireup_ep, ucp_ep);
+    ucp_cm_copy_ep_lanes(ucp_ep, cm_wireup_ep->tmp_ep, 1);
 
 #ifdef ENABLE_ASSERT
     ucs_for_each_bit(rsc_index, tl_bitmap) {
@@ -333,7 +357,7 @@ static unsigned ucp_cm_client_connect_progress(void *arg)
 #endif
 
     tl_bitmap = ucp_context_dev_idx_tl_bitmap(context, dev_index);
-    status    = ucp_wireup_init_lanes(ucp_ep, wireup_ep->ep_init_flags,
+    status    = ucp_wireup_init_lanes(ucp_ep, cm_wireup_ep->ep_init_flags,
                                       tl_bitmap, &addr, addr_indices);
     if (status != UCS_OK) {
         goto out_free_addr;
@@ -357,7 +381,7 @@ static unsigned ucp_cm_client_connect_progress(void *arg)
     ucs_free(addr.address_list);
 out:
     if (status != UCS_OK) {
-        ucp_worker_set_ep_failed(worker, ucp_ep, &wireup_ep->super.super,
+        ucp_worker_set_ep_failed(worker, ucp_ep, &cm_wireup_ep->super.super,
                                  ucp_ep_get_cm_lane(ucp_ep), status);
     }
 
@@ -640,7 +664,7 @@ ucs_status_t ucp_ep_client_cm_connect_start(ucp_ep_h ucp_ep,
         return status;
     }
 
-    ucp_wireup_ep_set_next_ep(&wireup_ep->super.super, cm_ep);
+    ucp_wireup_ep_set_next_ep(&wireup_ep->super.super, cm_ep, 1);
     ucp_ep_flush_state_reset(ucp_ep);
 
     return UCS_OK;
@@ -966,7 +990,7 @@ ucs_status_t ucp_ep_cm_connect_server_lane(ucp_ep_h ep,
         return status;
     }
 
-    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep);
+    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1);
     return UCS_OK;
 }
 
diff --git a/src/ucp/wireup/wireup_ep.c b/src/ucp/wireup/wireup_ep.c
index a462c05e01e..5d70f81c990 100644
--- a/src/ucp/wireup/wireup_ep.c
+++ b/src/ucp/wireup/wireup_ep.c
@@ -646,14 +646,14 @@ ucs_status_t ucp_wireup_ep_connect_to_sockaddr(uct_ep_h uct_ep,
     return status;
 }
 
-void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep)
+void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep, int is_owner)
 {
     ucp_wireup_ep_t *wireup_ep = ucp_wireup_ep(uct_ep);
 
     ucs_assert(wireup_ep != NULL);
     ucs_assert(wireup_ep->super.uct_ep == NULL);
     wireup_ep->flags |= UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED;
-    ucp_proxy_ep_set_uct_ep(&wireup_ep->super, next_ep, 1);
+    ucp_proxy_ep_set_uct_ep(&wireup_ep->super, next_ep, is_owner);
 }
 
 uct_ep_h ucp_wireup_ep_extract_next_ep(uct_ep_h uct_ep)
diff --git a/src/ucp/wireup/wireup_ep.h b/src/ucp/wireup/wireup_ep.h
index efa3eb60f17..4579bf74335 100644
--- a/src/ucp/wireup/wireup_ep.h
+++ b/src/ucp/wireup/wireup_ep.h
@@ -82,7 +82,7 @@ ucs_status_t
 ucp_wireup_ep_connect_aux(ucp_wireup_ep_t *wireup_ep, unsigned ep_init_flags,
                           const ucp_unpacked_address_t *remote_address);
 
-void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep);
+void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep, int is_owner);
 
 uct_ep_h ucp_wireup_ep_extract_next_ep(uct_ep_h uct_ep);
 

From 9f70f8d03ca2aa829c94a4c6f3151a07bf8619b9 Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Tue, 25 Aug 2020 16:26:55 +0000
Subject: [PATCH 02/15] UCP/WIREUP: Add assertion to make sure that from_ep is
 an owner of UCT EP

---
 src/ucp/wireup/wireup_cm.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index 0f398953eb3..25eb91d969f 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -280,6 +280,9 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
             continue;
         }
 
+        ucs_assert(!ucp_wireup_ep_test(from_ep->uct_eps[lane_idx]) ||
+                   ucp_wireup_ep(from_ep->uct_eps[lane_idx])->super.is_owner);
+
         uct_ep = ucp_wireup_extract_lane(from_ep, lane_idx);
         ucp_wireup_ep_set_next_ep(to_ep->uct_eps[lane_idx],
                                   uct_ep, to_is_owner);

From 9b9e72dd5bd39e858db414518dfaef1b53b3ce66 Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Tue, 25 Aug 2020 20:18:19 +0000
Subject: [PATCH 03/15] UCP/WIREUP: Fix assertion

---
 src/ucp/wireup/wireup_cm.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index 25eb91d969f..ac6a473a9ea 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -280,10 +280,14 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
             continue;
         }
 
+        uct_ep = ucp_wireup_extract_lane(from_ep, lane_idx);
+        if (uct_ep == NULL) {
+            continue;
+        }
+
         ucs_assert(!ucp_wireup_ep_test(from_ep->uct_eps[lane_idx]) ||
                    ucp_wireup_ep(from_ep->uct_eps[lane_idx])->super.is_owner);
 
-        uct_ep = ucp_wireup_extract_lane(from_ep, lane_idx);
         ucp_wireup_ep_set_next_ep(to_ep->uct_eps[lane_idx],
                                   uct_ep, to_is_owner);
 

From 224b2443047d664e381cbd3f75f05033f4244bd1 Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Wed, 26 Aug 2020 07:14:45 +0000
Subject: [PATCH 04/15] UCP/WIREUP: Don't create WIREUP EP in to_ep if there is
 no UCT_EP

---
 src/ucp/wireup/wireup_cm.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index ac6a473a9ea..c8d43629a00 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -274,17 +274,18 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
         }
 
         ucs_assert(to_ep->uct_eps[lane_idx] == NULL);
-        status = ucp_wireup_ep_create(to_ep, &to_ep->uct_eps[lane_idx]);
-        if (status != UCS_OK) {
-            /* coverity[leaked_storage] */
-            continue;
-        }
 
         uct_ep = ucp_wireup_extract_lane(from_ep, lane_idx);
         if (uct_ep == NULL) {
             continue;
         }
 
+        status = ucp_wireup_ep_create(to_ep, &to_ep->uct_eps[lane_idx]);
+        if (status != UCS_OK) {
+            /* coverity[leaked_storage] */
+            continue;
+        }
+
         ucs_assert(!ucp_wireup_ep_test(from_ep->uct_eps[lane_idx]) ||
                    ucp_wireup_ep(from_ep->uct_eps[lane_idx])->super.is_owner);
 

From 312321a6fe2486f2b066ea3c8c9999d59dcda032 Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Wed, 26 Aug 2020 07:33:39 +0000
Subject: [PATCH 05/15] UCP/WIREUP: Fix review comments

---
 src/ucp/wireup/wireup_cm.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index c8d43629a00..65075115dc3 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -251,6 +251,16 @@ ucp_cm_client_connect_prog_arg_free(ucp_cm_client_connect_progress_arg_t *arg)
     ucs_free(arg);
 }
 
+/**
+ * Copies lanes from the one UCP EP to the another UCP EP. The function
+ * creates new WIREUP EPs for all lanes in @to_ep and sets UCT EP of
+ * the TLs from @from_ep. Both EPs have to be created and initalized.
+ *
+ * @param [in] to_ep               UCP EP handle to copy the lanes to.
+ * @param [in] from_ep             UCP EP handle to copy the lanes from.
+ * @param [in] change_ownership    Make WIREUP EPs in @to_ep are owner for
+ *                                 copied UCT EPs from @from_ep.
+ */
 static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
                                  int change_ownership)
 {
@@ -259,13 +269,8 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
     ucs_status_t status;
     uct_ep_h uct_ep;
 
-    if (change_ownership) {
-        to_is_owner   = 1;
-        from_is_owner = 0;
-    } else {
-        to_is_owner   = 0;
-        from_is_owner = 1;
-    }
+    to_is_owner   = change_ownership;
+    from_is_owner = !change_ownership;
 
     for (lane_idx = 0; lane_idx < ucp_ep_num_lanes(from_ep); ++lane_idx) {
         if ((lane_idx == ucp_ep_get_cm_lane(from_ep)) ||
@@ -277,6 +282,10 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
 
         uct_ep = ucp_wireup_extract_lane(from_ep, lane_idx);
         if (uct_ep == NULL) {
+            /* UCT EP could be NULL only for non-P2P TLs */
+            ucs_assert(ucp_worker_is_tl_2iface(from_ep->worker,
+                                               ucp_ep_config(from_ep)->key.
+                                               lanes[lane_idx].rsc_index));
             continue;
         }
 

From 2754df7d1703cf8bdd8438b901c257e2ccf0a5da Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Wed, 26 Aug 2020 10:29:48 +0000
Subject: [PATCH 06/15] UCP/WIREUP: Report fatal error if failed to allocate
 WIREUP EP

---
 src/ucp/wireup/wireup_cm.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index 65075115dc3..89c32211fa2 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -291,8 +291,8 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
 
         status = ucp_wireup_ep_create(to_ep, &to_ep->uct_eps[lane_idx]);
         if (status != UCS_OK) {
-            /* coverity[leaked_storage] */
-            continue;
+            ucs_fatal("%p: failed to create WIREUP EP to wrap %p UCT EP: %s",
+                      to_ep, uct_ep, ucs_status_string(status));
         }
 
         ucs_assert(!ucp_wireup_ep_test(from_ep->uct_eps[lane_idx]) ||

From d8c27f134f5735e507befd02e6aa9b011740ed04 Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Wed, 26 Aug 2020 13:24:59 +0000
Subject: [PATCH 07/15] UCP/WIREUP: Check that to_ep and from_ep have the same
 config

---
 src/ucp/wireup/wireup_cm.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index 89c32211fa2..e61987a4f49 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -272,6 +272,11 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
     to_is_owner   = change_ownership;
     from_is_owner = !change_ownership;
 
+    ucs_assertv_always(to_ep->cfg_index == from_ep->cfg_index,
+                       "to_ep=%p (cfg_index=%u) must has the same "
+                       "configuration as from_ep=%p (cfg_index=%u)",
+                       to_ep, to_ep->cfg_index, from_ep, from_ep->cfg_index);
+
     for (lane_idx = 0; lane_idx < ucp_ep_num_lanes(from_ep); ++lane_idx) {
         if ((lane_idx == ucp_ep_get_cm_lane(from_ep)) ||
             (from_ep->uct_eps[lane_idx] == NULL)) {

From 469168c3b6437f2f2dbbfe5d658e06bfdacfc537 Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Wed, 26 Aug 2020 13:51:55 +0000
Subject: [PATCH 08/15] UCP/WIREUP: Compare config_kets instead of cfg_index

---
 src/ucp/wireup/wireup_cm.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index e61987a4f49..ee63813814b 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -272,9 +272,10 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
     to_is_owner   = change_ownership;
     from_is_owner = !change_ownership;
 
-    ucs_assertv_always(to_ep->cfg_index == from_ep->cfg_index,
-                       "to_ep=%p (cfg_index=%u) must has the same "
-                       "configuration as from_ep=%p (cfg_index=%u)",
+    ucs_assertv_always(ucp_ep_config_is_equal(&ucp_ep_config(to_ep)->key,
+                                              &ucp_ep_config(from_ep)->key),
+                       "to_ep=%p (cfg_index=%u) must has a configuration that "
+                       "is equal to from_ep=%p (cfg_index=%u) configuration",
                        to_ep, to_ep->cfg_index, from_ep, from_ep->cfg_index);
 
     for (lane_idx = 0; lane_idx < ucp_ep_num_lanes(from_ep); ++lane_idx) {

From d698bf6d4706b70570fc4ee979a00bbbab09a4ea Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Wed, 26 Aug 2020 14:01:42 +0000
Subject: [PATCH 09/15] UCP/WIREUP: No need for assert

---
 src/ucp/wireup/wireup_cm.c | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index ee63813814b..89c32211fa2 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -272,12 +272,6 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
     to_is_owner   = change_ownership;
     from_is_owner = !change_ownership;
 
-    ucs_assertv_always(ucp_ep_config_is_equal(&ucp_ep_config(to_ep)->key,
-                                              &ucp_ep_config(from_ep)->key),
-                       "to_ep=%p (cfg_index=%u) must has a configuration that "
-                       "is equal to from_ep=%p (cfg_index=%u) configuration",
-                       to_ep, to_ep->cfg_index, from_ep, from_ep->cfg_index);
-
     for (lane_idx = 0; lane_idx < ucp_ep_num_lanes(from_ep); ++lane_idx) {
         if ((lane_idx == ucp_ep_get_cm_lane(from_ep)) ||
             (from_ep->uct_eps[lane_idx] == NULL)) {

From ed8a029dd9ac0f4ac83bbbaa78757f0de56ea8f2 Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Thu, 27 Aug 2020 04:46:39 +0000
Subject: [PATCH 10/15] UCP/WIREUP: Improve documentation

---
 src/ucp/wireup/wireup_cm.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index 89c32211fa2..9fcab1376c4 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -255,6 +255,10 @@ ucp_cm_client_connect_prog_arg_free(ucp_cm_client_connect_progress_arg_t *arg)
  * Copies lanes from the one UCP EP to the another UCP EP. The function
  * creates new WIREUP EPs for all lanes in @to_ep and sets UCT EP of
  * the TLs from @from_ep. Both EPs have to be created and initalized.
+ * @to_ep should not have initialized the lanes by UCT EPs that will be
+ * overwritten by UCT EPs from the @from_ep's lanes.
+ * After copying UCT EPs, @to_ep should try to reconfigure lanes and
+ * some of the copied UCT EPs could be re-used in the new configuration.
  *
  * @param [in] to_ep               UCP EP handle to copy the lanes to.
  * @param [in] from_ep             UCP EP handle to copy the lanes from.
@@ -278,7 +282,7 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
             continue;
         }
 
-        ucs_assert(to_ep->uct_eps[lane_idx] == NULL);
+        ucs_assert_always(to_ep->uct_eps[lane_idx] == NULL);
 
         uct_ep = ucp_wireup_extract_lane(from_ep, lane_idx);
         if (uct_ep == NULL) {
@@ -302,7 +306,7 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
                                   uct_ep, to_is_owner);
 
         if (from_ep->uct_eps[lane_idx] == NULL) {
-            /* from_ep must be the owner EP in this case */
+            /* from_ep must be the owner of the UCT EP in this case */
             ucs_assert(from_is_owner);
             from_ep->uct_eps[lane_idx] = uct_ep;
         } else {

From be70c9371bc1d6c98fd9dfdbe4db387f775bd1e3 Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Fri, 28 Aug 2020 09:23:44 +0000
Subject: [PATCH 11/15] UCP/WIREUP: Set local_conencted if needed

---
 src/ucp/wireup/wireup.c    |  8 +++++---
 src/ucp/wireup/wireup_cm.c | 26 ++++++++++++++++----------
 src/ucp/wireup/wireup_ep.c |  9 ++++++---
 src/ucp/wireup/wireup_ep.h |  3 ++-
 4 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/src/ucp/wireup/wireup.c b/src/ucp/wireup/wireup.c
index 50d4df54fc0..adf97ba6ea6 100644
--- a/src/ucp/wireup/wireup.c
+++ b/src/ucp/wireup/wireup.c
@@ -701,7 +701,7 @@ void ucp_wireup_assign_lane(ucp_ep_h ep, ucp_lane_index_t lane, uct_ep_h uct_ep,
         ucs_assert(ucp_wireup_ep_test(ep->uct_eps[lane]));
         ucs_trace("ep %p: wireup uct_ep[%d]=%p next set to %p%s", ep, lane,
                   ep->uct_eps[lane], uct_ep, info);
-        ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1);
+        ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1, 1);
         ucp_wireup_ep_remote_connected(ep->uct_eps[lane]);
     }
 }
@@ -868,6 +868,9 @@ ucs_status_t ucp_wireup_resolve_proxy_lanes(ucp_ep_h ep)
                               iface_attr->cap.am.max_bcopy);
         }
 
+        ucs_assert(ucp_wireup_ep(ep->uct_eps[proxy_lane])->flags &
+                   UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED);
+
         /* Create a signaling ep to the proxy lane */
         if (proxy_lane == lane) {
             /* If proxy is to the same lane, temporarily remove the existing
@@ -876,7 +879,6 @@ ucs_status_t ucp_wireup_resolve_proxy_lanes(ucp_ep_h ep)
              * proxy, so ucp_wireup_extract_lane() handles both cases.
              */
             uct_ep = ucp_wireup_extract_lane(ep, proxy_lane);
-            ucs_assert_always(uct_ep != NULL);
             status = ucp_signaling_ep_create(ep, uct_ep, 1, &signaling_ep);
             if (status != UCS_OK) {
                 /* coverity[leaked_storage] */
@@ -1223,7 +1225,7 @@ ucs_status_t ucp_wireup_connect_remote(ucp_ep_h ep, ucp_lane_index_t lane)
     uct_ep_pending_purge(uct_ep, ucp_wireup_connect_remote_purge_cb, &tmp_q);
 
     /* the wireup ep should use the existing [am_lane] as next_ep */
-    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1);
+    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1, 1);
 
     if (!(ep->flags & UCP_EP_FLAG_CONNECT_REQ_QUEUED)) {
         status = ucp_wireup_send_request(ep);
diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index 46317bb44b4..2e6b68a8744 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -266,6 +266,8 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
     ucp_lane_index_t lane_idx;
     ucs_status_t status;
     uct_ep_h uct_ep;
+    int is_local_connected;
+    ucp_wireup_ep_t *from_wireup_ep;
 
     to_is_owner   = change_ownership;
     from_is_owner = !change_ownership;
@@ -287,25 +289,29 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
             continue;
         }
 
+        from_wireup_ep = ucp_wireup_ep(from_ep->uct_eps[lane_idx]);
+
+        is_local_connected = (from_wireup_ep == NULL) ||
+                             (from_wireup_ep->flags &
+                              UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED);
+
         status = ucp_wireup_ep_create(to_ep, &to_ep->uct_eps[lane_idx]);
         if (status != UCS_OK) {
             ucs_fatal("%p: failed to create WIREUP EP to wrap %p UCT EP: %s",
                       to_ep, uct_ep, ucs_status_string(status));
         }
 
-        ucs_assert(!ucp_wireup_ep_test(from_ep->uct_eps[lane_idx]) ||
-                   ucp_wireup_ep(from_ep->uct_eps[lane_idx])->super.is_owner);
-
-        ucp_wireup_ep_set_next_ep(to_ep->uct_eps[lane_idx],
-                                  uct_ep, to_is_owner);
+        ucp_wireup_ep_set_next_ep(to_ep->uct_eps[lane_idx], uct_ep,
+                                  to_is_owner, is_local_connected);
 
-        if (from_ep->uct_eps[lane_idx] == NULL) {
+        if (from_wireup_ep == NULL) {
             /* from_ep must be the owner of the UCT EP in this case */
             ucs_assert(from_is_owner);
             from_ep->uct_eps[lane_idx] = uct_ep;
         } else {
-            ucp_wireup_ep_set_next_ep(from_ep->uct_eps[lane_idx],
-                                      uct_ep, from_is_owner);
+            ucs_assert(from_wireup_ep->super.is_owner);
+            ucp_wireup_ep_set_next_ep(from_ep->uct_eps[lane_idx], uct_ep,
+                                      from_is_owner, is_local_connected);
         }
     }
 }
@@ -679,7 +685,7 @@ ucs_status_t ucp_ep_client_cm_connect_start(ucp_ep_h ucp_ep,
         return status;
     }
 
-    ucp_wireup_ep_set_next_ep(&wireup_ep->super.super, cm_ep, 1);
+    ucp_wireup_ep_set_next_ep(&wireup_ep->super.super, cm_ep, 1, 1);
     ucp_ep_flush_state_reset(ucp_ep);
 
     return UCS_OK;
@@ -1005,7 +1011,7 @@ ucs_status_t ucp_ep_cm_connect_server_lane(ucp_ep_h ep,
         return status;
     }
 
-    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1);
+    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1, 1);
     return UCS_OK;
 }
 
diff --git a/src/ucp/wireup/wireup_ep.c b/src/ucp/wireup/wireup_ep.c
index 054fdbd5e10..d5415cd17a3 100644
--- a/src/ucp/wireup/wireup_ep.c
+++ b/src/ucp/wireup/wireup_ep.c
@@ -439,7 +439,7 @@ ucs_status_t ucp_wireup_ep_connect(uct_ep_h uct_ep, unsigned ep_init_flags,
         goto err;
     }
 
-    ucp_proxy_ep_set_uct_ep(&wireup_ep->super, next_ep, 1);
+    ucp_wireup_ep_set_next_ep(uct_ep, next_ep, 1, 0);
 
     ucs_debug("ep %p: created next_ep %p to %s using " UCT_TL_RESOURCE_DESC_FMT,
               ucp_ep, wireup_ep->super.uct_ep, ucp_ep_peer_name(ucp_ep),
@@ -647,13 +647,16 @@ ucs_status_t ucp_wireup_ep_connect_to_sockaddr(uct_ep_h uct_ep,
     return status;
 }
 
-void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep, int is_owner)
+void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep,
+                               int is_owner, int is_local_connected)
 {
     ucp_wireup_ep_t *wireup_ep = ucp_wireup_ep(uct_ep);
 
     ucs_assert(wireup_ep != NULL);
     ucs_assert(wireup_ep->super.uct_ep == NULL);
-    wireup_ep->flags |= UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED;
+    if (is_local_connected) {
+        wireup_ep->flags |= UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED;
+    }
     ucp_proxy_ep_set_uct_ep(&wireup_ep->super, next_ep, is_owner);
 }
 
diff --git a/src/ucp/wireup/wireup_ep.h b/src/ucp/wireup/wireup_ep.h
index 4579bf74335..ac095b9f773 100644
--- a/src/ucp/wireup/wireup_ep.h
+++ b/src/ucp/wireup/wireup_ep.h
@@ -82,7 +82,8 @@ ucs_status_t
 ucp_wireup_ep_connect_aux(ucp_wireup_ep_t *wireup_ep, unsigned ep_init_flags,
                           const ucp_unpacked_address_t *remote_address);
 
-void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep, int is_owner);
+void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep,
+                               int is_owner, int is_local_connected);
 
 uct_ep_h ucp_wireup_ep_extract_next_ep(uct_ep_h uct_ep);
 

From 27d718c16a0e5493e4e588d191178a6b161168c8 Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Fri, 28 Aug 2020 10:58:36 +0000
Subject: [PATCH 12/15] UCP/WIREUP: Fix resolving proxy lanes

---
 src/ucp/wireup/wireup.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/ucp/wireup/wireup.c b/src/ucp/wireup/wireup.c
index adf97ba6ea6..b27f8bd96ab 100644
--- a/src/ucp/wireup/wireup.c
+++ b/src/ucp/wireup/wireup.c
@@ -868,8 +868,9 @@ ucs_status_t ucp_wireup_resolve_proxy_lanes(ucp_ep_h ep)
                               iface_attr->cap.am.max_bcopy);
         }
 
-        ucs_assert(ucp_wireup_ep(ep->uct_eps[proxy_lane])->flags &
-                   UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED);
+        ucs_assert((ucp_wireup_ep(ep->uct_eps[proxy_lane]) == NULL) ||
+                   (ucp_wireup_ep(ep->uct_eps[proxy_lane])->flags &
+                    UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED));
 
         /* Create a signaling ep to the proxy lane */
         if (proxy_lane == lane) {

From ec96435bbab7f028ac7b5fd088d6e471d77ffcae Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Tue, 1 Sep 2020 13:04:13 +0000
Subject: [PATCH 13/15] UCP/WIREUP: Fix review comments

---
 src/ucp/core/ucp_proxy_ep.c |  1 +
 src/ucp/wireup/wireup.c     |  4 ++--
 src/ucp/wireup/wireup_cm.c  | 28 ++++++++++++++++------------
 src/ucp/wireup/wireup_ep.c  | 17 ++++++-----------
 src/ucp/wireup/wireup_ep.h  |  3 +--
 5 files changed, 26 insertions(+), 27 deletions(-)

diff --git a/src/ucp/core/ucp_proxy_ep.c b/src/ucp/core/ucp_proxy_ep.c
index 8e384205f37..0fbf92556e4 100644
--- a/src/ucp/core/ucp_proxy_ep.c
+++ b/src/ucp/core/ucp_proxy_ep.c
@@ -239,6 +239,7 @@ void ucp_proxy_ep_replace(ucp_proxy_ep_t *proxy_ep)
 void ucp_proxy_ep_set_uct_ep(ucp_proxy_ep_t *proxy_ep, uct_ep_h uct_ep,
                              int is_owner)
 {
+    ucs_assert((proxy_ep != NULL) && (proxy_ep->uct_ep == NULL));
     proxy_ep->uct_ep   = uct_ep;
     proxy_ep->is_owner = is_owner;
 }
diff --git a/src/ucp/wireup/wireup.c b/src/ucp/wireup/wireup.c
index b27f8bd96ab..0c82292bb76 100644
--- a/src/ucp/wireup/wireup.c
+++ b/src/ucp/wireup/wireup.c
@@ -701,7 +701,7 @@ void ucp_wireup_assign_lane(ucp_ep_h ep, ucp_lane_index_t lane, uct_ep_h uct_ep,
         ucs_assert(ucp_wireup_ep_test(ep->uct_eps[lane]));
         ucs_trace("ep %p: wireup uct_ep[%d]=%p next set to %p%s", ep, lane,
                   ep->uct_eps[lane], uct_ep, info);
-        ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1, 1);
+        ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1);
         ucp_wireup_ep_remote_connected(ep->uct_eps[lane]);
     }
 }
@@ -1226,7 +1226,7 @@ ucs_status_t ucp_wireup_connect_remote(ucp_ep_h ep, ucp_lane_index_t lane)
     uct_ep_pending_purge(uct_ep, ucp_wireup_connect_remote_purge_cb, &tmp_q);
 
     /* the wireup ep should use the existing [am_lane] as next_ep */
-    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1, 1);
+    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1);
 
     if (!(ep->flags & UCP_EP_FLAG_CONNECT_REQ_QUEUED)) {
         status = ucp_wireup_send_request(ep);
diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index 2e6b68a8744..1224923f943 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -266,8 +266,8 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
     ucp_lane_index_t lane_idx;
     ucs_status_t status;
     uct_ep_h uct_ep;
-    int is_local_connected;
     ucp_wireup_ep_t *from_wireup_ep;
+    ucp_wireup_ep_t *to_wireup_ep;
 
     to_is_owner   = change_ownership;
     from_is_owner = !change_ownership;
@@ -291,27 +291,31 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
 
         from_wireup_ep = ucp_wireup_ep(from_ep->uct_eps[lane_idx]);
 
-        is_local_connected = (from_wireup_ep == NULL) ||
-                             (from_wireup_ep->flags &
-                              UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED);
-
         status = ucp_wireup_ep_create(to_ep, &to_ep->uct_eps[lane_idx]);
         if (status != UCS_OK) {
             ucs_fatal("%p: failed to create WIREUP EP to wrap %p UCT EP: %s",
                       to_ep, uct_ep, ucs_status_string(status));
         }
 
-        ucp_wireup_ep_set_next_ep(to_ep->uct_eps[lane_idx], uct_ep,
-                                  to_is_owner, is_local_connected);
+        from_wireup_ep = ucp_wireup_ep(from_ep->uct_eps[lane_idx]);
+        to_wireup_ep   = ucp_wireup_ep(to_ep->uct_eps[lane_idx]);
+
+        ucp_proxy_ep_set_uct_ep(&to_wireup_ep->super, uct_ep, to_is_owner);
 
         if (from_wireup_ep == NULL) {
-            /* from_ep must be the owner of the UCT EP in this case */
+            /* wireup EPs couldn't exist for the from_ep lanes if this is
+             * called on the server side of the conenction to copy the lanes
+             * from the main UCP EP to the TMP EP.
+             * from_ep must be the owner of the UCT EP in this case. */
             ucs_assert(from_is_owner);
             from_ep->uct_eps[lane_idx] = uct_ep;
+            to_wireup_ep->flags        = UCP_WIREUP_EP_FLAG_READY |
+                                         UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED;
         } else {
             ucs_assert(from_wireup_ep->super.is_owner);
-            ucp_wireup_ep_set_next_ep(from_ep->uct_eps[lane_idx], uct_ep,
-                                      from_is_owner, is_local_connected);
+            ucp_proxy_ep_set_uct_ep(&from_wireup_ep->super, uct_ep,
+                                    from_is_owner);
+            to_wireup_ep->flags = from_wireup_ep->flags;
         }
     }
 }
@@ -685,7 +689,7 @@ ucs_status_t ucp_ep_client_cm_connect_start(ucp_ep_h ucp_ep,
         return status;
     }
 
-    ucp_wireup_ep_set_next_ep(&wireup_ep->super.super, cm_ep, 1, 1);
+    ucp_wireup_ep_set_next_ep(&wireup_ep->super.super, cm_ep, 1);
     ucp_ep_flush_state_reset(ucp_ep);
 
     return UCS_OK;
@@ -1011,7 +1015,7 @@ ucs_status_t ucp_ep_cm_connect_server_lane(ucp_ep_h ep,
         return status;
     }
 
-    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1, 1);
+    ucp_wireup_ep_set_next_ep(ep->uct_eps[lane], uct_ep, 1);
     return UCS_OK;
 }
 
diff --git a/src/ucp/wireup/wireup_ep.c b/src/ucp/wireup/wireup_ep.c
index d5415cd17a3..56926bececf 100644
--- a/src/ucp/wireup/wireup_ep.c
+++ b/src/ucp/wireup/wireup_ep.c
@@ -419,9 +419,9 @@ ucs_status_t ucp_wireup_ep_connect(uct_ep_h uct_ep, unsigned ep_init_flags,
                                    unsigned path_index, int connect_aux,
                                    const ucp_unpacked_address_t *remote_address)
 {
-    ucp_wireup_ep_t *wireup_ep     = ucp_wireup_ep(uct_ep);
-    ucp_ep_h ucp_ep                = wireup_ep->super.ucp_ep;
-    ucp_worker_h worker            = ucp_ep->worker;
+    ucp_wireup_ep_t *wireup_ep = ucp_wireup_ep(uct_ep);
+    ucp_ep_h ucp_ep            = wireup_ep->super.ucp_ep;
+    ucp_worker_h worker        = ucp_ep->worker;
     uct_ep_params_t uct_ep_params;
     ucs_status_t status;
     uct_ep_h next_ep;
@@ -439,7 +439,7 @@ ucs_status_t ucp_wireup_ep_connect(uct_ep_h uct_ep, unsigned ep_init_flags,
         goto err;
     }
 
-    ucp_wireup_ep_set_next_ep(uct_ep, next_ep, 1, 0);
+    ucp_proxy_ep_set_uct_ep(&wireup_ep->super, next_ep, 1);
 
     ucs_debug("ep %p: created next_ep %p to %s using " UCT_TL_RESOURCE_DESC_FMT,
               ucp_ep, wireup_ep->super.uct_ep, ucp_ep_peer_name(ucp_ep),
@@ -647,17 +647,12 @@ ucs_status_t ucp_wireup_ep_connect_to_sockaddr(uct_ep_h uct_ep,
     return status;
 }
 
-void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep,
-                               int is_owner, int is_local_connected)
+void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep, int is_owner)
 {
     ucp_wireup_ep_t *wireup_ep = ucp_wireup_ep(uct_ep);
 
-    ucs_assert(wireup_ep != NULL);
-    ucs_assert(wireup_ep->super.uct_ep == NULL);
-    if (is_local_connected) {
-        wireup_ep->flags |= UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED;
-    }
     ucp_proxy_ep_set_uct_ep(&wireup_ep->super, next_ep, is_owner);
+    wireup_ep->flags |= UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED;
 }
 
 uct_ep_h ucp_wireup_ep_extract_next_ep(uct_ep_h uct_ep)
diff --git a/src/ucp/wireup/wireup_ep.h b/src/ucp/wireup/wireup_ep.h
index ac095b9f773..4579bf74335 100644
--- a/src/ucp/wireup/wireup_ep.h
+++ b/src/ucp/wireup/wireup_ep.h
@@ -82,8 +82,7 @@ ucs_status_t
 ucp_wireup_ep_connect_aux(ucp_wireup_ep_t *wireup_ep, unsigned ep_init_flags,
                           const ucp_unpacked_address_t *remote_address);
 
-void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep,
-                               int is_owner, int is_local_connected);
+void ucp_wireup_ep_set_next_ep(uct_ep_h uct_ep, uct_ep_h next_ep, int is_owner);
 
 uct_ep_h ucp_wireup_ep_extract_next_ep(uct_ep_h uct_ep);
 

From 5d2aa0a3dda61afb6235e89ea0555c9ae6e9fa0d Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Tue, 1 Sep 2020 13:28:02 +0000
Subject: [PATCH 14/15] UCP/WIREUP: Remove debug note from LOCAL_CONENCTED flag

---
 src/ucp/wireup/wireup_ep.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ucp/wireup/wireup_ep.h b/src/ucp/wireup/wireup_ep.h
index 4579bf74335..dd92c35e805 100644
--- a/src/ucp/wireup/wireup_ep.h
+++ b/src/ucp/wireup/wireup_ep.h
@@ -20,7 +20,7 @@
  */
 enum {
     UCP_WIREUP_EP_FLAG_READY           = UCS_BIT(0), /**< next_ep is fully connected */
-    UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED = UCS_BIT(1), /**< Debug: next_ep connected to remote */
+    UCP_WIREUP_EP_FLAG_LOCAL_CONNECTED = UCS_BIT(1), /**< next_ep connected to remote */
 };
 
 

From 863b57d585b1b663e8c97c63c892cf3f823d7bb0 Mon Sep 17 00:00:00 2001
From: dmitrygx <dmitrygla@mellanox.com>
Date: Tue, 1 Sep 2020 20:45:13 +0000
Subject: [PATCH 15/15] UCP/WIREUP: Fix static checker issue

---
 src/ucp/wireup/wireup_cm.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/ucp/wireup/wireup_cm.c b/src/ucp/wireup/wireup_cm.c
index 1224923f943..285efe00b04 100644
--- a/src/ucp/wireup/wireup_cm.c
+++ b/src/ucp/wireup/wireup_cm.c
@@ -288,9 +288,6 @@ static void ucp_cm_copy_ep_lanes(ucp_ep_h to_ep, ucp_ep_h from_ep,
                                                lanes[lane_idx].rsc_index));
             continue;
         }
-
-        from_wireup_ep = ucp_wireup_ep(from_ep->uct_eps[lane_idx]);
-
         status = ucp_wireup_ep_create(to_ep, &to_ep->uct_eps[lane_idx]);
         if (status != UCS_OK) {
             ucs_fatal("%p: failed to create WIREUP EP to wrap %p UCT EP: %s",