DAOS-16630 control: improve dmg pool exclude

When invoking “dmg pool exclude”, an administrator may not be aware of
whether this will break the pool-wise RF or not. If this breaks the pool-wise RF,
there may be data loss. Hence, we should perform the pool-wise RF check
(i.e., the pw_rf check in pool_svc_update_map_internal) before changing the pool map.

On the other hand, the administrator may also want to exclude rank/targets even if
there may be data loss. We should introduce a new, explicit option for the administrator
to say that she’s aware of the danger and would like to bypass the pool-wise RF check.

Signed-off-by: Wang Shilong <[email protected]>

docs/admin/pool_operations.md

@@ -1235,6 +1235,13 @@ The pool target exclude command accepts 2 parameters:
 Upon successful manual exclusion, the self-healing mechanism will be triggered
 to restore redundancy on the remaining engines.
 
+!!! note
+    Exclusion may compromise the Pool Redundancy Factor (RF), potentially leading
+    to data loss. If this occurs, the command will return the error code -DER_RF.
+    In such cases, you can proceed with the exclusion by specifying the --force option.
+    Please note that forcing the operation may result in data loss, and it is strongly
+    recommended to verify the RF status before proceeding.
+
 ### Drain
 
 Alternatively, when an operator would like to remove one or more engines or

src/control/cmd/dmg/pool.go

@@ -533,6 +533,7 @@ func (cmd *poolEvictCmd) Execute(args []string) error {
 // PoolExcludeCmd is the struct representing the command to exclude a DAOS target.
 type poolExcludeCmd struct {
 	poolCmd
+	Force     bool   `short:"f" long:"force" description:"Force the operation to continue, potentially leading to data loss"`
 	Rank      uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be excluded"`
 	TargetIdx string `long:"target-idx" description:"Comma-separated list of target idx(s) to be excluded from the rank"`
 }
@@ -546,7 +547,7 @@ func (cmd *poolExcludeCmd) Execute(args []string) error {
 		return errors.WithMessage(err, "parsing target list")
 	}
 
-	req := &control.PoolExcludeReq{ID: cmd.PoolID().String(), Rank: ranklist.Rank(cmd.Rank), TargetIdx: idxList}
+	req := &control.PoolExcludeReq{ID: cmd.PoolID().String(), Rank: ranklist.Rank(cmd.Rank), TargetIdx: idxList, Force: cmd.Force}
 
 	err := control.PoolExclude(cmd.MustLogCtx(), cmd.ctlInvoker, req)
 	if err != nil {

src/control/cmd/dmg/pool_test.go

@@ -621,30 +621,33 @@ func TestPoolCommands(t *testing.T) {
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
 					Rank:      0,
 					TargetIdx: []uint32{1},
+					Force:     false,
 				}),
 			}, " "),
 			nil,
 		},
 		{
 			"Exclude a target with multiple idx",
-			"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1,2,3",
+			"pool exclude -f 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0 --target-idx 1,2,3",
 			strings.Join([]string{
 				printRequest(t, &control.PoolExcludeReq{
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
 					Rank:      0,
 					TargetIdx: []uint32{1, 2, 3},
+					Force:     true,
 				}),
 			}, " "),
 			nil,
 		},
 		{
 			"Exclude a target with no idx given",
-			"pool exclude 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0",
+			"pool exclude -f 031bcaf8-f0f5-42ef-b3c5-ee048676dceb --rank 0",
 			strings.Join([]string{
 				printRequest(t, &control.PoolExcludeReq{
 					ID:        "031bcaf8-f0f5-42ef-b3c5-ee048676dceb",
 					Rank:      0,
 					TargetIdx: []uint32{},
+					Force:     true,
 				}),
 			}, " "),
 			nil,

src/control/lib/control/pool.go

@@ -748,6 +748,7 @@ type PoolExcludeReq struct {
 	ID        string
 	Rank      ranklist.Rank
 	TargetIdx []uint32
+	Force     bool
 }
 
 // ExcludeResp has no other parameters other than success/failure for now.
@@ -761,6 +762,7 @@ func PoolExclude(ctx context.Context, rpcClient UnaryInvoker, req *PoolExcludeRe
 		Id:        req.ID,
 		Rank:      req.Rank.Uint32(),
 		TargetIdx: req.TargetIdx,
+		Force:     req.Force,
 	}
 	req.setRPC(func(ctx context.Context, conn *grpc.ClientConn) (proto.Message, error) {
 		return mgmtpb.NewMgmtSvcClient(conn).PoolExclude(ctx, pbReq)

src/include/daos_srv/pool.h

@@ -285,9 +285,10 @@ int ds_pool_start(uuid_t uuid, bool aft_chk, bool immutable);
 int ds_pool_stop(uuid_t uuid);
 int dsc_pool_svc_extend(uuid_t pool_uuid, d_rank_list_t *svc_ranks, uint64_t deadline, int ntargets,
 			const d_rank_list_t *rank_list, int ndomains, const uint32_t *domains);
-int dsc_pool_svc_update_target_state(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline,
-				     struct pool_target_addr_list *target_list,
-				     pool_comp_state_t state);
+int
+	 dsc_pool_svc_update_target_state(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline,
+					  struct pool_target_addr_list *target_list, pool_comp_state_t state,
+					  bool skip_rf_check);
 
 uint32_t ds_pool_get_vos_df_version_default(void);
 int ds_pool_svc_dist_create(const uuid_t pool_uuid, int ntargets, const char *group,

src/mgmt/pool.pb-c.c

@@ -2109,95 +2109,65 @@ const ProtobufCMessageDescriptor mgmt__pool_evict_resp__descriptor =
   (ProtobufCMessageInit) mgmt__pool_evict_resp__init,
   NULL,NULL,NULL    /* reserved[123] */
 };
-static const ProtobufCFieldDescriptor mgmt__pool_exclude_req__field_descriptors[5] =
-{
-  {
-    "sys",
+static const ProtobufCFieldDescriptor mgmt__pool_exclude_req__field_descriptors[6] = {
+    {
+	"sys", 1, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */
+	offsetof(Mgmt__PoolExcludeReq, sys), NULL, &protobuf_c_empty_string, 0, /* flags */
+	0, NULL, NULL /* reserved1,reserved2, etc */
+    },
+    {
+	"id", 2, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_STRING, 0, /* quantifier_offset */
+	offsetof(Mgmt__PoolExcludeReq, id), NULL, &protobuf_c_empty_string, 0, /* flags */
+	0, NULL, NULL /* reserved1,reserved2, etc */
+    },
+    {
+	"rank", 3, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_UINT32, 0, /* quantifier_offset */
+	offsetof(Mgmt__PoolExcludeReq, rank), NULL, NULL, 0,         /* flags */
+	0, NULL, NULL                                                /* reserved1,reserved2, etc */
+    },
+    {
+	"target_idx", 4, PROTOBUF_C_LABEL_REPEATED, PROTOBUF_C_TYPE_UINT32,
+	offsetof(Mgmt__PoolExcludeReq, n_target_idx), offsetof(Mgmt__PoolExcludeReq, target_idx),
+	NULL, NULL, 0, /* flags */
+	0, NULL, NULL  /* reserved1,reserved2, etc */
+    },
+    {
+	"svc_ranks", 5, PROTOBUF_C_LABEL_REPEATED, PROTOBUF_C_TYPE_UINT32,
+	offsetof(Mgmt__PoolExcludeReq, n_svc_ranks), offsetof(Mgmt__PoolExcludeReq, svc_ranks),
+	NULL, NULL, 0, /* flags */
+	0, NULL, NULL  /* reserved1,reserved2, etc */
+    },
+    {
+	"force", 6, PROTOBUF_C_LABEL_NONE, PROTOBUF_C_TYPE_BOOL, 0, /* quantifier_offset */
+	offsetof(Mgmt__PoolExcludeReq, force), NULL, NULL, 0,       /* flags */
+	0, NULL, NULL                                               /* reserved1,reserved2, etc */
+    },
+};
+static const unsigned mgmt__pool_exclude_req__field_indices_by_name[] = {
+    5, /* field[5] = force */
+    1, /* field[1] = id */
+    2, /* field[2] = rank */
+    4, /* field[4] = svc_ranks */
+    0, /* field[0] = sys */
+    3, /* field[3] = target_idx */
+};
+static const ProtobufCIntRange   mgmt__pool_exclude_req__number_ranges[1 + 1] = {{1, 0}, {0, 6}};
+const ProtobufCMessageDescriptor mgmt__pool_exclude_req__descriptor           = {
+    PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC,
+    "mgmt.PoolExcludeReq",
+    "PoolExcludeReq",
+    "Mgmt__PoolExcludeReq",
+    "mgmt",
+    sizeof(Mgmt__PoolExcludeReq),
+    6,
+    mgmt__pool_exclude_req__field_descriptors,
+    mgmt__pool_exclude_req__field_indices_by_name,
     1,
-    PROTOBUF_C_LABEL_NONE,
-    PROTOBUF_C_TYPE_STRING,
-    0,   /* quantifier_offset */
-    offsetof(Mgmt__PoolExcludeReq, sys),
-    NULL,
-    &protobuf_c_empty_string,
-    0,             /* flags */
-    0,NULL,NULL    /* reserved1,reserved2, etc */
-  },
-  {
-    "id",
-    2,
-    PROTOBUF_C_LABEL_NONE,
-    PROTOBUF_C_TYPE_STRING,
-    0,   /* quantifier_offset */
-    offsetof(Mgmt__PoolExcludeReq, id),
-    NULL,
-    &protobuf_c_empty_string,
-    0,             /* flags */
-    0,NULL,NULL    /* reserved1,reserved2, etc */
-  },
-  {
-    "rank",
-    3,
-    PROTOBUF_C_LABEL_NONE,
-    PROTOBUF_C_TYPE_UINT32,
-    0,   /* quantifier_offset */
-    offsetof(Mgmt__PoolExcludeReq, rank),
-    NULL,
-    NULL,
-    0,             /* flags */
-    0,NULL,NULL    /* reserved1,reserved2, etc */
-  },
-  {
-    "target_idx",
-    4,
-    PROTOBUF_C_LABEL_REPEATED,
-    PROTOBUF_C_TYPE_UINT32,
-    offsetof(Mgmt__PoolExcludeReq, n_target_idx),
-    offsetof(Mgmt__PoolExcludeReq, target_idx),
-    NULL,
+    mgmt__pool_exclude_req__number_ranges,
+    (ProtobufCMessageInit)mgmt__pool_exclude_req__init,
     NULL,
-    0,             /* flags */
-    0,NULL,NULL    /* reserved1,reserved2, etc */
-  },
-  {
-    "svc_ranks",
-    5,
-    PROTOBUF_C_LABEL_REPEATED,
-    PROTOBUF_C_TYPE_UINT32,
-    offsetof(Mgmt__PoolExcludeReq, n_svc_ranks),
-    offsetof(Mgmt__PoolExcludeReq, svc_ranks),
     NULL,
-    NULL,
-    0,             /* flags */
-    0,NULL,NULL    /* reserved1,reserved2, etc */
-  },
-};
-static const unsigned mgmt__pool_exclude_req__field_indices_by_name[] = {
-  1,   /* field[1] = id */
-  2,   /* field[2] = rank */
-  4,   /* field[4] = svc_ranks */
-  0,   /* field[0] = sys */
-  3,   /* field[3] = target_idx */
-};
-static const ProtobufCIntRange mgmt__pool_exclude_req__number_ranges[1 + 1] =
-{
-  { 1, 0 },
-  { 0, 5 }
-};
-const ProtobufCMessageDescriptor mgmt__pool_exclude_req__descriptor =
-{
-  PROTOBUF_C__MESSAGE_DESCRIPTOR_MAGIC,
-  "mgmt.PoolExcludeReq",
-  "PoolExcludeReq",
-  "Mgmt__PoolExcludeReq",
-  "mgmt",
-  sizeof(Mgmt__PoolExcludeReq),
-  5,
-  mgmt__pool_exclude_req__field_descriptors,
-  mgmt__pool_exclude_req__field_indices_by_name,
-  1,  mgmt__pool_exclude_req__number_ranges,
-  (ProtobufCMessageInit) mgmt__pool_exclude_req__init,
-  NULL,NULL,NULL    /* reserved[123] */
+    NULL /* reserved[123] */
 };
 static const ProtobufCFieldDescriptor mgmt__pool_exclude_resp__field_descriptors[1] =
 {

src/mgmt/srv_drpc.c

@@ -709,7 +709,7 @@ ds_mgmt_drpc_pool_evict(Drpc__Call *drpc_req, Drpc__Response *drpc_resp)
 static int
 pool_change_target_state(char *id, d_rank_list_t *svc_ranks, size_t n_target_idx,
 			 uint32_t *target_idx, uint32_t rank, pool_comp_state_t state,
-			 size_t scm_size, size_t nvme_size, size_t meta_size)
+			 size_t scm_size, size_t nvme_size, size_t meta_size, bool skip_rf_check)
 {
 	uuid_t				uuid;
 	struct pool_target_addr_list	target_addr_list;
@@ -738,7 +738,7 @@ pool_change_target_state(char *id, d_rank_list_t *svc_ranks, size_t n_target_idx
 	}
 
 	rc = ds_mgmt_pool_target_update_state(uuid, svc_ranks, &target_addr_list, state, scm_size,
-					      nvme_size, meta_size);
+					      nvme_size, meta_size, skip_rf_check);
 	if (rc != 0) {
 		D_ERROR("Failed to set pool target up "DF_UUID": "DF_RC"\n",
 			DP_UUID(uuid), DP_RC(rc));
@@ -778,7 +778,7 @@ ds_mgmt_drpc_pool_exclude(Drpc__Call *drpc_req, Drpc__Response *drpc_resp)
 
 	rc = pool_change_target_state(req->id, svc_ranks, req->n_target_idx, req->target_idx,
 				      req->rank, PO_COMP_ST_DOWN, 0 /* scm_size */,
-				      0 /* nvme_size */, 0 /* meta_size */);
+				      0 /* nvme_size */, 0 /* meta_size */, req->force);
 
 	d_rank_list_free(svc_ranks);
 
@@ -827,7 +827,7 @@ ds_mgmt_drpc_pool_drain(Drpc__Call *drpc_req, Drpc__Response *drpc_resp)
 
 	rc = pool_change_target_state(req->id, svc_ranks, req->n_target_idx, req->target_idx,
 				      req->rank, PO_COMP_ST_DRAIN, 0 /* scm_size */,
-				      0 /* nvme_size */, 0 /* meta_size */);
+				      0 /* nvme_size */, 0 /* meta_size */, false);
 
 	d_rank_list_free(svc_ranks);
 
@@ -976,7 +976,7 @@ ds_mgmt_drpc_pool_reintegrate(Drpc__Call *drpc_req, Drpc__Response *drpc_resp)
 
 	rc = pool_change_target_state(req->id, svc_ranks, req->n_target_idx, req->target_idx,
 				      req->rank, PO_COMP_ST_UP, scm_bytes, nvme_bytes,
-				      req->tier_bytes[DAOS_MEDIA_SCM] /* meta_size */);
+				      req->tier_bytes[DAOS_MEDIA_SCM] /* meta_size */, false);
 
 	d_rank_list_free(svc_ranks);
 

src/mgmt/srv_internal.h

@@ -92,7 +92,7 @@ int
 ds_mgmt_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *svc_ranks,
 				 struct pool_target_addr_list *target_addrs,
 				 pool_comp_state_t state, size_t scm_size, size_t nvme_size,
-				 size_t meta_size);
+				 size_t meta_size, bool skip_rf_check);
 int
 ds_mgmt_pool_reintegrate(uuid_t pool_uuid, d_rank_list_t *svc_ranks, uint32_t reint_rank,
 			 struct pool_target_id_list *reint_list);

src/mgmt/srv_pool.c

@@ -337,7 +337,7 @@ int
 ds_mgmt_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *svc_ranks,
 				 struct pool_target_addr_list *target_addrs,
 				 pool_comp_state_t state, size_t scm_size, size_t nvme_size,
-				 size_t meta_size)
+				 size_t meta_size, bool skip_rf_check)
 {
 	int			rc;
 
@@ -364,7 +364,7 @@ ds_mgmt_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *svc_ranks,
 	}
 
 	rc = dsc_pool_svc_update_target_state(pool_uuid, svc_ranks, mgmt_ps_call_deadline(),
-					      target_addrs, state);
+					      target_addrs, state, skip_rf_check);
 
 	return rc;
 }

src/mgmt/tests/mocks.c

@@ -446,7 +446,7 @@ int
 ds_mgmt_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *svc_ranks,
 				 struct pool_target_addr_list *target_addrs,
 				 pool_comp_state_t state, size_t scm_size, size_t nvme_size,
-				 size_t meta_size)
+				 size_t meta_size, bool skip_rf_check)
 {
 	uuid_copy(ds_mgmt_target_update_uuid, pool_uuid);
 	return ds_mgmt_target_update_return;

src/pool/cli.c

@@ -1607,6 +1607,7 @@ pool_tgt_update_cp(tse_task_t *task, void *data)
 	int                              n_addrs;
 	bool                             free_tpriv = true;
 	int				 rc = task->dt_result;
+	uint32_t                         flags;
 
 	rc = rsvc_client_complete_rpc(&tpriv->state->client, &rpc->cr_ep, rc, out->pto_op.po_rc,
 				      &out->pto_op.po_hint);
@@ -1636,7 +1637,7 @@ pool_tgt_update_cp(tse_task_t *task, void *data)
 		DP_UUID(in->pti_op.pi_uuid), DP_UUID(in->pti_op.pi_hdl),
 		(int)out->pto_addr_list.ca_count);
 
-	pool_tgt_update_in_get_data(rpc, &addrs, &n_addrs);
+	pool_tgt_update_in_get_data(rpc, &addrs, &n_addrs, &flags);
 	D_FREE(addrs);
 
 	if (out->pto_addr_list.ca_arrays != NULL &&
@@ -1732,7 +1733,8 @@ dc_pool_update_internal(tse_task_t *task, daos_pool_update_t *args, int opc)
 		list.pta_addrs[i].pta_target = args->tgts->tl_tgts[i];
 	}
 
-	pool_tgt_update_in_set_data(rpc, list.pta_addrs, (size_t)list.pta_number);
+	pool_tgt_update_in_set_data(rpc, list.pta_addrs, (size_t)list.pta_number,
+				    POOL_TGT_UPDATE_SKIP_RF_CHECK);
 
 	crt_req_addref(rpc);