diff --git a/Makefile.am b/Makefile.am index d671d09d8..612f45d8d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -48,6 +48,7 @@ uadk_drivers_LTLIBRARIES=libhisi_sec.la libhisi_hpre.la libhisi_zip.la \ libisa_ce.la libisa_sve.la libhisi_dae.la libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \ + wd_sched.c wd_util.c \ v1/wd.c v1/wd.h v1/wd_adapter.c v1/wd_adapter.h \ v1/wd_rng.c v1/wd_rng.h \ v1/wd_rsa.c v1/wd_rsa.h \ @@ -64,7 +65,8 @@ libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \ v1/drv/hisi_zip_udrv.c v1/drv/hisi_zip_udrv.h \ v1/drv/hisi_hpre_udrv.c v1/drv/hisi_hpre_udrv.h \ v1/drv/hisi_sec_udrv.c v1/drv/hisi_sec_udrv.h \ - v1/drv/hisi_rng_udrv.c v1/drv/hisi_rng_udrv.h + v1/drv/hisi_rng_udrv.c v1/drv/hisi_rng_udrv.h \ + adapter.c libwd_dae_la_SOURCES=wd_dae.h wd_agg.h wd_agg_drv.h wd_agg.c \ wd_util.c wd_util.h wd_sched.c wd_sched.h wd.c wd.h diff --git a/adapter.c b/adapter.c new file mode 100644 index 000000000..58ec63587 --- /dev/null +++ b/adapter.c @@ -0,0 +1,197 @@ + +/* SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright 2024-2025 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2024-2025 Linaro ltd. + */ + +#include "adapter.h" + +#define CONFIG_FILE_ENV "UADK_CONF" +#define DRIVER_NAME_KEY "driver_name" +#define MODE_KEY "mode" +#define LOOP_KEY "looptime" +#define THRESHOLD_KEY "threshold" +#define WORKERS_NB 8 + +static int read_value_int(char *conf, const char *key) +{ + FILE *fp = fopen(conf, "r"); + char line[1024]; + int ret = 0; + + if (fp == NULL) + return 0; + + while (fgets(line, sizeof(line), fp) != NULL) { + char *key_value = strtok(line, "="); + if (key_value && strcmp(key_value, key) == 0) { + char *value = strtok(NULL, "\n"); + + if (value) { + ret = atoi(value); + goto exit; + } + } + } +exit: + fclose(fp); + return ret; +} + +static void read_config_entries(char *conf, struct uadk_adapter *adapter, char *alg_name) +{ + struct uadk_adapter_worker *worker; + FILE *fp = fopen(conf, "r"); + struct wd_alg_driver *drv; + char *drv_name = NULL; + char line[1024]; + int i = 0; + + if (fp == NULL) + return; + + while (fgets(line, sizeof(line), fp) != NULL) { + char *key_value = strtok(line, "="); + + if (key_value && strcmp(key_value, DRIVER_NAME_KEY) == 0) + drv_name = strdup(strtok(NULL, "\n")); + + if ((drv_name != NULL) && (alg_name != NULL)) { + drv = wd_find_drv(drv_name, alg_name, 0); + if (!drv) + continue; + + worker = &adapter->workers[i]; + worker->driver = drv; + worker->idx = i; + pthread_mutex_init(&worker->mutex, NULL); + adapter->workers_nb++; + if (drv_name) { + free(drv_name); + drv_name = NULL; + } + + if (++i >= UADK_MAX_NB_WORKERS) + break; + } + } + + if (drv_name) + free(drv_name); + + fclose(fp); +} + +int uadk_adapter_add_workers(struct uadk_adapter *adapter, char *alg) +{ + char *conf = getenv(CONFIG_FILE_ENV); + struct uadk_adapter_worker workers[WORKERS_NB]; + struct uadk_adapter_worker worker; + struct wd_alg_driver *drv; + int idx = 0, i, j; + + adapter->looptime = UADK_WORKER_LOOPTIME; + adapter->threshold = UADK_PKT_THRESHOLD; + + if (conf != NULL) { + int looptime = 0; + int threshold = 0; + + /* if env UADK_CONF exist, parse config first */ + adapter->mode = read_value_int(conf, MODE_KEY); + looptime = read_value_int(conf, LOOP_KEY); + if (looptime != 0) + adapter->looptime = looptime; + + threshold = read_value_int(conf, THRESHOLD_KEY); + if (threshold != 0) + adapter->threshold = threshold; + + read_config_entries(conf, adapter, alg); + if (adapter->workers_nb != 0) + return 0; + } + + /* Then parse all system drivers to workers */ + do { + drv = wd_find_drv(NULL, alg, idx); + if (!drv) + break; + + workers[idx++].driver = drv; + + if (idx >= WORKERS_NB) + break; + } while (drv); + + /* Sorted as priority */ + for (i = 0; i < idx; i++) { + for (j = i; j < idx; j++) { + if (workers[i].driver->priority < + workers[j].driver->priority) { + worker.driver = workers[i].driver; + workers[i].driver = workers[j].driver; + workers[j].driver = worker.driver; + } + } + } + + for (i = 0; i < idx; i++) { + adapter->workers[i].driver = workers[i].driver; + adapter->workers[i].idx = i; + adapter->workers_nb++; + pthread_mutex_init(&adapter->workers[i].mutex, NULL); + + if (adapter->workers_nb >= UADK_MAX_NB_WORKERS) + break; + } + + return (adapter->workers_nb == 0); +} + +struct uadk_adapter_worker *uadk_adapter_choose_worker( + struct uadk_adapter *adapter, + enum alg_task_type type) +{ + struct uadk_adapter_worker *worker; + + /* use worker[0] for simplicity now */ + worker = &adapter->workers[0]; + worker->valid = true; + + return worker; +} + +struct uadk_adapter_worker *uadk_adapter_switch_worker( + struct uadk_adapter *adapter, + struct uadk_adapter_worker *worker, + int para) +{ + struct uadk_adapter_worker *new_worker; + int idx = worker->idx; + + if (adapter->workers_nb == 1) + return worker; + + if (para) { + idx += 1; + } else { + if (idx == 0) + idx = adapter->workers_nb - 1; + else + idx -= 1; + } + + new_worker = &adapter->workers[idx]; + new_worker->valid = true; + + return new_worker; +} + +void uadk_adapter_free(struct uadk_adapter *adapter) +{ + for (int i = 0; i < adapter->workers_nb; i++) + pthread_mutex_destroy(&adapter->workers[i].mutex); + free(adapter); +} diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index 07d875488..d30da966a 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -1003,9 +1003,10 @@ static void get_ctx_buf(struct hisi_zip_sqe *sqe, } } -static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, - struct wd_comp_msg *msg) +static int parse_zip_sqe(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_zip_sqe *sqe, struct wd_comp_msg *msg) { + struct hisi_zip_ctx *priv = (struct hisi_zip_ctx *)drv->priv; __u32 buf_type = (sqe->dw9 & HZ_BUF_TYPE_MASK) >> BUF_TYPE_SHIFT; __u16 ctx_st = sqe->ctx_dw0 & HZ_CTX_ST_MASK; __u16 lstblk = sqe->dw3 & HZ_LSTBLK_MASK; @@ -1030,7 +1031,7 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, recv_msg->tag = tag; if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - recv_msg = wd_comp_get_msg(qp->q_info.idx, tag); + recv_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, tag); if (unlikely(!recv_msg)) { WD_ERR("failed to get send msg! idx = %u, tag = %u!\n", qp->q_info.idx, tag); @@ -1091,7 +1092,7 @@ static int hisi_zip_comp_recv(struct wd_alg_driver *drv, handle_t ctx, void *com if (unlikely(ret < 0)) return ret; - return parse_zip_sqe(qp, &sqe, recv_msg); + return parse_zip_sqe(drv, qp, &sqe, recv_msg); } #define GEN_ZIP_ALG_DRIVER(zip_alg_name) \ diff --git a/drv/hisi_dae.c b/drv/hisi_dae.c index 864d21bae..aa5da9072 100644 --- a/drv/hisi_dae.c +++ b/drv/hisi_dae.c @@ -653,6 +653,7 @@ static void fill_hashagg_msg_task_err(struct dae_sqe *sqe, struct wd_agg_msg *ms static int hashagg_recv(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_msg) { + struct hisi_dae_ctx *priv = (struct hisi_dae_ctx *)drv->priv; handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct dae_extend_addr *ext_addr = qp->priv; @@ -673,7 +674,7 @@ static int hashagg_recv(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_m msg->tag = sqe.low_tag; if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_agg_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, msg->tag); if (!temp_msg) { msg->result = WD_AGG_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c index 313cdcc8f..e9d077573 100644 --- a/drv/hisi_hpre.c +++ b/drv/hisi_hpre.c @@ -659,6 +659,7 @@ static void hpre_result_check(struct hisi_hpre_sqe *hw_msg, static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) { + struct hisi_hpre_ctx *priv = (struct hisi_hpre_ctx *)drv->priv; handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct hisi_hpre_sqe hw_msg = {0}; @@ -677,7 +678,7 @@ static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) msg->tag = LW_U16(hw_msg.low_tag); if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_rsa_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, msg->tag); if (!temp_msg) { WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", qp->q_info.idx, msg->tag); @@ -802,6 +803,7 @@ static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) { + struct hisi_hpre_ctx *priv = (struct hisi_hpre_ctx *)drv->priv; handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct wd_dh_msg *msg = dh_msg; @@ -820,7 +822,7 @@ static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) msg->tag = LW_U16(hw_msg.low_tag); if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_dh_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, msg->tag); if (!temp_msg) { WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", qp->q_info.idx, msg->tag); @@ -2037,7 +2039,6 @@ static int ecc_out_transfer(struct wd_ecc_msg *msg, return ret; } - static __u32 get_hash_bytes(__u8 type) { __u32 val = 0; @@ -2303,15 +2304,16 @@ static int sm2_convert_dec_out(struct wd_ecc_msg *src, return ret; } -static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int ecc_sqe_parse(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { + struct hisi_hpre_ctx *priv = (struct hisi_hpre_ctx *)drv->priv; struct wd_ecc_msg *temp_msg; int ret; msg->tag = LW_U16(hw_msg->low_tag); if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { - temp_msg = wd_ecc_get_msg(qp->q_info.idx, msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, msg->tag); if (!temp_msg) { WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", qp->q_info.idx, msg->tag); @@ -2342,7 +2344,7 @@ static int ecc_sqe_parse(struct hisi_qp *qp, struct wd_ecc_msg *msg, return ret; } -static int parse_second_sqe(handle_t h_qp, +static int parse_second_sqe(struct wd_alg_driver *drv, handle_t h_qp, struct wd_ecc_msg *msg, struct wd_ecc_msg **second) { @@ -2371,15 +2373,15 @@ static int parse_second_sqe(handle_t h_qp, hsz = (hw_msg.task_len1 + 1) * BYTE_BITS; dst = *(struct wd_ecc_msg **)((uintptr_t)data + hsz * ECDH_OUT_PARAM_NUM); - ret = ecc_sqe_parse((struct hisi_qp *)h_qp, dst, &hw_msg); + ret = ecc_sqe_parse(drv, (struct hisi_qp *)h_qp, dst, &hw_msg); msg->result = dst->result; *second = dst; return ret; } -static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int sm2_enc_parse(struct wd_alg_driver *drv, handle_t h_qp, + struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { __u16 tag = LW_U16(hw_msg->low_tag); struct wd_ecc_msg *second = NULL; @@ -2397,14 +2399,14 @@ static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg, memcpy(&src, first + 1, sizeof(src)); /* parse first sqe */ - ret = ecc_sqe_parse((struct hisi_qp *)h_qp, first, hw_msg); + ret = ecc_sqe_parse(drv, (struct hisi_qp *)h_qp, first, hw_msg); if (ret) { WD_ERR("failed to parse first BD, ret = %d!\n", ret); goto free_first; } /* parse second sqe */ - ret = parse_second_sqe(h_qp, msg, &second); + ret = parse_second_sqe(drv, h_qp, msg, &second); if (unlikely(ret)) { WD_ERR("failed to parse second BD, ret = %d!\n", ret); goto free_first; @@ -2424,8 +2426,8 @@ static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg, return ret; } -static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg, - struct hisi_hpre_sqe *hw_msg) +static int sm2_dec_parse(struct wd_alg_driver *drv, handle_t ctx, + struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg) { __u16 tag = LW_U16(hw_msg->low_tag); struct wd_ecc_msg *dst; @@ -2441,7 +2443,7 @@ static int sm2_dec_parse(handle_t ctx, struct wd_ecc_msg *msg, memcpy(&src, dst + 1, sizeof(src)); /* parse first sqe */ - ret = ecc_sqe_parse((struct hisi_qp *)ctx, dst, hw_msg); + ret = ecc_sqe_parse(drv, (struct hisi_qp *)ctx, dst, hw_msg); if (ret) { WD_ERR("failed to parse decode BD, ret = %d!\n", ret); goto fail; @@ -2480,12 +2482,12 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) if (hw_msg.alg == HPRE_ALG_ECDH_MULTIPLY && hw_msg.sm2_mlen == HPRE_SM2_ENC) - return sm2_enc_parse(h_qp, msg, &hw_msg); + return sm2_enc_parse(drv, h_qp, msg, &hw_msg); else if (hw_msg.alg == HPRE_ALG_ECDH_MULTIPLY && hw_msg.sm2_mlen == HPRE_SM2_DEC) - return sm2_dec_parse(h_qp, msg, &hw_msg); + return sm2_dec_parse(drv, h_qp, msg, &hw_msg); - return ecc_sqe_parse((struct hisi_qp *)h_qp, msg, &hw_msg); + return ecc_sqe_parse(drv, (struct hisi_qp *)h_qp, msg, &hw_msg); } static int hpre_get_usage(void *param) diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 10db124b7..d422a4a3c 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -920,9 +920,10 @@ static void fill_cipher_bd2_addr(struct wd_cipher_msg *msg, sqe->type2.c_key_addr = (__u64)(uintptr_t)msg->key; } -static void parse_cipher_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, - struct wd_cipher_msg *recv_msg) +static void parse_cipher_bd2(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe *sqe, struct wd_cipher_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_cipher_msg *temp_msg; __u16 done; __u32 tag; @@ -944,7 +945,7 @@ static void parse_cipher_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->type2.data_dst_addr; - temp_msg = wd_cipher_get_msg(qp->q_info.idx, tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -1218,7 +1219,7 @@ static int hisi_sec_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *w if (ret) return ret; - parse_cipher_bd2((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_cipher_bd2(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, @@ -1405,9 +1406,10 @@ static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void return 0; } -static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, - struct wd_cipher_msg *recv_msg) +static void parse_cipher_bd3(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe3 *sqe, struct wd_cipher_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_cipher_msg *temp_msg; __u16 done; __u32 tag; @@ -1429,7 +1431,7 @@ static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->data_dst_addr; - temp_msg = wd_cipher_get_msg(qp->q_info.idx, tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -1466,7 +1468,7 @@ static int hisi_sec_cipher_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void if (ret) return ret; - parse_cipher_bd3((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_cipher_bd3(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, @@ -1576,9 +1578,10 @@ static int fill_digest_long_hash(handle_t h_qp, struct wd_digest_msg *msg, return 0; } -static void parse_digest_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, - struct wd_digest_msg *recv_msg) +static void parse_digest_bd2(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe *sqe, struct wd_digest_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_digest_msg *temp_msg; __u16 done; @@ -1597,7 +1600,7 @@ static void parse_digest_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, recv_msg->alg_type = WD_DIGEST; recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; - temp_msg = wd_digest_get_msg(qp->q_info.idx, recv_msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -1787,7 +1790,7 @@ static int hisi_sec_digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *w if (ret) return ret; - parse_digest_bd2((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_digest_bd2(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, @@ -2010,9 +2013,10 @@ static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void return ret; } -static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, - struct wd_digest_msg *recv_msg) +static void parse_digest_bd3(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe3 *sqe, struct wd_digest_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_digest_msg *temp_msg; __u16 done; @@ -2031,7 +2035,7 @@ static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, recv_msg->alg_type = WD_DIGEST; recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; - temp_msg = wd_digest_get_msg(qp->q_info.idx, recv_msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -2063,7 +2067,7 @@ static int hisi_sec_digest_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void if (ret) return ret; - parse_digest_bd3((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_digest_bd3(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, @@ -2618,9 +2622,10 @@ static void update_stream_counter(struct wd_aead_msg *recv_msg) } } -static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, - struct wd_aead_msg *recv_msg) +static void parse_aead_bd2(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe *sqe, struct wd_aead_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_aead_msg *temp_msg; __u16 done, icv; @@ -2642,7 +2647,7 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->type2.data_dst_addr; - temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -2697,7 +2702,7 @@ static int hisi_sec_aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_ if (ret) return ret; - parse_aead_bd2((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_aead_bd2(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, @@ -3006,9 +3011,10 @@ static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void * return ret; } -static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, - struct wd_aead_msg *recv_msg) +static void parse_aead_bd3(struct wd_alg_driver *drv, struct hisi_qp *qp, + struct hisi_sec_sqe3 *sqe, struct wd_aead_msg *recv_msg) { + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; struct wd_aead_msg *temp_msg; __u16 done, icv; @@ -3031,7 +3037,7 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; recv_msg->out = (__u8 *)(uintptr_t)sqe->data_dst_addr; - temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); + temp_msg = wd_find_msg_in_pool(priv->config.pool, qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", @@ -3068,7 +3074,7 @@ static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void * if (ret) return ret; - parse_aead_bd3((struct hisi_qp *)h_qp, &sqe, recv_msg); + parse_aead_bd3(drv, (struct hisi_qp *)h_qp, &sqe, recv_msg); if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c index 54c2a9e72..88532297a 100644 --- a/drv/isa_ce_sm3.c +++ b/drv/isa_ce_sm3.c @@ -32,6 +32,7 @@ static struct wd_alg_driver sm3_ce_alg_driver = { .drv_name = "isa_ce_sm3", .alg_name = "sm3", .calc_type = UADK_ALG_CE_INSTR, + .mode = UADK_DRV_SYNCONLY, .priority = 200, .queue_num = 1, .op_type_num = 1, diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c index 5e448fa0f..60a09f296 100644 --- a/drv/isa_ce_sm4.c +++ b/drv/isa_ce_sm4.c @@ -419,6 +419,7 @@ static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) .drv_name = "isa_ce_sm4",\ .alg_name = (ce_alg_name),\ .calc_type = UADK_ALG_CE_INSTR,\ + .mode = UADK_DRV_SYNCONLY,\ .priority = 200,\ .op_type_num = 1,\ .fallback = 0,\ diff --git a/include/adapter.h b/include/adapter.h new file mode 100644 index 000000000..be77f4337 --- /dev/null +++ b/include/adapter.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright 2024-2025 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2024-2025 Linaro ltd. + */ + +#ifndef __ADAPTER_H +#define __ADAPTER_H + +#include "wd_alg.h" +#include "wd_util.h" + +#define UADK_MAX_NB_WORKERS (2) +#define UADK_WORKER_LOOPTIME (10) +#define UADK_PKT_THRESHOLD 8192 + +enum uadk_adapter_mode { + UADK_ADAPT_MODE_PRIMARY, + UADK_ADAPT_MODE_ROUNDROBIN, + UADK_ADAPT_MODE_THRESHOLD, +}; + +struct uadk_adapter_worker { + struct wd_alg_driver *driver; + struct wd_ctx_config *ctx_config; + struct wd_sched *sched; + struct wd_ctx_config_internal config; + struct wd_async_msg_pool pool; + bool valid; + int idx; + __u64 async_recv; + pthread_mutex_t mutex; +}; + +struct uadk_adapter { + unsigned int workers_nb; + unsigned int looptime; + unsigned int threshold; + enum uadk_adapter_mode mode; + struct uadk_adapter_worker workers[UADK_MAX_NB_WORKERS]; +}; + +int uadk_adapter_add_workers(struct uadk_adapter *adapter, char *alg); + +struct uadk_adapter_worker *uadk_adapter_choose_worker( + struct uadk_adapter *adapter, + enum alg_task_type type +); + +struct uadk_adapter_worker *uadk_adapter_switch_worker( + struct uadk_adapter *adapter, + struct uadk_adapter_worker *worker, + int para +); + +void uadk_adapter_free(struct uadk_adapter *adapter); +#endif diff --git a/include/drv/wd_aead_drv.h b/include/drv/wd_aead_drv.h index a9c0e7c09..18696e238 100644 --- a/include/drv/wd_aead_drv.h +++ b/include/drv/wd_aead_drv.h @@ -70,8 +70,6 @@ struct wd_aead_msg { enum wd_aead_msg_state msg_state; }; -struct wd_aead_msg *wd_aead_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_agg_drv.h b/include/drv/wd_agg_drv.h index 3d26eeb3c..ea7134399 100644 --- a/include/drv/wd_agg_drv.h +++ b/include/drv/wd_agg_drv.h @@ -48,8 +48,6 @@ struct wd_agg_ops { int (*hash_table_init)(struct wd_dae_hash_table *hash_table, void *priv); }; -struct wd_agg_msg *wd_agg_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_cipher_drv.h b/include/drv/wd_cipher_drv.h index c6d8ddfbc..15529e190 100644 --- a/include/drv/wd_cipher_drv.h +++ b/include/drv/wd_cipher_drv.h @@ -50,8 +50,6 @@ struct wd_cipher_msg { __u8 *out; }; -struct wd_cipher_msg *wd_cipher_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_comp_drv.h b/include/drv/wd_comp_drv.h index 213cf2d59..3628ae690 100644 --- a/include/drv/wd_comp_drv.h +++ b/include/drv/wd_comp_drv.h @@ -55,8 +55,6 @@ struct wd_comp_msg { __u32 tag; }; -struct wd_comp_msg *wd_comp_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_dh_drv.h b/include/drv/wd_dh_drv.h index d205dc45b..de8b06832 100644 --- a/include/drv/wd_dh_drv.h +++ b/include/drv/wd_dh_drv.h @@ -24,8 +24,6 @@ struct wd_dh_msg { __u8 result; /* Data format, denoted by WD error code */ }; -struct wd_dh_msg *wd_dh_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_digest_drv.h b/include/drv/wd_digest_drv.h index a55ef5bb1..4cea1cf78 100644 --- a/include/drv/wd_digest_drv.h +++ b/include/drv/wd_digest_drv.h @@ -80,8 +80,6 @@ static inline enum hash_block_type get_hash_block_type(struct wd_digest_msg *msg return HASH_SINGLE_BLOCK; } -struct wd_digest_msg *wd_digest_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_ecc_drv.h b/include/drv/wd_ecc_drv.h index f5805cdc0..e9624ef9e 100644 --- a/include/drv/wd_ecc_drv.h +++ b/include/drv/wd_ecc_drv.h @@ -175,8 +175,6 @@ struct wd_ecc_out { char data[]; }; -struct wd_ecc_msg *wd_ecc_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/drv/wd_rsa_drv.h b/include/drv/wd_rsa_drv.h index d231ecfb8..77406fc68 100644 --- a/include/drv/wd_rsa_drv.h +++ b/include/drv/wd_rsa_drv.h @@ -49,8 +49,6 @@ struct wd_rsa_msg { __u8 *key; /* Input key VA pointer, should be DMA buffer */ }; -struct wd_rsa_msg *wd_rsa_get_msg(__u32 idx, __u32 tag); - #ifdef __cplusplus } #endif diff --git a/include/wd_aead.h b/include/wd_aead.h index 01f6980f7..6c1b37f21 100644 --- a/include/wd_aead.h +++ b/include/wd_aead.h @@ -207,6 +207,7 @@ int wd_aead_get_maxauthsize(handle_t h_sess); * @count: how many respondences this poll has to get. */ int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_aead_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); /** * wd_aead_poll() Poll finished request. diff --git a/include/wd_alg.h b/include/wd_alg.h index aba855d65..01cb10486 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -69,6 +69,11 @@ enum alg_dev_type { UADK_ALG_HW = 0x3 }; +enum alg_drv_mode { + UADK_DRV_NORMAL = 0x0, + UADK_DRV_SYNCONLY = 0x1, +}; + /* * @drv_name: name of the current device driver * @alg_name: name of the algorithm supported by the driver @@ -102,6 +107,7 @@ struct wd_alg_driver { const char *alg_name; int priority; int calc_type; + int mode; int queue_num; int op_type_num; void *priv; @@ -193,6 +199,7 @@ bool wd_drv_alg_support(const char *alg_name, */ void wd_enable_drv(struct wd_alg_driver *drv); void wd_disable_drv(struct wd_alg_driver *drv); +struct wd_alg_driver *wd_find_drv(char *drv_name, char *alg_name, int idx); struct wd_alg_list *wd_get_alg_head(void); diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index fd7742698..00370535c 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -151,6 +151,7 @@ struct wd_ctx_config_internal { void *priv; bool epoll_en; unsigned long *msg_cnt; + struct wd_async_msg_pool *pool; }; /* @@ -170,16 +171,17 @@ struct wd_ctx_config_internal { struct wd_sched { const char *name; int sched_policy; + struct uadk_adapter_worker *worker; handle_t (*sched_init)(handle_t h_sched_ctx, void *sched_param); __u32 (*pick_next_ctx)(handle_t h_sched_ctx, void *sched_key, const int sched_mode); - int (*poll_policy)(handle_t h_sched_ctx, __u32 expect, __u32 *count); + int (*poll_policy)(struct wd_sched *sched, __u32 expect, __u32 *count); handle_t h_sched_ctx; }; -typedef int (*wd_alg_init)(struct wd_ctx_config *config, struct wd_sched *sched); -typedef int (*wd_alg_poll_ctx)(__u32 idx, __u32 expt, __u32 *count); +typedef int (*wd_alg_init)(struct uadk_adapter_worker *worker, struct wd_sched *sched); +typedef int (*wd_alg_poll_ctx)(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); struct wd_datalist { void *data; diff --git a/include/wd_cipher.h b/include/wd_cipher.h index d54f7fef2..94e91eaaa 100644 --- a/include/wd_cipher.h +++ b/include/wd_cipher.h @@ -171,6 +171,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req); * @count: how many respondences this poll has to get. */ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_cipher_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); /** * wd_cipher_poll() Poll finished request. * this function will call poll_policy function which is registered to wd cipher diff --git a/include/wd_comp.h b/include/wd_comp.h index 45994ff69..5fa2b8423 100644 --- a/include/wd_comp.h +++ b/include/wd_comp.h @@ -199,6 +199,7 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req); * specific ctx, this function should be used. */ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_comp_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); int wd_comp_poll(__u32 expt, __u32 *count); diff --git a/include/wd_dh.h b/include/wd_dh.h index afc2f7c08..b0f2dd3da 100644 --- a/include/wd_dh.h +++ b/include/wd_dh.h @@ -57,6 +57,7 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup); void wd_dh_free_sess(handle_t sess); int wd_do_dh_async(handle_t sess, struct wd_dh_req *req); int wd_do_dh_sync(handle_t sess, struct wd_dh_req *req); +int wd_dh_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count); int wd_dh_poll(__u32 expt, __u32 *count); int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched); diff --git a/include/wd_digest.h b/include/wd_digest.h index f0916c362..69ffdf576 100644 --- a/include/wd_digest.h +++ b/include/wd_digest.h @@ -230,6 +230,7 @@ int wd_digest_set_key(handle_t h_sess, const __u8 *key, __u32 key_len); * @count: recv poll nums. */ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_digest_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); /** * wd_digest_poll() - Poll operation for asynchronous operation. diff --git a/include/wd_ecc.h b/include/wd_ecc.h index 6f670e2d0..e981920c7 100644 --- a/include/wd_ecc.h +++ b/include/wd_ecc.h @@ -509,6 +509,7 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req); * specific ctx, this function should be used. */ int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_ecc_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); /** * wd_ecc_env_init() - Init ctx and schedule resources according to wd ecc diff --git a/include/wd_rsa.h b/include/wd_rsa.h index 733d0b74f..b1092c037 100644 --- a/include/wd_rsa.h +++ b/include/wd_rsa.h @@ -202,6 +202,7 @@ __u32 wd_rsa_poll(void); * specific ctx, this function should be used. */ int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +int wd_rsa_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); /** * wd_rsa_env_init() - Init ctx and schedule resources according to wd rsa diff --git a/include/wd_sched.h b/include/wd_sched.h index be541c6dd..553f63d61 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -34,7 +34,8 @@ struct sched_params { __u32 end; }; -typedef int (*user_poll_func)(__u32 pos, __u32 expect, __u32 *count); +typedef int (*user_poll_func)(struct wd_sched *sched, __u32 pos, + __u32 expect, __u32 *count); /* * wd_sched_rr_instance - Instante the schedule min region. diff --git a/include/wd_util.h b/include/wd_util.h index dc1e41fa4..23961c25e 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -143,21 +143,6 @@ struct wd_init_attrs { int wd_init_ctx_config(struct wd_ctx_config_internal *in, struct wd_ctx_config *cfg); -/* - * wd_init_sched() - Init internal scheduler configuration. - * @in: Scheduler configuration in global setting. - * @from: Scheduler configuration input by user. - * - * Return 0 if successful or less than 0 otherwise. - */ -int wd_init_sched(struct wd_sched *in, struct wd_sched *from); - -/* - * wd_clear_sched() - Clear internal scheduler configuration. - * @in: Scheduler configuration in global setting. - */ -void wd_clear_sched(struct wd_sched *in); - /* * wd_clear_ctx_config() - Clear internal ctx configuration. * @in: ctx configuration in global setting. @@ -461,12 +446,14 @@ void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params); /** * wd_alg_attrs_init() - Request the ctxs and initialize the sched_domain * with the given devices list, ctxs number and numa mask. + * @worker: uadk_adapter_worker. * @attrs: the algorithm initialization parameters. * * Return device if succeed and other error number if fail. */ -int wd_alg_attrs_init(struct wd_init_attrs *attrs); -void wd_alg_attrs_uninit(struct wd_init_attrs *attrs); +int wd_alg_attrs_init(struct uadk_adapter_worker *worker, + struct wd_init_attrs *attrs); +void wd_alg_attrs_uninit(struct uadk_adapter_worker *worker); /** * wd_alg_drv_bind() - Request the ctxs and initialize the sched_domain diff --git a/libwd.map b/libwd.map index 5522ec07f..f9d55207b 100644 --- a/libwd.map +++ b/libwd.map @@ -45,9 +45,20 @@ global: wd_alg_driver_unregister; wd_request_drv; wd_release_drv; + wd_find_drv; wd_drv_alg_support; wd_enable_drv; wd_disable_drv; wd_get_alg_head; + + wd_sched_rr_instance; + wd_sched_rr_alloc; + wd_sched_rr_release; + wd_find_msg_in_pool; + + uadk_adapter_add_workers; + uadk_adapter_choose_worker; + uadk_adapter_switch_worker; + uadk_adapter_free; local: *; }; diff --git a/libwd_comp.map b/libwd_comp.map index 6b1f8c25e..dba9536d3 100644 --- a/libwd_comp.map +++ b/libwd_comp.map @@ -11,6 +11,7 @@ global: wd_do_comp_strm; wd_do_comp_async; wd_comp_poll_ctx; + wd_comp_poll_ctx_; wd_comp_poll; wd_do_comp_sync2; wd_comp_env_init; @@ -20,13 +21,8 @@ global: wd_comp_get_env_param; wd_comp_set_driver; wd_comp_get_driver; - wd_comp_get_msg; wd_comp_reset_sess; - wd_sched_rr_instance; - wd_sched_rr_alloc; - wd_sched_rr_release; - wd_deflate_init; wd_deflate; wd_deflate_reset; @@ -36,6 +32,5 @@ global: wd_inflate; wd_inflate_reset; wd_inflate_end; - local: *; }; diff --git a/libwd_crypto.map b/libwd_crypto.map index e8555c982..450b2bd3e 100644 --- a/libwd_crypto.map +++ b/libwd_crypto.map @@ -11,6 +11,7 @@ global: wd_do_cipher_sync; wd_do_cipher_async; wd_cipher_poll_ctx; + wd_cipher_poll_ctx_; wd_cipher_poll; wd_cipher_env_init; wd_cipher_env_uninit; @@ -19,7 +20,6 @@ global: wd_cipher_get_env_param; wd_cipher_set_driver; wd_cipher_get_driver; - wd_cipher_get_msg; wd_aead_init; wd_aead_uninit; @@ -36,6 +36,7 @@ global: wd_aead_get_authsize; wd_aead_get_maxauthsize; wd_aead_poll_ctx; + wd_aead_poll_ctx_; wd_aead_poll; wd_aead_env_init; wd_aead_env_uninit; @@ -44,7 +45,6 @@ global: wd_aead_get_env_param; wd_aead_set_driver; wd_aead_get_driver; - wd_aead_get_msg; wd_digest_init; wd_digest_uninit; @@ -57,6 +57,7 @@ global: wd_do_digest_async; wd_digest_set_key; wd_digest_poll_ctx; + wd_digest_poll_ctx_; wd_digest_poll; wd_digest_env_init; wd_digest_env_uninit; @@ -65,7 +66,6 @@ global: wd_digest_get_env_param; wd_digest_set_driver; wd_digest_get_driver; - wd_digest_get_msg; wd_rsa_is_crt; wd_rsa_get_key_bits; @@ -100,6 +100,7 @@ global: wd_do_rsa_sync; wd_do_rsa_async; wd_rsa_poll_ctx; + wd_rsa_poll_ctx_; wd_rsa_env_init; wd_rsa_env_uninit; wd_rsa_ctx_num_init; @@ -107,7 +108,6 @@ global: wd_rsa_get_env_param; wd_rsa_set_driver; wd_rsa_get_driver; - wd_rsa_get_msg; wd_dh_get_mode; wd_dh_key_bits; @@ -118,6 +118,7 @@ global: wd_do_dh_async; wd_do_dh_sync; wd_dh_poll_ctx; + wd_dh_poll_ctx_; wd_dh_poll; wd_dh_init; wd_dh_uninit; @@ -131,7 +132,6 @@ global: wd_dh_get_env_param; wd_dh_set_driver; wd_dh_get_driver; - wd_dh_get_msg; wd_ecc_get_key_bits; wd_ecc_get_key; @@ -148,7 +148,6 @@ global: wd_ecxdh_get_out_params; wd_ecc_set_driver; wd_ecc_get_driver; - wd_ecc_get_msg; wd_sm2_new_sign_in; wd_sm2_new_verf_in; @@ -178,14 +177,11 @@ global: wd_do_ecc_sync; wd_do_ecc_async; wd_ecc_poll_ctx; + wd_ecc_poll_ctx_; wd_ecc_env_init; wd_ecc_env_uninit; wd_ecc_ctx_num_init; wd_ecc_ctx_num_uninit; wd_ecc_get_env_param; - - wd_sched_rr_instance; - wd_sched_rr_alloc; - wd_sched_rr_release; local: *; }; diff --git a/libwd_dae.map b/libwd_dae.map index 4c51b856f..1243c0616 100644 --- a/libwd_dae.map +++ b/libwd_dae.map @@ -11,11 +11,6 @@ global: wd_agg_get_output_sync; wd_agg_get_output_async; wd_agg_rehash_sync; - wd_agg_get_msg; wd_agg_poll; - - wd_sched_rr_instance; - wd_sched_rr_alloc; - wd_sched_rr_release; local: *; }; diff --git a/sample/uadk_comp.c b/sample/uadk_comp.c index 908c7bcb3..e3243631c 100644 --- a/sample/uadk_comp.c +++ b/sample/uadk_comp.c @@ -127,7 +127,7 @@ static struct uacce_dev_list* get_dev_list(char *alg_name) return NULL; } -static int lib_poll_func(__u32 pos, __u32 expect, __u32 *count) +static int lib_poll_func(struct wd_sched *sched, __u32 pos, __u32 expect, __u32 *count) { int ret; diff --git a/test/hisi_hpre_test/test_hisi_hpre.c b/test/hisi_hpre_test/test_hisi_hpre.c index fbefef87c..1c7351432 100644 --- a/test/hisi_hpre_test/test_hisi_hpre.c +++ b/test/hisi_hpre_test/test_hisi_hpre.c @@ -1789,7 +1789,7 @@ static __u32 hpre_pick_next_ctx(handle_t sched_ctx, return last_ctx; } -int rsa_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +int rsa_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { int ret; struct wd_ctx *ctxs; @@ -1813,7 +1813,7 @@ int rsa_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) return 0; } -static int dh_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int dh_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { int ret; struct wd_ctx *ctxs; @@ -1836,7 +1836,7 @@ static int dh_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) return 0; } -static int ecc_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int ecc_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { int ret; struct wd_ctx *ctxs; diff --git a/test/hisi_zip_test/test_lib.c b/test/hisi_zip_test/test_lib.c index 384be4964..2ab9807e4 100644 --- a/test/hisi_zip_test/test_lib.c +++ b/test/hisi_zip_test/test_lib.c @@ -590,7 +590,7 @@ void *send_thread_func(void *arg) return NULL; } -int lib_poll_func(__u32 pos, __u32 expect, __u32 *count) +int lib_poll_func(struct wd_sched *sched, __u32 pos, __u32 expect, __u32 *count) { int ret; diff --git a/test/hisi_zip_test/test_lib.h b/test/hisi_zip_test/test_lib.h index 70d2c9d69..994ae888f 100644 --- a/test/hisi_zip_test/test_lib.h +++ b/test/hisi_zip_test/test_lib.h @@ -267,7 +267,7 @@ int hizip_verify_random_output(struct test_options *opts, void *mmap_alloc(size_t len); int mmap_free(void *addr, size_t len); -int lib_poll_func(__u32 pos, __u32 expect, __u32 *count); +int lib_poll_func(struct wd_sched *sched, __u32 pos, __u32 expect, __u32 *count); typedef int (*check_output_fn)(unsigned char *buf, unsigned int size, void *opaque); /* for block interface */ diff --git a/test/test.sh b/test/test.sh new file mode 100755 index 000000000..faab1a0ec --- /dev/null +++ b/test/test.sh @@ -0,0 +1,31 @@ +set -x + +echo "Testing sm3 /sm4, sync & async" +echo "Testing hw, hw+ce, ce" +echo "Testing hw, hw+sve, sve" + +default_size=8192 +values=(1 2 4 8 16 32) + +if [ -n "$1" ]; then + size=$1 +else + size=$default_size +fi + +for x in "${values[@]}" +do + echo $x + #sm4 sync + numactl --cpubind=0 --membind=0 uadk_tool benchmark --alg sm4-128-ecb --mode sva --opt 0 --sync --pktlen $size --seconds 10 --multi 1 --thread $x --ctxnum $x --init2 --prefetch + + #sm4 async + numactl --cpubind=0 --membind=0 uadk_tool benchmark --alg sm4-128-ecb --mode sva --opt 0 --async --pktlen $size --seconds 10 --multi 1 --thread $x --ctxnum $x --init2 --prefetch + + + #sm3 sync + numactl --cpubind=0 --membind=0 uadk_tool benchmark --alg sm3 --mode sva --opt 0 --sync --pktlen $size --seconds 20 --multi 1 --thread $x --ctxnum $x --init2 --prefetch + + #sm3 async + numactl --cpubind=0 --membind=0 uadk_tool benchmark --alg sm3 --mode sva --opt 0 --async --pktlen $size --seconds 20 --multi 1 --thread $x --ctxnum $x --init2 --prefetch +done diff --git a/test/wd_mempool_test.c b/test/wd_mempool_test.c index 6e28b462a..0b1e4e1a2 100644 --- a/test/wd_mempool_test.c +++ b/test/wd_mempool_test.c @@ -436,7 +436,7 @@ static __u32 sva_sched_pick_next_ctx(handle_t h_sched_ctx, return index; } -static int sva_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, +static int sva_sched_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { int recv = 0; diff --git a/uadk_tool/benchmark/hpre_uadk_benchmark.c b/uadk_tool/benchmark/hpre_uadk_benchmark.c index bc0687da5..0000d72d3 100644 --- a/uadk_tool/benchmark/hpre_uadk_benchmark.c +++ b/uadk_tool/benchmark/hpre_uadk_benchmark.c @@ -486,17 +486,17 @@ static int init_hpre_ctx_config(struct acc_option *options) switch(subtype) { case RSA_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_rsa_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_rsa_poll_ctx_); break; case DH_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_dh_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_dh_poll_ctx_); break; case ECDH_TYPE: case ECDSA_TYPE: case SM2_TYPE: case X25519_TYPE: case X448_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_ecc_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_ecc_poll_ctx_); break; default: HPRE_TST_PRT("failed to parse alg subtype!\n"); diff --git a/uadk_tool/benchmark/sec_uadk_benchmark.c b/uadk_tool/benchmark/sec_uadk_benchmark.c index 411d0c47c..68e7928d8 100644 --- a/uadk_tool/benchmark/sec_uadk_benchmark.c +++ b/uadk_tool/benchmark/sec_uadk_benchmark.c @@ -686,13 +686,13 @@ static int init_ctx_config(struct acc_option *options) switch(subtype) { case CIPHER_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_cipher_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_cipher_poll_ctx_); break; case AEAD_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_aead_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_aead_poll_ctx_); break; case DIGEST_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_digest_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_digest_poll_ctx_); break; default: SEC_TST_PRT("failed to parse alg subtype!\n"); diff --git a/uadk_tool/benchmark/zip_uadk_benchmark.c b/uadk_tool/benchmark/zip_uadk_benchmark.c index 2133297f7..5adf6e874 100644 --- a/uadk_tool/benchmark/zip_uadk_benchmark.c +++ b/uadk_tool/benchmark/zip_uadk_benchmark.c @@ -490,7 +490,7 @@ static int init_ctx_config(struct acc_option *options) goto free_ctxs; } - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 2, max_node, wd_comp_poll_ctx); + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 2, max_node, wd_comp_poll_ctx_); if (!g_sched) { ZIP_TST_PRT("failed to alloc sched!\n"); ret = -ENOMEM; diff --git a/uadk_tool/test/comp_lib.c b/uadk_tool/test/comp_lib.c index 014aa9895..6cb494a68 100644 --- a/uadk_tool/test/comp_lib.c +++ b/uadk_tool/test/comp_lib.c @@ -229,7 +229,8 @@ int hw_stream_compress(struct test_options *opts, return ret; } -static int lib_poll_func(__u32 pos, __u32 expect, __u32 *count) +static int lib_poll_func(struct wd_sched *sched, __u32 pos, + __u32 expect, __u32 *count) { int ret; diff --git a/uadk_tool/test/test_sec.c b/uadk_tool/test/test_sec.c index 87fc71840..9ee3ad419 100644 --- a/uadk_tool/test/test_sec.c +++ b/uadk_tool/test/test_sec.c @@ -42,6 +42,7 @@ static struct wd_ctx_config g_ctx_cfg; static struct wd_sched *g_sched; +static struct wd_sched g_sched_test; static long long int g_times; static unsigned int g_thread_num; @@ -593,7 +594,7 @@ int get_cipher_resource(struct cipher_testvec **alg_tv, int* alg, int* mode) return 0; } -static int sched_single_poll_policy(handle_t h_sched_ctx, +static int sched_single_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { return 0; @@ -618,7 +619,7 @@ static int init_ctx_config(int type, int mode) g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, numa_max_node() + 1, - wd_cipher_poll_ctx); + wd_cipher_poll_ctx_); if (!g_sched) { printf("Fail to alloc sched!\n"); goto out; @@ -1226,7 +1227,7 @@ static void *poll_func(void *arg) int expt = g_times * g_thread_num; while (1) { - ret = g_sched->poll_policy(g_sched->h_sched_ctx, 1, &count); + ret = g_sched->poll_policy(g_sched, 1, &count); if (ret != -EAGAIN && ret < 0) { SEC_TST_PRT("poll ctx is error----------->\n"); break; @@ -1416,7 +1417,6 @@ static __u32 sched_digest_pick_next_ctx(handle_t h_sched_ctx, static int digest_init1(int type, int mode) { struct uacce_dev_list *list; - struct wd_sched sched; int ret; if (g_use_env) @@ -1443,12 +1443,12 @@ static int digest_init1(int type, int mode) g_ctx_cfg.ctxs[0].op_type = type; g_ctx_cfg.ctxs[0].ctx_mode = mode; - sched.name = SCHED_SINGLE; - sched.pick_next_ctx = sched_digest_pick_next_ctx; - sched.poll_policy = sched_single_poll_policy; - sched.sched_init = sched_digest_init; + g_sched_test.name = SCHED_SINGLE; + g_sched_test.pick_next_ctx = sched_digest_pick_next_ctx; + g_sched_test.poll_policy = sched_single_poll_policy; + g_sched_test.sched_init = sched_digest_init; /* digest init */ - ret = wd_digest_init(&g_ctx_cfg, &sched); + ret = wd_digest_init(&g_ctx_cfg, &g_sched_test); if (ret) { SEC_TST_PRT("Fail to digest ctx!\n"); goto out; @@ -2685,7 +2685,6 @@ static __u32 sched_aead_pick_next_ctx(handle_t h_sched_ctx, static int aead_init1(int type, int mode) { struct uacce_dev_list *list; - struct wd_sched sched; int ret; if (g_use_env) @@ -2712,12 +2711,12 @@ static int aead_init1(int type, int mode) g_ctx_cfg.ctxs[0].op_type = type; g_ctx_cfg.ctxs[0].ctx_mode = mode; - sched.name = SCHED_SINGLE; - sched.pick_next_ctx = sched_aead_pick_next_ctx; - sched.poll_policy = sched_single_poll_policy; - sched.sched_init = sched_aead_init; + g_sched_test.name = SCHED_SINGLE; + g_sched_test.pick_next_ctx = sched_aead_pick_next_ctx; + g_sched_test.poll_policy = sched_single_poll_policy; + g_sched_test.sched_init = sched_aead_init; /* aead init*/ - ret = wd_aead_init(&g_ctx_cfg, &sched); + ret = wd_aead_init(&g_ctx_cfg, &g_sched_test); if (ret) { SEC_TST_PRT("Fail to aead ctx!\n"); goto out; diff --git a/wd_aead.c b/wd_aead.c index 608f0e6d7..6da6456e8 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -9,6 +9,7 @@ #include #include "include/drv/wd_aead_drv.h" #include "wd_aead.h" +#include "adapter.h" #define WD_AEAD_CCM_GCM_MIN 4U #define WD_AEAD_CCM_GCM_MAX 16 @@ -30,12 +31,9 @@ static char *wd_aead_alg_name[WD_CIPHER_ALG_TYPE_MAX][WD_CIPHER_MODE_TYPE_MAX] = struct wd_aead_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_alg_driver *driver; - struct wd_async_msg_pool pool; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_aead_setting; struct wd_aead_sess { @@ -52,11 +50,14 @@ struct wd_aead_sess { __u16 akey_bytes; __u16 auth_bytes; void *priv; - void *sched_key; + void **sched_key; /* Stored the counter for gcm stream mode */ __u8 iv[MAX_IV_SIZE]; /* Total of data for stream mode */ __u64 long_data_len; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; struct wd_env_config wd_aead_env_config; @@ -71,20 +72,16 @@ static void wd_aead_close_driver(int init_type) } if (wd_aead_setting.dlhandle) { - wd_release_drv(wd_aead_setting.driver); dlclose(wd_aead_setting.dlhandle); wd_aead_setting.dlhandle = NULL; } #else - wd_release_drv(wd_aead_setting.driver); hisi_sec2_remove(); #endif } static int wd_aead_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "gcm(aes)"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -118,15 +115,6 @@ static int wd_aead_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_aead_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_aead_setting.driver = driver; - return WD_SUCCESS; } @@ -307,7 +295,9 @@ int wd_aead_get_maxauthsize(handle_t h_sess) handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) { struct wd_aead_sess *sess = NULL; - bool ret; + struct uadk_adapter_worker *worker; + int nb = wd_aead_setting.adapter->workers_nb; + int ret, i; if (unlikely(!setup)) { WD_ERR("failed to check session input parameter!\n"); @@ -320,6 +310,10 @@ handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) return (handle_t)0; } + worker = sess->worker = &wd_aead_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; + sess = malloc(sizeof(struct wd_aead_sess)); if (!sess) { WD_ERR("failed to alloc session memory!\n"); @@ -332,24 +326,31 @@ handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) sess->cmode = setup->cmode; sess->dalg = setup->dalg; sess->dmode = setup->dmode; - ret = wd_drv_alg_support(sess->alg_name, wd_aead_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, worker->driver); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); goto err_sess; } - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_aead_setting.sched.sched_init( - wd_aead_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto err_sess; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_aead_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto err_sess; + } } return (handle_t)sess; err_sess: - if (sess->sched_key) + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -366,8 +367,11 @@ void wd_aead_free_sess(handle_t h_sess) wd_memset_zero(sess->ckey, sess->ckey_bytes); wd_memset_zero(sess->akey, sess->akey_bytes); - if (sess->sched_key) + if (sess->sched_key) { + for (int i = 0; i < wd_aead_setting.adapter->workers_nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); } @@ -440,49 +444,49 @@ static void wd_aead_clear_status(void) wd_alg_clear_init(&wd_aead_setting.status); } -static int wd_aead_init_nolock(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_aead_init_nolock(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; ret = wd_set_epoll_en("WD_AEAD_EPOLL_EN", - &wd_aead_setting.config.epoll_en); + &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_aead_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret) return ret; - ret = wd_init_sched(&wd_aead_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; /* init async request pool */ - ret = wd_init_async_request_pool(&wd_aead_setting.pool, - config, WD_POOL_MAX_ENTRIES, - sizeof(struct wd_aead_msg)); + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, + sizeof(struct wd_aead_msg)); if (ret < 0) - goto out_clear_sched; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_aead_setting.config, - wd_aead_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return 0; out_clear_pool: - wd_uninit_async_request_pool(&wd_aead_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_aead_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_aead_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } int wd_aead_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "gcm(aes)"; int ret; pthread_atfork(NULL, NULL, wd_aead_clear_status); @@ -495,11 +499,24 @@ int wd_aead_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_aead_setting.adapter = adapter; + ret = wd_aead_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_aead_init_nolock(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_aead_init_nolock(worker, sched); if (ret) goto out_close_driver; @@ -510,22 +527,26 @@ int wd_aead_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_aead_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_aead_setting.status); return ret; } static int wd_aead_uninit_nolock(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_aead_setting.status, &status); if (status == WD_UNINIT) return -WD_EINVAL; - wd_uninit_async_request_pool(&wd_aead_setting.pool); - wd_clear_sched(&wd_aead_setting.sched); - wd_alg_uninit_driver(&wd_aead_setting.config, - wd_aead_setting.driver); + for (int i = 0; i < wd_aead_setting.adapter->workers_nb; i++) { + worker = &wd_aead_setting.adapter->workers[i]; + + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return 0; } @@ -538,6 +559,7 @@ void wd_aead_uninit(void) if (ret) return; + free(wd_aead_setting.adapter); wd_aead_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_aead_setting.status); } @@ -561,7 +583,10 @@ int wd_aead_init2_(char *alg, __u32 sched_type, int task_type, { struct wd_ctx_nums aead_ctx_num[WD_DIGEST_CIPHER_DECRYPTION + 1] = {0}; struct wd_ctx_params aead_ctx_params = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; + int i; pthread_atfork(NULL, NULL, wd_aead_clear_status); @@ -580,77 +605,70 @@ int wd_aead_init2_(char *alg, __u32 sched_type, int task_type, goto out_uninit; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_uninit; + wd_aead_setting.adapter = adapter; + state = wd_aead_open_driver(WD_TYPE_V2); if (state) goto out_uninit; - while (ret != 0) { - memset(&wd_aead_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlopen; - /* Get alg driver and dev name */ - wd_aead_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_aead_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; aead_ctx_params.ctx_set_num = aead_ctx_num; ret = wd_ctx_param_init(&aead_ctx_params, ctx_params, - wd_aead_setting.driver, WD_AEAD_TYPE, + worker->driver, WD_AEAD_TYPE, WD_DIGEST_CIPHER_DECRYPTION + 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_aead_setting.driver); - wd_alg_drv_unbind(wd_aead_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlopen; } wd_aead_init_attrs.alg = alg; - wd_aead_init_attrs.sched_type = sched_type; - wd_aead_init_attrs.driver = wd_aead_setting.driver; wd_aead_init_attrs.ctx_params = &aead_ctx_params; wd_aead_init_attrs.alg_init = wd_aead_init_nolock; - wd_aead_init_attrs.alg_poll_ctx = wd_aead_poll_ctx; - ret = wd_alg_attrs_init(&wd_aead_init_attrs); + wd_aead_init_attrs.alg_poll_ctx = wd_aead_poll_ctx_; + ret = wd_alg_attrs_init(worker, &wd_aead_init_attrs); + wd_ctx_param_uninit(&aead_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_aead_setting.driver); - wd_alg_drv_unbind(wd_aead_setting.driver); - wd_ctx_param_uninit(&aead_ctx_params); - continue; - } WD_ERR("failed to init alg attrs.\n"); - goto out_params_uninit; + goto out_dlopen; } } + wd_alg_set_init(&wd_aead_setting.status); - wd_ctx_param_uninit(&aead_ctx_params); return 0; -out_params_uninit: - wd_ctx_param_uninit(&aead_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_aead_setting.driver); out_dlopen: wd_aead_close_driver(WD_TYPE_V2); out_uninit: + free(adapter); wd_alg_clear_init(&wd_aead_setting.status); return ret; } void wd_aead_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_aead_uninit_nolock(); if (ret) return; - wd_alg_attrs_uninit(&wd_aead_init_attrs); - wd_alg_drv_unbind(wd_aead_setting.driver); + for (int i = 0; i < wd_aead_setting.adapter->workers_nb; i++) { + worker = &wd_aead_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_aead_setting.adapter); wd_aead_close_driver(WD_TYPE_V2); wd_aead_setting.dlh_list = NULL; wd_alg_clear_init(&wd_aead_setting.status); @@ -725,18 +743,18 @@ static void fill_request_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, fill_stream_msg(msg, req, sess); } -static int send_recv_sync(struct wd_ctx_internal *ctx, +static int send_recv_sync(struct uadk_adapter_worker *worker, struct wd_ctx_internal *ctx, struct wd_aead_msg *msg) { struct wd_msg_handle msg_handle; int ret; - msg_handle.send = wd_aead_setting.driver->send; - msg_handle.recv = wd_aead_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_aead_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_aead_setting.config.epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + msg, NULL, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); return ret; @@ -744,8 +762,8 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, int wd_do_aead_sync(handle_t h_sess, struct wd_aead_req *req) { - struct wd_ctx_config_internal *config = &wd_aead_setting.config; struct wd_aead_sess *sess = (struct wd_aead_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_aead_msg msg; __u32 idx; @@ -755,20 +773,24 @@ int wd_do_aead_sync(handle_t h_sess, struct wd_aead_req *req) if (unlikely(ret)) return -WD_EINVAL; + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + memset(&msg, 0, sizeof(struct wd_aead_msg)); fill_request_msg(&msg, req, sess); req->state = 0; - idx = wd_aead_setting.sched.pick_next_ctx( - wd_aead_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (unlikely(ret)) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; - ret = send_recv_sync(ctx, &msg); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; + ret = send_recv_sync(worker, ctx, &msg); req->state = msg.result; return ret; @@ -776,8 +798,8 @@ int wd_do_aead_sync(handle_t h_sess, struct wd_aead_req *req) int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) { - struct wd_ctx_config_internal *config = &wd_aead_setting.config; struct wd_aead_sess *sess = (struct wd_aead_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_aead_msg *msg; int msg_id, ret; @@ -792,16 +814,20 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) return -WD_EINVAL; } - idx = wd_aead_setting.sched.pick_next_ctx( - wd_aead_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - msg_id = wd_get_msg_from_pool(&wd_aead_setting.pool, + msg_id = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(msg_id < 0)) { WD_ERR("failed to get msg from pool!\n"); @@ -811,7 +837,7 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) fill_request_msg(msg, req, sess); msg->tag = msg_id; - ret = wd_alg_driver_send(wd_aead_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("failed to send BD, hw is err!\n"); @@ -819,7 +845,7 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_aead_env_config, idx); if (ret) goto fail_with_msg; @@ -827,18 +853,13 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) return 0; fail_with_msg: - wd_put_msg_to_pool(&wd_aead_setting.pool, idx, msg->tag); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); return ret; } -struct wd_aead_msg *wd_aead_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_aead_setting.pool, idx, tag); -} - -int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_aead_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_aead_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_aead_msg resp_msg, *msg; struct wd_aead_req *req; @@ -851,16 +872,22 @@ int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_aead_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_aead_setting.driver, ctx->ctx, &resp_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { @@ -869,7 +896,7 @@ int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count) } recv_count++; - msg = wd_find_msg_in_pool(&wd_aead_setting.pool, + msg = wd_find_msg_in_pool(&worker->pool, idx, resp_msg.tag); if (!msg) { WD_ERR("failed to find msg from pool!\n"); @@ -880,25 +907,47 @@ int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count) msg->req.state = resp_msg.result; req = &msg->req; req->cb(req, req->cb_param); - wd_put_msg_to_pool(&wd_aead_setting.pool, - idx, resp_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, resp_msg.tag); *count = recv_count; } while (--tmp); return ret; } +int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +{ + return wd_aead_poll_ctx_(NULL, idx, expt, count); +} + int wd_aead_poll(__u32 expt, __u32 *count) { - handle_t h_ctx = wd_aead_setting.sched.h_sched_ctx; - struct wd_sched *sched = &wd_aead_setting.sched; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!count)) { WD_ERR("invalid: aead poll input param is NULL!\n"); return -WD_EINVAL; } - return sched->poll_policy(h_ctx, expt, count); + for (int i = 0; i < wd_aead_setting.adapter->workers_nb; i++) { + worker = &wd_aead_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + return ret; } static const struct wd_config_variable table[] = { diff --git a/wd_agg.c b/wd_agg.c index b536efe58..c90dd530a 100644 --- a/wd_agg.c +++ b/wd_agg.c @@ -8,6 +8,7 @@ #include #include #include "include/drv/wd_agg_drv.h" +#include "adapter.h" #include "wd_agg.h" #define DECIMAL_PRECISION_OFFSET 8 @@ -29,13 +30,10 @@ enum wd_agg_sess_state { struct wd_agg_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *priv; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_agg_setting; struct wd_agg_sess_key_conf { @@ -59,7 +57,7 @@ struct wd_agg_sess { wd_dev_mask_t *dev_mask; struct wd_alg_agg *drv; void *priv; - void *sched_key; + void **sched_key; enum wd_agg_sess_state state; struct wd_agg_ops ops; struct wd_agg_sess_key_conf key_conf; @@ -67,18 +65,20 @@ struct wd_agg_sess { struct wd_dae_charset charset_info; struct wd_dae_hash_table hash_table; struct wd_dae_hash_table rehash_table; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; static char *wd_agg_alg_name = "hashagg"; static struct wd_init_attrs wd_agg_init_attrs; -static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count); +static int wd_agg_poll_ctx(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count); static void wd_agg_close_driver(void) { #ifndef WD_STATIC_DRV wd_dlclose_drv(wd_agg_setting.dlh_list); #else - wd_release_drv(wd_agg_setting.driver); hisi_dae_remove(); #endif } @@ -384,9 +384,11 @@ static int wd_agg_init_sess_priv(struct wd_agg_sess *sess, struct wd_agg_sess_se handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) { + struct uadk_adapter_worker *worker; + int nb = wd_agg_setting.adapter->workers_nb; __u32 out_agg_cols_num = 0; struct wd_agg_sess *sess; - int ret; + int ret, i; ret = wd_agg_check_sess_params(setup, &out_agg_cols_num); if (ret) @@ -400,22 +402,30 @@ handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) memset(sess, 0, sizeof(struct wd_agg_sess)); sess->agg_conf.out_cols_num = out_agg_cols_num; + worker = sess->worker = &wd_agg_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; + sess->alg_name = wd_agg_alg_name; - ret = wd_drv_alg_support(sess->alg_name, wd_agg_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, worker->driver); if (!ret) { WD_ERR("failed to support agg algorithm: %s!\n", sess->alg_name); goto err_sess; } - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_agg_setting.sched.sched_init( - wd_agg_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init agg session schedule key!\n"); - goto err_sess; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_agg_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto err_sess; + } } - ret = wd_agg_setting.driver->get_extend_ops(&sess->ops); + ret = worker->driver->get_extend_ops(&sess->ops); if (ret) { WD_ERR("failed to get agg extend ops!\n"); goto err_sess; @@ -437,8 +447,11 @@ handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) if (sess->ops.sess_uninit) sess->ops.sess_uninit(sess->priv); err_sess: - if (sess->sched_key) + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -458,8 +471,11 @@ void wd_agg_free_sess(handle_t h_sess) if (sess->ops.sess_uninit) sess->ops.sess_uninit(sess->priv); - if (sess->sched_key) + if (sess->sched_key) { + for (int i = 0; i < wd_agg_setting.adapter->workers_nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); } @@ -570,67 +586,50 @@ static void wd_agg_clear_status(void) wd_alg_clear_init(&wd_agg_setting.status); } -static int wd_agg_alg_init(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_agg_alg_init(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; - ret = wd_set_epoll_en("WD_AGG_EPOLL_EN", &wd_agg_setting.config.epoll_en); + ret = wd_set_epoll_en("WD_AGG_EPOLL_EN", &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_agg_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_agg_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; /* Allocate async pool for every ctx */ - ret = wd_init_async_request_pool(&wd_agg_setting.pool, config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_agg_msg)); if (ret < 0) - goto out_clear_sched; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_agg_setting.config, wd_agg_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return WD_SUCCESS; out_clear_pool: - wd_uninit_async_request_pool(&wd_agg_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_agg_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_agg_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } -static int wd_agg_alg_uninit(void) -{ - void *priv = wd_agg_setting.priv; - - if (!priv) - return -WD_EINVAL; - - /* Uninit async request pool */ - wd_uninit_async_request_pool(&wd_agg_setting.pool); - - /* Unset config, sched, driver */ - wd_clear_sched(&wd_agg_setting.sched); - - wd_alg_uninit_driver(&wd_agg_setting.config, wd_agg_setting.driver); - - return WD_SUCCESS; -} - int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params) { struct wd_ctx_params agg_ctx_params = {0}; struct wd_ctx_nums agg_ctx_num = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int ret = -WD_EINVAL; - int state; + int state, i; bool flag; pthread_atfork(NULL, NULL, wd_agg_clear_status); @@ -651,60 +650,46 @@ int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params goto out_uninit; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_uninit; + wd_agg_setting.adapter = adapter; + state = wd_agg_open_driver(); if (state) goto out_uninit; - while (ret != 0) { - memset(&wd_agg_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlopen; - /* Get alg driver and dev name */ - wd_agg_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_agg_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; agg_ctx_params.ctx_set_num = &agg_ctx_num; - ret = wd_ctx_param_init(&agg_ctx_params, ctx_params, wd_agg_setting.driver, + ret = wd_ctx_param_init(&agg_ctx_params, ctx_params, worker->driver, WD_AGG_TYPE, 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_agg_setting.driver); - wd_alg_drv_unbind(wd_agg_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlopen; } wd_agg_init_attrs.alg = alg; - wd_agg_init_attrs.sched_type = sched_type; - wd_agg_init_attrs.driver = wd_agg_setting.driver; wd_agg_init_attrs.ctx_params = &agg_ctx_params; wd_agg_init_attrs.alg_init = wd_agg_alg_init; wd_agg_init_attrs.alg_poll_ctx = wd_agg_poll_ctx; - ret = wd_alg_attrs_init(&wd_agg_init_attrs); + ret = wd_alg_attrs_init(worker, &wd_agg_init_attrs); + wd_ctx_param_uninit(&agg_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_agg_setting.driver); - wd_alg_drv_unbind(wd_agg_setting.driver); - wd_ctx_param_uninit(&agg_ctx_params); - continue; - } WD_ERR("fail to init alg attrs.\n"); - goto out_params_uninit; + goto out_dlopen; } } wd_alg_set_init(&wd_agg_setting.status); - wd_ctx_param_uninit(&agg_ctx_params); return WD_SUCCESS; -out_params_uninit: - wd_ctx_param_uninit(&agg_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_agg_setting.driver); out_dlopen: wd_agg_close_driver(); out_uninit: @@ -714,14 +699,15 @@ int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params void wd_agg_uninit(void) { - int ret; + struct uadk_adapter_worker *worker; - ret = wd_agg_alg_uninit(); - if (ret) - return; + for (int i = 0; i < wd_agg_setting.adapter->workers_nb; i++) { + worker = &wd_agg_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } - wd_alg_attrs_uninit(&wd_agg_init_attrs); - wd_alg_drv_unbind(wd_agg_setting.driver); wd_agg_close_driver(); wd_agg_setting.dlh_list = NULL; wd_alg_clear_init(&wd_agg_setting.status); @@ -1076,30 +1062,30 @@ static int wd_agg_check_rehash_params(struct wd_agg_sess *sess, struct wd_agg_re return ret; } -static int wd_agg_sync_job(struct wd_agg_sess *sess, struct wd_agg_req *req, - struct wd_agg_msg *msg) +static int wd_agg_sync_job(struct uadk_adapter_worker *worker, struct wd_agg_sess *sess, + struct wd_agg_req *req, struct wd_agg_msg *msg) { - struct wd_ctx_config_internal *config = &wd_agg_setting.config; struct wd_msg_handle msg_handle; struct wd_ctx_internal *ctx; __u32 idx; int ret; - idx = wd_agg_setting.sched.pick_next_ctx(wd_agg_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + idx = worker->sched->pick_next_ctx(worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], + CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (unlikely(ret)) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; - msg_handle.send = wd_agg_setting.driver->send; - msg_handle.recv = wd_agg_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_agg_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, config->epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + msg, NULL, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); return ret; @@ -1144,6 +1130,7 @@ int wd_agg_add_input_sync(handle_t h_sess, struct wd_agg_req *req) { struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; enum wd_agg_sess_state expected = WD_AGG_SESS_INIT; + struct uadk_adapter_worker *worker; struct wd_agg_msg msg; int ret; @@ -1157,11 +1144,15 @@ int wd_agg_add_input_sync(handle_t h_sess, struct wd_agg_req *req) if (unlikely(ret)) return ret; + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + memset(&msg, 0, sizeof(struct wd_agg_msg)); fill_request_msg_input(&msg, req, sess, false); req->state = 0; - ret = wd_agg_sync_job(sess, req, &msg); + ret = wd_agg_sync_job(worker, sess, req, &msg); if (unlikely(ret)) { if (expected == WD_AGG_SESS_INIT) __atomic_store_n(&sess->state, expected, __ATOMIC_RELEASE); @@ -1177,20 +1168,26 @@ int wd_agg_add_input_sync(handle_t h_sess, struct wd_agg_req *req) static int wd_agg_async_job(struct wd_agg_sess *sess, struct wd_agg_req *req, bool is_input) { - struct wd_ctx_config_internal *config = &wd_agg_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; __u32 idx; int msg_id, ret; struct wd_agg_msg *msg; - idx = wd_agg_setting.sched.pick_next_ctx(wd_agg_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx(worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (unlikely(ret)) return ret; - ctx = config->ctxs + idx; - msg_id = wd_get_msg_from_pool(&wd_agg_setting.pool, idx, (void **)&msg); + ctx = worker->config.ctxs + idx; + + msg_id = wd_get_msg_from_pool(&worker->pool, idx, + (void **)&msg); if (unlikely(msg_id < 0)) { WD_ERR("failed to get agg msg from pool!\n"); return msg_id; @@ -1201,7 +1198,7 @@ static int wd_agg_async_job(struct wd_agg_sess *sess, struct wd_agg_req *req, bo else fill_request_msg_output(msg, req, sess, false); msg->tag = msg_id; - ret = wd_alg_driver_send(wd_agg_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd agg async send err!\n"); @@ -1209,12 +1206,12 @@ static int wd_agg_async_job(struct wd_agg_sess *sess, struct wd_agg_req *req, bo goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); return WD_SUCCESS; fail_with_msg: - wd_put_msg_to_pool(&wd_agg_setting.pool, idx, msg->tag); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); return ret; } @@ -1263,6 +1260,7 @@ int wd_agg_get_output_sync(handle_t h_sess, struct wd_agg_req *req) { struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; enum wd_agg_sess_state expected = WD_AGG_SESS_INPUT; + struct uadk_adapter_worker *worker; struct wd_agg_msg msg; int ret; @@ -1276,11 +1274,15 @@ int wd_agg_get_output_sync(handle_t h_sess, struct wd_agg_req *req) if (unlikely(ret)) return ret; + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + memset(&msg, 0, sizeof(struct wd_agg_msg)); fill_request_msg_output(&msg, req, sess, false); req->state = 0; - ret = wd_agg_sync_job(sess, req, &msg); + ret = wd_agg_sync_job(worker, sess, req, &msg); if (unlikely(ret)) { if (expected == WD_AGG_SESS_INPUT) __atomic_store_n(&sess->state, expected, __ATOMIC_RELEASE); @@ -1405,7 +1407,9 @@ static int wd_agg_set_col_size(struct wd_agg_sess *sess, struct wd_agg_req *req, return WD_SUCCESS; } -static int wd_agg_rehash_sync_inner(struct wd_agg_sess *sess, struct wd_agg_req *req) +static int wd_agg_rehash_sync_inner(struct uadk_adapter_worker *worker, + struct wd_agg_sess *sess, + struct wd_agg_req *req) { struct wd_agg_msg msg = {0}; bool output_done; @@ -1414,7 +1418,7 @@ static int wd_agg_rehash_sync_inner(struct wd_agg_sess *sess, struct wd_agg_req fill_request_msg_output(&msg, req, sess, true); req->state = 0; - ret = wd_agg_sync_job(sess, req, &msg); + ret = wd_agg_sync_job(worker, sess, req, &msg); if (unlikely(ret)) return ret; @@ -1439,7 +1443,7 @@ static int wd_agg_rehash_sync_inner(struct wd_agg_sess *sess, struct wd_agg_req memset(&msg, 0, sizeof(struct wd_agg_msg)); fill_request_msg_input(&msg, req, sess, true); - ret = wd_agg_sync_job(sess, req, &msg); + ret = wd_agg_sync_job(worker, sess, req, &msg); if (unlikely(ret)) return ret; @@ -1471,6 +1475,7 @@ int wd_agg_rehash_sync(handle_t h_sess, struct wd_agg_req *req) { struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; enum wd_agg_sess_state expected = WD_AGG_SESS_RESET; + struct uadk_adapter_worker *worker; struct wd_agg_req src_req; __u64 cnt = 0; __u64 max_cnt; @@ -1486,10 +1491,14 @@ int wd_agg_rehash_sync(handle_t h_sess, struct wd_agg_req *req) if (unlikely(ret)) return ret; + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + memcpy(&src_req, req, sizeof(struct wd_agg_req)); max_cnt = MAX_HASH_TABLE_ROW_NUM / req->out_row_count; while (cnt < max_cnt) { - ret = wd_agg_rehash_sync_inner(sess, &src_req); + ret = wd_agg_rehash_sync_inner(worker, sess, &src_req); if (ret) { __atomic_store_n(&sess->state, WD_AGG_SESS_RESET, __ATOMIC_RELEASE); WD_ERR("failed to do agg rehash task!\n"); @@ -1504,14 +1513,9 @@ int wd_agg_rehash_sync(handle_t h_sess, struct wd_agg_req *req) return WD_SUCCESS; } -struct wd_agg_msg *wd_agg_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_agg_setting.pool, idx, tag); -} - -static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +static int wd_agg_poll_ctx(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_agg_setting.config; + struct uadk_adapter_worker *worker = sched->worker; struct wd_agg_msg resp_msg, *msg; struct wd_ctx_internal *ctx; struct wd_agg_req *req; @@ -1521,14 +1525,14 @@ static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count) *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (unlikely(ret)) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_agg_setting.driver, ctx->ctx, &resp_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) { return ret; } else if (unlikely(ret < 0)) { @@ -1536,7 +1540,7 @@ static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return ret; } recv_count++; - msg = wd_find_msg_in_pool(&wd_agg_setting.pool, idx, resp_msg.tag); + msg = wd_find_msg_in_pool(&worker->pool, idx, resp_msg.tag); if (unlikely(!msg)) { WD_ERR("failed to get agg msg from pool!\n"); return -WD_EINVAL; @@ -1551,7 +1555,7 @@ static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count) req->cb(req, req->cb_param); /* Free msg cache to msg_pool */ - wd_put_msg_to_pool(&wd_agg_setting.pool, idx, resp_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, resp_msg.tag); *count = recv_count; } while (--tmp); @@ -1560,13 +1564,31 @@ static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count) int wd_agg_poll(__u32 expt, __u32 *count) { - handle_t h_ctx = wd_agg_setting.sched.h_sched_ctx; - struct wd_sched *sched = &wd_agg_setting.sched; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!expt || !count)) { WD_ERR("invalid: agg poll input param is NULL!\n"); return -WD_EINVAL; } - return sched->poll_policy(h_ctx, expt, count); + for (int i = 0; i < wd_agg_setting.adapter->workers_nb; i++) { + worker = &wd_agg_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + return ret; } diff --git a/wd_alg.c b/wd_alg.c index 3d014e18e..2692fc7ce 100644 --- a/wd_alg.c +++ b/wd_alg.c @@ -395,3 +395,44 @@ void wd_release_drv(struct wd_alg_driver *drv) select_node->refcnt--; pthread_mutex_unlock(&mutex); } + +struct wd_alg_driver *wd_find_drv(char *drv_name, char *alg_name, int idx) +{ + struct wd_alg_list *head = &alg_list_head; + struct wd_alg_list *pnext = head->next; + struct wd_alg_driver *drv = NULL; + + if (!pnext || !alg_name) { + WD_ERR("invalid: request alg param is error!\n"); + return NULL; + } + + pthread_mutex_lock(&mutex); + + if (drv_name) { + while (pnext) { + if (!strcmp(alg_name, pnext->alg_name) && + !strcmp(drv_name, pnext->drv_name)) { + drv = pnext->drv; + break; + } + pnext = pnext->next; + } + } else { + int i = 0; + + while (pnext) { + if (!strcmp(alg_name, pnext->alg_name)) { + if (i++ == idx) { + drv = pnext->drv; + break; + } + } + pnext = pnext->next; + } + } + + pthread_mutex_unlock(&mutex); + + return drv; +} diff --git a/wd_cipher.c b/wd_cipher.c index ec6fb1562..486ca47b2 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -10,6 +10,7 @@ #include #include "include/drv/wd_cipher_drv.h" #include "wd_cipher.h" +#include "adapter.h" #define XTS_MODE_KEY_SHIFT 1 #define XTS_MODE_KEY_LEN_MASK 0x1 @@ -49,12 +50,9 @@ static char *wd_cipher_alg_name[WD_CIPHER_ALG_TYPE_MAX][WD_CIPHER_MODE_TYPE_MAX] struct wd_cipher_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_cipher_setting; struct wd_cipher_sess { @@ -66,12 +64,46 @@ struct wd_cipher_sess { void *priv; unsigned char key[MAX_CIPHER_KEY_SIZE]; __u32 key_bytes; - void *sched_key; + void **sched_key; + struct uadk_adapter_worker *worker; + int worker_looptime; + enum uadk_adapter_mode adapter_mode; }; struct wd_env_config wd_cipher_env_config; static struct wd_init_attrs wd_cipher_init_attrs; +static void wd_cipher_switch_worker(struct wd_cipher_sess *sess, int para) +{ + struct uadk_adapter_worker *worker; + + worker = uadk_adapter_switch_worker(wd_cipher_setting.adapter, + sess->worker, para); + if (worker) + sess->worker = worker; + sess->worker_looptime = 0; +} + +static struct uadk_adapter_worker * +wd_cipher_get_worker(struct wd_cipher_sess *sess, struct wd_cipher_req *req) +{ + if (sess->adapter_mode != wd_cipher_setting.adapter->mode) { + sess->worker = &wd_cipher_setting.adapter->workers[0]; + sess->worker_looptime = 0; + sess->adapter_mode = wd_cipher_setting.adapter->mode; + } + + if (sess->adapter_mode == UADK_ADAPT_MODE_THRESHOLD && + wd_cipher_setting.adapter->workers_nb == 2) { + if (req->in_bytes >= wd_cipher_setting.adapter->threshold) + sess->worker = &wd_cipher_setting.adapter->workers[0]; + else + sess->worker = &wd_cipher_setting.adapter->workers[1]; + } + + return sess->worker; +} + static void wd_cipher_close_driver(int init_type) { #ifndef WD_STATIC_DRV @@ -81,20 +113,16 @@ static void wd_cipher_close_driver(int init_type) } if (wd_cipher_setting.dlhandle) { - wd_release_drv(wd_cipher_setting.driver); dlclose(wd_cipher_setting.dlhandle); wd_cipher_setting.dlhandle = NULL; } #else - wd_release_drv(wd_cipher_setting.driver); hisi_sec2_remove(); #endif } static int wd_cipher_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "cbc(aes)"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -128,14 +156,6 @@ static int wd_cipher_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_cipher_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_cipher_setting.driver = driver; return WD_SUCCESS; } @@ -252,8 +272,10 @@ int wd_cipher_set_key(handle_t h_sess, const __u8 *key, __u32 key_len) handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) { + struct uadk_adapter_worker *worker; struct wd_cipher_sess *sess = NULL; - bool ret; + int nb = wd_cipher_setting.adapter->workers_nb; + int ret, i; if (unlikely(!setup)) { WD_ERR("invalid: cipher input setup is NULL!\n"); @@ -273,28 +295,40 @@ handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) goto err_sess; } + worker = sess->worker = &wd_cipher_setting.adapter->workers[0]; + worker->valid = true; + + sess->worker_looptime = 0; sess->alg_name = wd_cipher_alg_name[setup->alg][setup->mode]; - ret = wd_drv_alg_support(sess->alg_name, wd_cipher_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, worker->driver); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); goto err_sess; } sess->alg = setup->alg; sess->mode = setup->mode; + sess->adapter_mode = wd_cipher_setting.adapter->mode; - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_cipher_setting.sched.sched_init( - wd_cipher_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto err_sess; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_cipher_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto err_sess; + } } return (handle_t)sess; err_sess: - if (sess->sched_key) + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -310,8 +344,11 @@ void wd_cipher_free_sess(handle_t h_sess) wd_memset_zero(sess->key, sess->key_bytes); - if (sess->sched_key) + if (sess->sched_key) { + for (int i = 0; i < wd_cipher_setting.adapter->workers_nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); } @@ -320,49 +357,47 @@ static void wd_cipher_clear_status(void) wd_alg_clear_init(&wd_cipher_setting.status); } -static int wd_cipher_common_init(struct wd_ctx_config *config, +static int wd_cipher_common_init(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; ret = wd_set_epoll_en("WD_CIPHER_EPOLL_EN", - &wd_cipher_setting.config.epoll_en); + &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_cipher_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_cipher_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; /* allocate async pool for every ctx */ - ret = wd_init_async_request_pool(&wd_cipher_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_cipher_msg)); if (ret < 0) - goto out_clear_sched; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return 0; out_clear_pool: - wd_uninit_async_request_pool(&wd_cipher_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_cipher_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_cipher_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } static int wd_cipher_common_uninit(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_cipher_setting.status, &status); @@ -370,19 +405,20 @@ static int wd_cipher_common_uninit(void) return -WD_EINVAL; /* uninit async request pool */ - wd_uninit_async_request_pool(&wd_cipher_setting.pool); - - /* unset config, sched, driver */ - wd_clear_sched(&wd_cipher_setting.sched); - - wd_alg_uninit_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); + for (int i = 0; i < wd_cipher_setting.adapter->workers_nb; i++) { + worker = &wd_cipher_setting.adapter->workers[i]; + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return 0; } int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "cbc(aes)"; int ret; pthread_atfork(NULL, NULL, wd_cipher_clear_status); @@ -395,11 +431,24 @@ int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_cipher_setting.adapter = adapter; + ret = wd_cipher_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_cipher_common_init(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_cipher_common_init(worker, sched); if (ret) goto out_close_driver; @@ -410,6 +459,7 @@ int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_cipher_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_cipher_setting.status); return ret; } @@ -422,6 +472,7 @@ void wd_cipher_uninit(void) if (ret) return; + free(wd_cipher_setting.adapter); wd_cipher_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_cipher_setting.status); } @@ -430,8 +481,11 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p { struct wd_ctx_nums cipher_ctx_num[WD_CIPHER_DECRYPTION + 1] = {0}; struct wd_ctx_params cipher_ctx_params = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; bool flag; + int i; pthread_atfork(NULL, NULL, wd_cipher_clear_status); @@ -451,78 +505,69 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p goto out_uninit; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_uninit; + wd_cipher_setting.adapter = adapter; + state = wd_cipher_open_driver(WD_TYPE_V2); if (state) goto out_uninit; - while (ret != 0) { - memset(&wd_cipher_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_driver; - /* Get alg driver and dev name */ - wd_cipher_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_cipher_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; cipher_ctx_params.ctx_set_num = cipher_ctx_num; - ret = wd_ctx_param_init(&cipher_ctx_params, ctx_params, - wd_cipher_setting.driver, + ret = wd_ctx_param_init(&cipher_ctx_params, ctx_params, worker->driver, WD_CIPHER_TYPE, WD_CIPHER_DECRYPTION + 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_cipher_setting.driver); - wd_alg_drv_unbind(wd_cipher_setting.driver); - continue; - } + WD_ERR("fail to init ctx param\n"); goto out_driver; } wd_cipher_init_attrs.alg = alg; - wd_cipher_init_attrs.sched_type = sched_type; - wd_cipher_init_attrs.driver = wd_cipher_setting.driver; wd_cipher_init_attrs.ctx_params = &cipher_ctx_params; wd_cipher_init_attrs.alg_init = wd_cipher_common_init; - wd_cipher_init_attrs.alg_poll_ctx = wd_cipher_poll_ctx; - ret = wd_alg_attrs_init(&wd_cipher_init_attrs); + wd_cipher_init_attrs.alg_poll_ctx = wd_cipher_poll_ctx_; + ret = wd_alg_attrs_init(worker, &wd_cipher_init_attrs); + wd_ctx_param_uninit(&cipher_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_cipher_setting.driver); - wd_alg_drv_unbind(wd_cipher_setting.driver); - wd_ctx_param_uninit(&cipher_ctx_params); - continue; - } WD_ERR("fail to init alg attrs.\n"); - goto out_params_uninit; + goto out_driver; } } wd_alg_set_init(&wd_cipher_setting.status); - wd_ctx_param_uninit(&cipher_ctx_params); return 0; -out_params_uninit: - wd_ctx_param_uninit(&cipher_ctx_params); out_driver: - wd_alg_drv_unbind(wd_cipher_setting.driver); -out_dlopen: wd_cipher_close_driver(WD_TYPE_V2); out_uninit: + free(adapter); wd_alg_clear_init(&wd_cipher_setting.status); return ret; } void wd_cipher_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_cipher_common_uninit(); if (ret) return; - wd_alg_attrs_uninit(&wd_cipher_init_attrs); - wd_alg_drv_unbind(wd_cipher_setting.driver); + for (int i = 0; i < wd_cipher_setting.adapter->workers_nb; i++) { + worker = &wd_cipher_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_cipher_setting.adapter); wd_cipher_close_driver(WD_TYPE_V2); wd_cipher_setting.dlh_list = NULL; wd_alg_clear_init(&wd_cipher_setting.status); @@ -676,27 +721,27 @@ static int wd_cipher_check_params(handle_t h_sess, return cipher_iv_len_check(req, sess); } -static int send_recv_sync(struct wd_ctx_internal *ctx, +static int send_recv_sync(struct uadk_adapter_worker *worker, struct wd_ctx_internal *ctx, struct wd_cipher_msg *msg) { struct wd_msg_handle msg_handle; int ret; - msg_handle.send = wd_cipher_setting.driver->send; - msg_handle.recv = wd_cipher_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; - wd_ctx_spin_lock(ctx, wd_cipher_setting.driver->calc_type); - ret = wd_handle_msg_sync(wd_cipher_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_cipher_setting.config.epoll_en); - wd_ctx_spin_unlock(ctx, wd_cipher_setting.driver->calc_type); + wd_ctx_spin_lock(ctx, worker->driver->calc_type); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + msg, NULL, worker->config.epoll_en); + wd_ctx_spin_unlock(ctx, worker->driver->calc_type); return ret; } int wd_do_cipher_sync(handle_t h_sess, struct wd_cipher_req *req) { - struct wd_ctx_config_internal *config = &wd_cipher_setting.config; struct wd_cipher_sess *sess = (struct wd_cipher_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_cipher_msg msg; __u32 idx; @@ -708,30 +753,46 @@ int wd_do_cipher_sync(handle_t h_sess, struct wd_cipher_req *req) return ret; } + worker = wd_cipher_get_worker(sess, req); + memset(&msg, 0, sizeof(struct wd_cipher_msg)); fill_request_msg(&msg, req, sess); req->state = 0; - idx = wd_cipher_setting.sched.pick_next_ctx( - wd_cipher_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (unlikely(ret)) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; - ret = send_recv_sync(ctx, &msg); + ret = send_recv_sync(worker, ctx, &msg); req->state = msg.result; + if (ret) { + wd_cipher_switch_worker(sess, 1); + sess->worker_looptime++; + return ret; + } + + if ((sess->worker_looptime != 0) || + (wd_cipher_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) { + sess->worker_looptime++; + } + + if (sess->worker_looptime >= wd_cipher_setting.adapter->looptime) + wd_cipher_switch_worker(sess, 0); + return ret; } int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) { - struct wd_ctx_config_internal *config = &wd_cipher_setting.config; struct wd_cipher_sess *sess = (struct wd_cipher_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_cipher_msg *msg; int msg_id, ret; @@ -743,17 +804,28 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) return ret; } - idx = wd_cipher_setting.sched.pick_next_ctx( - wd_cipher_setting.sched.h_sched_ctx, - sess->sched_key, CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + worker = wd_cipher_get_worker(sess, req); + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + ret = wd_do_cipher_sync(h_sess, req); + if (!ret) { + pthread_mutex_lock(&worker->mutex); + worker->async_recv++; + pthread_mutex_unlock(&worker->mutex); + req->cb(req, req->cb_param); + } + return ret; + } + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - msg_id = wd_get_msg_from_pool(&wd_cipher_setting.pool, idx, - (void **)&msg); + msg_id = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(msg_id < 0)) { WD_ERR("failed to get msg from pool!\n"); return msg_id; @@ -762,7 +834,7 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) fill_request_msg(msg, req, sess); msg->tag = msg_id; - ret = wd_alg_driver_send(wd_cipher_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd cipher async send err!\n"); @@ -770,26 +842,30 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_cipher_env_config, idx); if (ret) goto fail_with_msg; + if ((sess->worker_looptime != 0) || + (wd_cipher_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) + sess->worker_looptime++; + + if (sess->worker_looptime >= wd_cipher_setting.adapter->looptime) + wd_cipher_switch_worker(sess, 0); + return 0; fail_with_msg: - wd_put_msg_to_pool(&wd_cipher_setting.pool, idx, msg->tag); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); + wd_cipher_switch_worker(sess, 1); + sess->worker_looptime++; return ret; } -struct wd_cipher_msg *wd_cipher_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_cipher_setting.pool, idx, tag); -} - -int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_cipher_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_cipher_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_cipher_msg resp_msg, *msg; struct wd_cipher_req *req; @@ -802,16 +878,32 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_cipher_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + pthread_mutex_lock(&worker->mutex); + if (worker->async_recv > 0) { + *count = worker->async_recv > expt ? expt : worker->async_recv; + worker->async_recv -= *count; + } + pthread_mutex_unlock(&worker->mutex); + return 0; + } + + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_cipher_setting.driver, ctx->ctx, &resp_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) return ret; else if (ret < 0) { @@ -819,7 +911,7 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return ret; } recv_count++; - msg = wd_find_msg_in_pool(&wd_cipher_setting.pool, idx, + msg = wd_find_msg_in_pool(&worker->pool, idx, resp_msg.tag); if (!msg) { WD_ERR("failed to find msg from pool!\n"); @@ -832,25 +924,48 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) req->cb(req, req->cb_param); /* free msg cache to msg_pool */ - wd_put_msg_to_pool(&wd_cipher_setting.pool, idx, - resp_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, resp_msg.tag); *count = recv_count; } while (--tmp); return ret; } +int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +{ + return wd_cipher_poll_ctx_(NULL, idx, expt, count); +} + int wd_cipher_poll(__u32 expt, __u32 *count) { - handle_t h_ctx = wd_cipher_setting.sched.h_sched_ctx; - struct wd_sched *sched = &wd_cipher_setting.sched; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!count)) { WD_ERR("invalid: cipher poll input param is NULL!\n"); return -WD_EINVAL; } - return sched->poll_policy(h_ctx, expt, count); + for (int i = 0; i < wd_cipher_setting.adapter->workers_nb; i++) { + worker = &wd_cipher_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + + return ret; } static const struct wd_config_variable table[] = { @@ -916,3 +1031,8 @@ int wd_cipher_get_env_param(__u32 node, __u32 type, __u32 mode, return wd_alg_get_env_param(&wd_cipher_env_config, ctx_attr, num, is_enable); } + +void wd_cipher_set_adapter_mode(enum uadk_adapter_mode mode) +{ + wd_cipher_setting.adapter->mode = mode; +} diff --git a/wd_comp.c b/wd_comp.c index 0fa5f9262..0eb084b49 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -14,6 +14,7 @@ #include "drv/wd_comp_drv.h" #include "wd_comp.h" +#include "adapter.h" #define HW_CTX_SIZE (64 * 1024) #define STREAM_CHUNK (128 * 1024) @@ -38,22 +39,53 @@ struct wd_comp_sess { __u32 isize; __u32 checksum; __u8 *ctx_buf; - void *sched_key; + void **sched_key; + struct uadk_adapter_worker *worker; + int worker_looptime; + enum uadk_adapter_mode adapter_mode; }; struct wd_comp_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_comp_setting; struct wd_env_config wd_comp_env_config; static struct wd_init_attrs wd_comp_init_attrs; +static void wd_comp_switch_worker(struct wd_comp_sess *sess, int para) +{ + struct uadk_adapter_worker *worker; + + worker = uadk_adapter_switch_worker(wd_comp_setting.adapter, + sess->worker, para); + if (worker) + sess->worker = worker; + sess->worker_looptime = 0; +} + +static struct uadk_adapter_worker * +wd_comp_get_worker(struct wd_comp_sess *sess, struct wd_comp_req *req) +{ + if (sess->adapter_mode != wd_comp_setting.adapter->mode) { + sess->worker = &wd_comp_setting.adapter->workers[0]; + sess->worker_looptime = 0; + sess->adapter_mode = wd_comp_setting.adapter->mode; + } + + if (sess->adapter_mode == UADK_ADAPT_MODE_THRESHOLD && + wd_comp_setting.adapter->workers_nb == 2) { + if (req->src_len >= wd_comp_setting.adapter->threshold) + sess->worker = &wd_comp_setting.adapter->workers[0]; + else + sess->worker = &wd_comp_setting.adapter->workers[1]; + } + + return sess->worker; +} + static void wd_comp_close_driver(int init_type) { #ifndef WD_STATIC_DRV @@ -63,20 +95,16 @@ static void wd_comp_close_driver(int init_type) } if (wd_comp_setting.dlhandle) { - wd_release_drv(wd_comp_setting.driver); dlclose(wd_comp_setting.dlhandle); wd_comp_setting.dlhandle = NULL; } #else - wd_release_drv(wd_comp_setting.driver); hisi_zip_remove(); #endif } static int wd_comp_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "zlib"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -110,15 +138,6 @@ static int wd_comp_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_comp_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_comp_setting.driver = driver; - return WD_SUCCESS; } @@ -142,67 +161,66 @@ static bool wd_comp_alg_check(const char *alg_name) return false; } -static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_comp_init_nolock(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; ret = wd_set_epoll_en("WD_COMP_EPOLL_EN", - &wd_comp_setting.config.epoll_en); + &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_comp_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_comp_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; - ret = wd_init_async_request_pool(&wd_comp_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_comp_msg)); if (ret < 0) - goto out_clear_sched; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_comp_setting.config, - wd_comp_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return 0; out_clear_pool: - wd_uninit_async_request_pool(&wd_comp_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_comp_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_comp_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } static int wd_comp_uninit_nolock(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_comp_setting.status, &status); if (status == WD_UNINIT) return -WD_EINVAL; - /* Uninit async request pool */ - wd_uninit_async_request_pool(&wd_comp_setting.pool); - - /* Unset config, sched, driver */ - wd_clear_sched(&wd_comp_setting.sched); + for (int i = 0; i < wd_comp_setting.adapter->workers_nb; i++) { + worker = &wd_comp_setting.adapter->workers[i]; - wd_alg_uninit_driver(&wd_comp_setting.config, - wd_comp_setting.driver); + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return 0; } int wd_comp_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "zlib"; int ret; pthread_atfork(NULL, NULL, wd_comp_clear_status); @@ -215,11 +233,24 @@ int wd_comp_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_comp_setting.adapter = adapter; + ret = wd_comp_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_comp_init_nolock(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_clear_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_comp_init_nolock(worker, sched); if (ret) goto out_clear_driver; @@ -230,6 +261,7 @@ int wd_comp_init(struct wd_ctx_config *config, struct wd_sched *sched) out_clear_driver: wd_comp_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_comp_setting.status); return ret; } @@ -242,6 +274,7 @@ void wd_comp_uninit(void) if (ret) return; + free(wd_comp_setting.adapter); wd_comp_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_comp_setting.status); } @@ -250,8 +283,11 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par { struct wd_ctx_nums comp_ctx_num[WD_DIR_MAX] = {0}; struct wd_ctx_params comp_ctx_params = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; bool flag; + int i; pthread_atfork(NULL, NULL, wd_comp_clear_status); @@ -271,90 +307,78 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par goto out_uninit; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_uninit; + wd_comp_setting.adapter = adapter; + state = wd_comp_open_driver(WD_TYPE_V2); if (state) goto out_uninit; - while (ret != 0) { - memset(&wd_comp_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlclose; - /* Get alg driver and dev name */ - wd_comp_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_comp_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlclose; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; comp_ctx_params.ctx_set_num = comp_ctx_num; ret = wd_ctx_param_init(&comp_ctx_params, ctx_params, - wd_comp_setting.driver, WD_COMP_TYPE, WD_DIR_MAX); + worker->driver, + WD_COMP_TYPE, WD_DIR_MAX); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_comp_setting.driver); - wd_alg_drv_unbind(wd_comp_setting.driver); - continue; - } - goto out_unbind_drv; + WD_ERR("fail to init ctx param\n"); + goto out_dlclose; } wd_comp_init_attrs.alg = alg; - wd_comp_init_attrs.sched_type = sched_type; - wd_comp_init_attrs.driver = wd_comp_setting.driver; wd_comp_init_attrs.ctx_params = &comp_ctx_params; wd_comp_init_attrs.alg_init = wd_comp_init_nolock; - wd_comp_init_attrs.alg_poll_ctx = wd_comp_poll_ctx; - ret = wd_alg_attrs_init(&wd_comp_init_attrs); + wd_comp_init_attrs.alg_poll_ctx = wd_comp_poll_ctx_; + ret = wd_alg_attrs_init(worker, &wd_comp_init_attrs); + wd_ctx_param_uninit(&comp_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_comp_setting.driver); - wd_alg_drv_unbind(wd_comp_setting.driver); - wd_ctx_param_uninit(&comp_ctx_params); - continue; - } WD_ERR("fail to init alg attrs.\n"); - goto out_params_uninit; + goto out_dlclose; } } wd_alg_set_init(&wd_comp_setting.status); - wd_ctx_param_uninit(&comp_ctx_params); return 0; -out_params_uninit: - wd_ctx_param_uninit(&comp_ctx_params); -out_unbind_drv: - wd_alg_drv_unbind(wd_comp_setting.driver); out_dlclose: wd_comp_close_driver(WD_TYPE_V2); out_uninit: + free(adapter); wd_alg_clear_init(&wd_comp_setting.status); return ret; } void wd_comp_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_comp_uninit_nolock(); if (ret) return; - wd_alg_attrs_uninit(&wd_comp_init_attrs); - wd_alg_drv_unbind(wd_comp_setting.driver); + for (int i = 0; i < wd_comp_setting.adapter->workers_nb; i++) { + worker = &wd_comp_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_comp_setting.adapter); wd_comp_close_driver(WD_TYPE_V2); wd_comp_setting.dlh_list = NULL; wd_alg_clear_init(&wd_comp_setting.status); } -struct wd_comp_msg *wd_comp_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_comp_setting.pool, idx, tag); -} - -int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_comp_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_comp_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_comp_msg resp_msg; struct wd_comp_msg *msg; @@ -368,16 +392,32 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_comp_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + pthread_mutex_lock(&worker->mutex); + if (worker->async_recv > 0) { + *count = worker->async_recv > expt ? expt : worker->async_recv; + worker->async_recv -= *count; + } + pthread_mutex_unlock(&worker->mutex); + return 0; + } + + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (unlikely(ret)) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_comp_setting.driver, ctx->ctx, &resp_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &resp_msg); if (unlikely(ret < 0)) { if (ret == -WD_HW_EACCESS) WD_ERR("wd comp recv hw error!\n"); @@ -386,7 +426,7 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) recv_count++; - msg = wd_find_msg_in_pool(&wd_comp_setting.pool, idx, + msg = wd_find_msg_in_pool(&worker->pool, idx, resp_msg.tag); if (unlikely(!msg)) { WD_ERR("failed to find msg from pool!\n"); @@ -399,13 +439,18 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) req->cb(req, req->cb_param); /* free msg cache to msg_pool */ - wd_put_msg_to_pool(&wd_comp_setting.pool, idx, resp_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, resp_msg.tag); *count = recv_count; } while (--tmp); return ret; } +int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +{ + return wd_comp_poll_ctx_(NULL, idx, expt, count); +} + static int wd_comp_check_sess_params(struct wd_comp_sess_setup *setup) { if (setup->alg_type >= WD_COMP_ALG_MAX) { @@ -436,8 +481,10 @@ static int wd_comp_check_sess_params(struct wd_comp_sess_setup *setup) handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) { + struct uadk_adapter_worker *worker; struct wd_comp_sess *sess; - int ret; + int nb = wd_comp_setting.adapter->workers_nb; + int ret, i; if (!setup) return (handle_t)0; @@ -454,22 +501,35 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) if (!sess->ctx_buf) goto sess_err; + worker = sess->worker = &wd_comp_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; sess->alg_type = setup->alg_type; sess->comp_lv = setup->comp_lv; sess->win_sz = setup->win_sz; sess->stream_pos = WD_COMP_STREAM_NEW; + sess->adapter_mode = wd_comp_setting.adapter->mode; - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_comp_setting.sched.sched_init( - wd_comp_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto sched_err; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_comp_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto sched_err; + } } return (handle_t)sess; sched_err: + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); + free(sess->sched_key); + } free(sess->ctx_buf); sess_err: free(sess); @@ -486,8 +546,11 @@ void wd_comp_free_sess(handle_t h_sess) if (sess->ctx_buf) free(sess->ctx_buf); - if (sess->sched_key) + if (sess->sched_key) { + for (int i = 0; i < wd_comp_setting.adapter->workers_nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); } @@ -578,11 +641,6 @@ static int wd_comp_check_params(struct wd_comp_sess *sess, return -WD_EINVAL; } - if (unlikely(mode == CTX_MODE_SYNC && req->cb)) { - WD_ERR("invalid: sync comp cb should be NULL!\n"); - return -WD_EINVAL; - } - return 0; } @@ -590,31 +648,46 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, struct wd_comp_req *req, struct wd_comp_msg *msg) { - struct wd_ctx_config_internal *config = &wd_comp_setting.config; - handle_t h_sched_ctx = wd_comp_setting.sched.h_sched_ctx; + struct uadk_adapter_worker *worker; struct wd_msg_handle msg_handle; struct wd_ctx_internal *ctx; __u32 idx; int ret; - idx = wd_comp_setting.sched.pick_next_ctx(h_sched_ctx, - sess->sched_key, - CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + worker = wd_comp_get_worker(sess, req); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (unlikely(ret)) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; - msg_handle.send = wd_comp_setting.driver->send; - msg_handle.recv = wd_comp_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_comp_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, config->epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + msg, NULL, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); + if (ret) { + wd_comp_switch_worker(sess, 1); + sess->worker_looptime++; + return ret; + } + + if ((sess->worker_looptime != 0) || + (wd_comp_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) { + sess->worker_looptime++; + } + + if (sess->worker_looptime >= wd_comp_setting.adapter->looptime) + wd_comp_switch_worker(sess, 0); + return ret; } @@ -831,9 +904,8 @@ int wd_do_comp_strm(handle_t h_sess, struct wd_comp_req *req) int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) { - struct wd_ctx_config_internal *config = &wd_comp_setting.config; struct wd_comp_sess *sess = (struct wd_comp_sess *)h_sess; - handle_t h_sched_ctx = wd_comp_setting.sched.h_sched_ctx; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_comp_msg *msg; int tag, ret; @@ -848,16 +920,28 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) return -WD_EINVAL; } - idx = wd_comp_setting.sched.pick_next_ctx(h_sched_ctx, - sess->sched_key, - CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + worker = wd_comp_get_worker(sess, req); + + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + ret = wd_do_comp_sync(h_sess, req); + if (!ret) { + pthread_mutex_lock(&worker->mutex); + worker->async_recv++; + pthread_mutex_unlock(&worker->mutex); + req->cb(req, req->cb_param); + } + return ret; + } + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (unlikely(ret)) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - tag = wd_get_msg_from_pool(&wd_comp_setting.pool, idx, (void **)&msg); + tag = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(tag < 0)) { WD_ERR("failed to get msg from pool!\n"); return tag; @@ -866,39 +950,63 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) msg->tag = tag; msg->stream_mode = WD_COMP_STATELESS; - ret = wd_alg_driver_send(wd_comp_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret < 0)) { WD_ERR("wd comp send error, ret = %d!\n", ret); goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_comp_env_config, idx); if (unlikely(ret)) goto fail_with_msg; + if ((sess->worker_looptime != 0) || + (wd_comp_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) + sess->worker_looptime++; + + if (sess->worker_looptime >= wd_comp_setting.adapter->looptime) + wd_comp_switch_worker(sess, 0); + return 0; fail_with_msg: - wd_put_msg_to_pool(&wd_comp_setting.pool, idx, msg->tag); - + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); + wd_comp_switch_worker(sess, 1); + sess->worker_looptime++; return ret; } int wd_comp_poll(__u32 expt, __u32 *count) { - handle_t h_sched_ctx; - struct wd_sched *sched; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!count)) { WD_ERR("invalid: comp poll count is NULL!\n"); return -WD_EINVAL; } - h_sched_ctx = wd_comp_setting.sched.h_sched_ctx; - sched = &wd_comp_setting.sched; + for (int i = 0; i < wd_comp_setting.adapter->workers_nb; i++) { + worker = &wd_comp_setting.adapter->workers[i]; - return sched->poll_policy(h_sched_ctx, expt, count); + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + + return ret; } static const struct wd_config_variable table[] = { @@ -983,3 +1091,8 @@ int wd_comp_get_env_param(__u32 node, __u32 type, __u32 mode, return wd_alg_get_env_param(&wd_comp_env_config, ctx_attr, num, is_enable); } + +void wd_comp_set_adapter_mode(enum uadk_adapter_mode mode) +{ + wd_comp_setting.adapter->mode = mode; +} diff --git a/wd_dh.c b/wd_dh.c index 043c3be95..c3209c6fb 100644 --- a/wd_dh.c +++ b/wd_dh.c @@ -13,6 +13,7 @@ #include #include "include/drv/wd_dh_drv.h" +#include "adapter.h" #include "wd_util.h" #define DH_MAX_KEY_SIZE 512 @@ -25,17 +26,17 @@ struct wd_dh_sess { __u32 key_size; struct wd_dtb g; struct wd_dh_sess_setup setup; - void *sched_key; + void **sched_key; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; static struct wd_dh_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_dh_setting; struct wd_env_config wd_dh_env_config; @@ -52,19 +53,15 @@ static void wd_dh_close_driver(int init_type) if (!wd_dh_setting.dlhandle) return; - wd_release_drv(wd_dh_setting.driver); dlclose(wd_dh_setting.dlhandle); wd_dh_setting.dlhandle = NULL; #else - wd_release_drv(wd_dh_setting.driver); hisi_hpre_remove(); #endif } static int wd_dh_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "dh"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -98,15 +95,6 @@ static int wd_dh_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_dh_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_dh_setting.driver = driver; - return WD_SUCCESS; } @@ -115,67 +103,67 @@ static void wd_dh_clear_status(void) wd_alg_clear_init(&wd_dh_setting.status); } -static int wd_dh_common_init(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_dh_common_init(struct uadk_adapter_worker *worker, + struct wd_sched *sched) { int ret; - ret = wd_set_epoll_en("WD_DH_EPOLL_EN", - &wd_dh_setting.config.epoll_en); + ret = wd_set_epoll_en("WD_DH_EPOLL_EN", &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_dh_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret) return ret; - ret = wd_init_sched(&wd_dh_setting.sched, sched); - if (ret) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; /* initialize async request pool */ - ret = wd_init_async_request_pool(&wd_dh_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_dh_msg)); if (ret) - goto out_clear_sched; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_dh_setting.config, - wd_dh_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return WD_SUCCESS; out_clear_pool: - wd_uninit_async_request_pool(&wd_dh_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_dh_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_dh_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } static int wd_dh_common_uninit(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_dh_setting.status, &status); if (status == WD_UNINIT) return -WD_EINVAL; - /* uninit async request pool */ - wd_uninit_async_request_pool(&wd_dh_setting.pool); + for (int i = 0; i < wd_dh_setting.adapter->workers_nb; i++) { + worker = &wd_dh_setting.adapter->workers[i]; - /* unset config, sched, driver */ - wd_clear_sched(&wd_dh_setting.sched); - wd_alg_uninit_driver(&wd_dh_setting.config, - wd_dh_setting.driver); + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return WD_SUCCESS; } int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "dh"; int ret; pthread_atfork(NULL, NULL, wd_dh_clear_status); @@ -188,11 +176,24 @@ int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_dh_setting.adapter = adapter; + ret = wd_dh_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_dh_common_init(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_dh_common_init(worker, sched); if (ret) goto out_close_driver; @@ -203,6 +204,7 @@ int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_dh_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_dh_setting.status); return ret; } @@ -215,6 +217,7 @@ void wd_dh_uninit(void) if (ret) return; + free(wd_dh_setting.adapter); wd_dh_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_dh_setting.status); } @@ -222,8 +225,11 @@ void wd_dh_uninit(void) int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params) { struct wd_ctx_nums dh_ctx_num[WD_DH_PHASE2] = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; struct wd_ctx_params dh_ctx_params = {0}; int state, ret = -WD_EINVAL; + int i; pthread_atfork(NULL, NULL, wd_dh_clear_status); @@ -242,78 +248,69 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param goto out_clear_init; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + wd_dh_setting.adapter = adapter; + state = wd_dh_open_driver(WD_TYPE_V2); if (state) goto out_clear_init; - while (ret) { - memset(&wd_dh_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlopen; - /* Get alg driver and dev name */ - wd_dh_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_dh_setting.driver) { - WD_ERR("fail to bind a valid driver.\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; dh_ctx_params.ctx_set_num = dh_ctx_num; ret = wd_ctx_param_init(&dh_ctx_params, ctx_params, - wd_dh_setting.driver, WD_DH_TYPE, WD_DH_PHASE2); + worker->driver, WD_DH_TYPE, WD_DH_PHASE2); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_dh_setting.driver); - wd_alg_drv_unbind(wd_dh_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlopen; } wd_dh_init_attrs.alg = alg; - wd_dh_init_attrs.sched_type = sched_type; - wd_dh_init_attrs.driver = wd_dh_setting.driver; wd_dh_init_attrs.ctx_params = &dh_ctx_params; wd_dh_init_attrs.alg_init = wd_dh_common_init; - wd_dh_init_attrs.alg_poll_ctx = wd_dh_poll_ctx; - ret = wd_alg_attrs_init(&wd_dh_init_attrs); + wd_dh_init_attrs.alg_poll_ctx = wd_dh_poll_ctx_; + ret = wd_alg_attrs_init(worker, &wd_dh_init_attrs); + wd_ctx_param_uninit(&dh_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_dh_setting.driver); - wd_alg_drv_unbind(wd_dh_setting.driver); - wd_ctx_param_uninit(&dh_ctx_params); - continue; - } WD_ERR("failed to init alg attrs!\n"); - goto out_params_uninit; + goto out_dlopen; } } wd_alg_set_init(&wd_dh_setting.status); - wd_ctx_param_uninit(&dh_ctx_params); return WD_SUCCESS; -out_params_uninit: - wd_ctx_param_uninit(&dh_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_dh_setting.driver); out_dlopen: wd_dh_close_driver(WD_TYPE_V2); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_dh_setting.status); return ret; } void wd_dh_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_dh_common_uninit(); if (ret) return; - wd_alg_attrs_uninit(&wd_dh_init_attrs); - wd_alg_drv_unbind(wd_dh_setting.driver); + for (int i = 0; i < wd_dh_setting.adapter->workers_nb; i++) { + worker = &wd_dh_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_dh_setting.adapter); wd_dh_close_driver(WD_TYPE_V2); wd_dh_setting.dlh_list = NULL; wd_alg_clear_init(&wd_dh_setting.status); @@ -350,11 +347,10 @@ static int fill_dh_msg(struct wd_dh_msg *msg, struct wd_dh_req *req, return WD_SUCCESS; } -int wd_do_dh_sync(handle_t sess, struct wd_dh_req *req) +int wd_do_dh_sync(handle_t h_sess, struct wd_dh_req *req) { - struct wd_ctx_config_internal *config = &wd_dh_setting.config; - handle_t h_sched_ctx = wd_dh_setting.sched.h_sched_ctx; - struct wd_dh_sess *sess_t = (struct wd_dh_sess *)sess; + struct wd_dh_sess *sess = (struct wd_dh_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_msg_handle msg_handle; struct wd_ctx_internal *ctx; struct wd_dh_msg msg; @@ -366,27 +362,31 @@ int wd_do_dh_sync(handle_t sess, struct wd_dh_req *req) return -WD_EINVAL; } - idx = wd_dh_setting.sched.pick_next_ctx(h_sched_ctx, - sess_t->sched_key, - CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (ret) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; memset(&msg, 0, sizeof(struct wd_dh_msg)); - ret = fill_dh_msg(&msg, req, sess_t); + ret = fill_dh_msg(&msg, req, sess); if (unlikely(ret)) return ret; - msg_handle.send = wd_dh_setting.driver->send; - msg_handle.recv = wd_dh_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_dh_setting.driver, &msg_handle, ctx->ctx, - &msg, &balance, wd_dh_setting.config.epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + &msg, &balance, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); if (unlikely(ret)) return ret; @@ -397,11 +397,10 @@ int wd_do_dh_sync(handle_t sess, struct wd_dh_req *req) return GET_NEGATIVE(msg.result); } -int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) +int wd_do_dh_async(handle_t h_sess, struct wd_dh_req *req) { - struct wd_ctx_config_internal *config = &wd_dh_setting.config; - handle_t h_sched_ctx = wd_dh_setting.sched.h_sched_ctx; - struct wd_dh_sess *sess_t = (struct wd_dh_sess *)sess; + struct wd_dh_sess *sess = (struct wd_dh_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_dh_msg *msg = NULL; struct wd_ctx_internal *ctx; int ret, mid; @@ -412,16 +411,20 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) return -WD_EINVAL; } - idx = wd_dh_setting.sched.pick_next_ctx(h_sched_ctx, - sess_t->sched_key, - CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - mid = wd_get_msg_from_pool(&wd_dh_setting.pool, idx, (void **)&msg); + mid = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(mid < 0)) { WD_ERR("failed to get msg from pool!\n"); return mid; @@ -432,7 +435,7 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) goto fail_with_msg; msg->tag = mid; - ret = wd_alg_driver_send(wd_dh_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send dh BD, hw is err!\n"); @@ -440,7 +443,7 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_dh_env_config, idx); if (ret) goto fail_with_msg; @@ -448,19 +451,14 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) return WD_SUCCESS; fail_with_msg: - wd_put_msg_to_pool(&wd_dh_setting.pool, idx, mid); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); return ret; } -struct wd_dh_msg *wd_dh_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_dh_setting.pool, idx, tag); -} - -int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_dh_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_dh_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_dh_msg rcv_msg; struct wd_dh_req *req; @@ -474,27 +472,33 @@ int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_dh_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_dh_setting.driver, ctx->ctx, &rcv_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &rcv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (unlikely(ret)) { WD_ERR("failed to async recv, ret = %d!\n", ret); *count = rcv_cnt; - wd_put_msg_to_pool(&wd_dh_setting.pool, idx, + wd_put_msg_to_pool(&worker->pool, idx, rcv_msg.tag); return ret; } rcv_cnt++; - msg = wd_find_msg_in_pool(&wd_dh_setting.pool, + msg = wd_find_msg_in_pool(&worker->pool, idx, rcv_msg.tag); if (!msg) { WD_ERR("failed to find msg!\n"); @@ -505,23 +509,47 @@ int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) msg->req.status = rcv_msg.result; req = &msg->req; req->cb(req); - wd_put_msg_to_pool(&wd_dh_setting.pool, idx, rcv_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, rcv_msg.tag); *count = rcv_cnt; } while (--tmp); return ret; } +int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +{ + return wd_dh_poll_ctx_(NULL, idx, expt, count); +} + int wd_dh_poll(__u32 expt, __u32 *count) { - handle_t h_sched_ctx = wd_dh_setting.sched.h_sched_ctx; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!count)) { WD_ERR("invalid: dh poll count is NULL!\n"); return -WD_EINVAL; } - return wd_dh_setting.sched.poll_policy(h_sched_ctx, expt, count); + for (int i = 0; i < wd_dh_setting.adapter->workers_nb; i++) { + worker = &wd_dh_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + return ret; } int wd_dh_get_mode(handle_t sess, __u8 *alg_mode) @@ -581,7 +609,10 @@ void wd_dh_get_g(handle_t sess, struct wd_dtb **g) handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) { + struct uadk_adapter_worker *worker; struct wd_dh_sess *sess; + int nb = wd_dh_setting.adapter->workers_nb; + int i; if (!setup) { WD_ERR("invalid: alloc dh sess setup NULL!\n"); @@ -604,6 +635,9 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) return (handle_t)0; memset(sess, 0, sizeof(struct wd_dh_sess)); + worker = sess->worker = &wd_dh_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; memcpy(&sess->setup, setup, sizeof(*setup)); sess->key_size = setup->key_bits >> BYTE_BITS_SHIFT; @@ -612,12 +646,16 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) goto sess_err; sess->g.bsize = sess->key_size; - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_dh_setting.sched.sched_init( - wd_dh_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto sched_err; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_dh_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto sched_err; + } } return (handle_t)sess; @@ -625,6 +663,11 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) sched_err: free(sess->g.data); sess_err: + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); + free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -641,8 +684,11 @@ void wd_dh_free_sess(handle_t sess) if (sess_t->g.data) free(sess_t->g.data); - if (sess_t->sched_key) + if (sess_t->sched_key) { + for (int i = 0; i < wd_dh_setting.adapter->workers_nb; i++) + free(sess_t->sched_key[i]); free(sess_t->sched_key); + } free(sess_t); } diff --git a/wd_digest.c b/wd_digest.c index 58f621af6..7948e0a6e 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -9,6 +9,7 @@ #include #include "include/drv/wd_digest_drv.h" #include "wd_digest.h" +#include "adapter.h" #define GMAC_IV_LEN 16 #define MAX_BLOCK_SIZE 128 @@ -38,12 +39,9 @@ static char *wd_digest_alg_name[WD_DIGEST_TYPE_MAX] = { struct wd_digest_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_alg_driver *driver; - struct wd_async_msg_pool pool; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_digest_setting; struct wd_digest_stream_data { @@ -66,10 +64,44 @@ struct wd_digest_sess { void *priv; unsigned char key[MAX_HMAC_KEY_SIZE]; __u32 key_bytes; - void *sched_key; + void **sched_key; struct wd_digest_stream_data stream_data; + struct uadk_adapter_worker *worker; + int worker_looptime; + enum uadk_adapter_mode adapter_mode; }; +static void wd_digest_switch_worker(struct wd_digest_sess *sess, int para) +{ + struct uadk_adapter_worker *worker; + + worker = uadk_adapter_switch_worker(wd_digest_setting.adapter, + sess->worker, para); + if (worker) + sess->worker = worker; + sess->worker_looptime = 0; +} + +static struct uadk_adapter_worker * +wd_digest_get_worker(struct wd_digest_sess *sess, struct wd_digest_req *req) +{ + if (sess->adapter_mode != wd_digest_setting.adapter->mode) { + sess->worker = &wd_digest_setting.adapter->workers[0]; + sess->worker_looptime = 0; + sess->adapter_mode = wd_digest_setting.adapter->mode; + } + + if (sess->adapter_mode == UADK_ADAPT_MODE_THRESHOLD && + wd_digest_setting.adapter->workers_nb == 2) { + if (req->in_bytes >= wd_digest_setting.adapter->threshold) + sess->worker = &wd_digest_setting.adapter->workers[0]; + else + sess->worker = &wd_digest_setting.adapter->workers[1]; + } + + return sess->worker; +} + struct wd_env_config wd_digest_env_config; static struct wd_init_attrs wd_digest_init_attrs; @@ -82,20 +114,16 @@ static void wd_digest_close_driver(int init_type) } if (wd_digest_setting.dlhandle) { - wd_release_drv(wd_digest_setting.driver); dlclose(wd_digest_setting.dlhandle); wd_digest_setting.dlhandle = NULL; } #else - wd_release_drv(wd_digest_setting.driver); hisi_sec2_remove(); #endif } static int wd_digest_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "sm3"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -129,14 +157,6 @@ static int wd_digest_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_digest_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_digest_setting.driver = driver; return WD_SUCCESS; } @@ -189,7 +209,9 @@ int wd_digest_set_key(handle_t h_sess, const __u8 *key, __u32 key_len) handle_t wd_digest_alloc_sess(struct wd_digest_sess_setup *setup) { struct wd_digest_sess *sess = NULL; - bool ret; + struct uadk_adapter_worker *worker; + int nb = wd_digest_setting.adapter->workers_nb; + int ret, i; if (unlikely(!setup)) { WD_ERR("failed to check alloc sess param!\n"); @@ -206,27 +228,40 @@ handle_t wd_digest_alloc_sess(struct wd_digest_sess_setup *setup) return (handle_t)0; memset(sess, 0, sizeof(struct wd_digest_sess)); + worker = sess->worker = &wd_digest_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; sess->alg_name = wd_digest_alg_name[setup->alg]; sess->alg = setup->alg; sess->mode = setup->mode; - ret = wd_drv_alg_support(sess->alg_name, wd_digest_setting.driver); + sess->adapter_mode = wd_digest_setting.adapter->mode; + + ret = wd_drv_alg_support(sess->alg_name, worker->driver); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); goto err_sess; } - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_digest_setting.sched.sched_init( - wd_digest_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto err_sess; + + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_digest_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto err_sess; + } } return (handle_t)sess; err_sess: - if (sess->sched_key) + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -241,8 +276,11 @@ void wd_digest_free_sess(handle_t h_sess) } wd_memset_zero(sess->key, sess->key_bytes); - if (sess->sched_key) + if (sess->sched_key) { + for (int i = 0; i < wd_digest_setting.adapter->workers_nb; i++) + free(sess->sched_key[i]); free(sess->sched_key); + } free(sess); } @@ -251,50 +289,50 @@ static void wd_digest_clear_status(void) wd_alg_clear_init(&wd_digest_setting.status); } -static int wd_digest_init_nolock(struct wd_ctx_config *config, +static int wd_digest_init_nolock(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; ret = wd_set_epoll_en("WD_DIGEST_EPOLL_EN", - &wd_digest_setting.config.epoll_en); + &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_digest_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_digest_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; /* allocate async pool for every ctx */ - ret = wd_init_async_request_pool(&wd_digest_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_digest_msg)); if (ret < 0) - goto out_clear_sched; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_digest_setting.config, - wd_digest_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return 0; out_clear_pool: - wd_uninit_async_request_pool(&wd_digest_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_digest_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_digest_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } int wd_digest_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "sm3"; int ret; pthread_atfork(NULL, NULL, wd_digest_clear_status); @@ -307,11 +345,23 @@ int wd_digest_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_digest_setting.adapter = adapter; + ret = wd_digest_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_digest_init_nolock(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + ret = wd_digest_init_nolock(worker, sched); if (ret) goto out_close_driver; @@ -322,6 +372,7 @@ int wd_digest_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_digest_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_digest_setting.status); return ret; } @@ -334,10 +385,13 @@ static int wd_digest_uninit_nolock(void) if (status == WD_UNINIT) return -WD_EINVAL; - wd_uninit_async_request_pool(&wd_digest_setting.pool); - wd_clear_sched(&wd_digest_setting.sched); - wd_alg_uninit_driver(&wd_digest_setting.config, - wd_digest_setting.driver); + for (int i = 0; i < wd_digest_setting.adapter->workers_nb; i++) { + struct uadk_adapter_worker *worker = &wd_digest_setting.adapter->workers[i]; + + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } + return 0; } @@ -349,6 +403,7 @@ void wd_digest_uninit(void) if (ret) return; + free(wd_digest_setting.adapter); wd_digest_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_digest_setting.status); } @@ -368,7 +423,10 @@ int wd_digest_init2_(char *alg, __u32 sched_type, int task_type, { struct wd_ctx_params digest_ctx_params = {0}; struct wd_ctx_nums digest_ctx_num = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; + int i; pthread_atfork(NULL, NULL, wd_digest_clear_status); @@ -387,77 +445,68 @@ int wd_digest_init2_(char *alg, __u32 sched_type, int task_type, goto out_uninit; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_uninit; + wd_digest_setting.adapter = adapter; + state = wd_digest_open_driver(WD_TYPE_V2); if (state) goto out_uninit; + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlclose; - while (ret != 0) { - memset(&wd_digest_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_digest_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_digest_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; digest_ctx_params.ctx_set_num = &digest_ctx_num; ret = wd_ctx_param_init(&digest_ctx_params, ctx_params, - wd_digest_setting.driver, WD_DIGEST_TYPE, 1); + worker->driver, WD_DIGEST_TYPE, 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_digest_setting.driver); - wd_alg_drv_unbind(wd_digest_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlclose; } wd_digest_init_attrs.alg = alg; - wd_digest_init_attrs.sched_type = sched_type; - wd_digest_init_attrs.driver = wd_digest_setting.driver; wd_digest_init_attrs.ctx_params = &digest_ctx_params; wd_digest_init_attrs.alg_init = wd_digest_init_nolock; - wd_digest_init_attrs.alg_poll_ctx = wd_digest_poll_ctx; - ret = wd_alg_attrs_init(&wd_digest_init_attrs); + wd_digest_init_attrs.alg_poll_ctx = wd_digest_poll_ctx_; + ret = wd_alg_attrs_init(worker, &wd_digest_init_attrs); + wd_ctx_param_uninit(&digest_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_digest_setting.driver); - wd_alg_drv_unbind(wd_digest_setting.driver); - wd_ctx_param_uninit(&digest_ctx_params); - continue; - } - WD_ERR("failed to init alg attrs.\n"); - goto out_params_uninit; + WD_ERR("fail to init alg attrs.\n"); + goto out_dlclose; } } - wd_alg_set_init(&wd_digest_setting.status); - wd_ctx_param_uninit(&digest_ctx_params); + wd_alg_set_init(&wd_digest_setting.status); return 0; -out_params_uninit: - wd_ctx_param_uninit(&digest_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_digest_setting.driver); -out_dlopen: +out_dlclose: wd_digest_close_driver(WD_TYPE_V2); out_uninit: + free(adapter); wd_alg_clear_init(&wd_digest_setting.status); return ret; } void wd_digest_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_digest_uninit_nolock(); if (ret) return; - wd_alg_attrs_uninit(&wd_digest_init_attrs); - wd_alg_drv_unbind(wd_digest_setting.driver); + for (int i = 0; i < wd_digest_setting.adapter->workers_nb; i++) { + worker = &wd_digest_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_digest_setting.adapter); wd_digest_close_driver(WD_TYPE_V2); wd_digest_setting.dlh_list = NULL; wd_alg_clear_init(&wd_digest_setting.status); @@ -608,19 +657,19 @@ static void fill_request_msg(struct wd_digest_msg *msg, msg->iv_bytes = sess->stream_data.msg_state; } -static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_digest_sess *dsess, - struct wd_digest_msg *msg) +static int send_recv_sync(struct uadk_adapter_worker *worker, struct wd_ctx_internal *ctx, + struct wd_digest_sess *dsess, struct wd_digest_msg *msg) { struct wd_msg_handle msg_handle; int ret; - msg_handle.send = wd_digest_setting.driver->send; - msg_handle.recv = wd_digest_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; - wd_ctx_spin_lock(ctx, wd_digest_setting.driver->calc_type); - ret = wd_handle_msg_sync(wd_digest_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_digest_setting.config.epoll_en); - wd_ctx_spin_unlock(ctx, wd_digest_setting.driver->calc_type); + wd_ctx_spin_lock(ctx, worker->driver->calc_type); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, + msg, NULL, worker->config.epoll_en); + wd_ctx_spin_unlock(ctx, worker->driver->calc_type); if (unlikely(ret)) return ret; @@ -646,8 +695,8 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_digest_sess *ds int wd_do_digest_sync(handle_t h_sess, struct wd_digest_req *req) { - struct wd_ctx_config_internal *config = &wd_digest_setting.config; struct wd_digest_sess *dsess = (struct wd_digest_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_digest_msg msg; __u32 idx; @@ -657,29 +706,44 @@ int wd_do_digest_sync(handle_t h_sess, struct wd_digest_req *req) if (unlikely(ret)) return -WD_EINVAL; + worker = wd_digest_get_worker(dsess, req); + memset(&msg, 0, sizeof(struct wd_digest_msg)); fill_request_msg(&msg, req, dsess); req->state = 0; - idx = wd_digest_setting.sched.pick_next_ctx( - wd_digest_setting.sched.h_sched_ctx, - dsess->sched_key, CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + dsess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (unlikely(ret)) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; - ret = send_recv_sync(ctx, dsess, &msg); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; + ret = send_recv_sync(worker, ctx, dsess, &msg); req->state = msg.result; + if (ret) { + wd_digest_switch_worker(dsess, 1); + dsess->worker_looptime++; + return ret; + } + + if ((dsess->worker_looptime != 0) || + (wd_digest_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) + dsess->worker_looptime++; + + if (dsess->worker_looptime >= wd_digest_setting.adapter->looptime) + wd_digest_switch_worker(dsess, 0); + return ret; } int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) { - struct wd_ctx_config_internal *config = &wd_digest_setting.config; struct wd_digest_sess *dsess = (struct wd_digest_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_digest_msg *msg; int msg_id, ret; @@ -694,16 +758,29 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) return -WD_EINVAL; } - idx = wd_digest_setting.sched.pick_next_ctx( - wd_digest_setting.sched.h_sched_ctx, - dsess->sched_key, CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + worker = wd_digest_get_worker(dsess, req); + + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + ret = wd_do_digest_sync(h_sess, req); + if (!ret) { + pthread_mutex_lock(&worker->mutex); + worker->async_recv++; + pthread_mutex_unlock(&worker->mutex); + req->cb(req); + } + return ret; + } + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + dsess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - msg_id = wd_get_msg_from_pool(&wd_digest_setting.pool, idx, + msg_id = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(msg_id < 0)) { WD_ERR("failed to get msg from pool!\n"); @@ -713,7 +790,7 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) fill_request_msg(msg, req, dsess); msg->tag = msg_id; - ret = wd_alg_driver_send(wd_digest_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("failed to send BD, hw is err!\n"); @@ -721,26 +798,30 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_digest_env_config, idx); if (ret) goto fail_with_msg; + if ((dsess->worker_looptime != 0) || + (wd_digest_setting.adapter->mode == UADK_ADAPT_MODE_ROUNDROBIN)) + dsess->worker_looptime++; + + if (dsess->worker_looptime >= wd_digest_setting.adapter->looptime) + wd_digest_switch_worker(dsess, 0); + return 0; fail_with_msg: - wd_put_msg_to_pool(&wd_digest_setting.pool, idx, msg->tag); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); + wd_digest_switch_worker(dsess, 1); + dsess->worker_looptime++; return ret; } -struct wd_digest_msg *wd_digest_get_msg(__u32 idx, __u32 tag) -{ - return wd_find_msg_in_pool(&wd_digest_setting.pool, idx, tag); -} - -int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +int wd_digest_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - struct wd_ctx_config_internal *config = &wd_digest_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_digest_msg recv_msg, *msg; struct wd_digest_req *req; @@ -748,6 +829,12 @@ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) __u32 tmp = expt; int ret; + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_digest_setting.adapter->workers[0]; + else + worker = sched->worker; + if (unlikely(!count || !expt)) { WD_ERR("invalid: digest poll ctx input param is NULL!\n"); return -WD_EINVAL; @@ -755,14 +842,24 @@ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + if (worker->driver->mode == UADK_DRV_SYNCONLY) { + pthread_mutex_lock(&worker->mutex); + if (worker->async_recv > 0) { + *count = worker->async_recv > expt ? expt : worker->async_recv; + worker->async_recv -= *count; + } + pthread_mutex_unlock(&worker->mutex); + return 0; + } + + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_digest_setting.driver, ctx->ctx, &recv_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { @@ -772,7 +869,7 @@ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) recv_cnt++; - msg = wd_find_msg_in_pool(&wd_digest_setting.pool, idx, + msg = wd_find_msg_in_pool(&worker->pool, idx, recv_msg.tag); if (!msg) { WD_ERR("failed to find msg from pool!\n"); @@ -784,7 +881,7 @@ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) if (likely(req)) req->cb(req); - wd_put_msg_to_pool(&wd_digest_setting.pool, idx, + wd_put_msg_to_pool(&worker->pool, idx, recv_msg.tag); *count = recv_cnt; } while (--tmp); @@ -792,17 +889,40 @@ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return ret; } +int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +{ + return wd_digest_poll_ctx_(NULL, idx, expt, count); +} + int wd_digest_poll(__u32 expt, __u32 *count) { - handle_t h_ctx = wd_digest_setting.sched.h_sched_ctx; - struct wd_sched *sched = &wd_digest_setting.sched; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!count)) { WD_ERR("invalid: digest poll input param is NULL!\n"); return -WD_EINVAL; } - return sched->poll_policy(h_ctx, expt, count); + for (int i = 0; i < wd_digest_setting.adapter->workers_nb; i++) { + worker = &wd_digest_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + return ret; } static const struct wd_config_variable table[] = { @@ -868,3 +988,8 @@ int wd_digest_get_env_param(__u32 node, __u32 type, __u32 mode, return wd_alg_get_env_param(&wd_digest_env_config, ctx_attr, num, is_enable); } + +void wd_digest_set_adapter_mode(enum uadk_adapter_mode mode) +{ + wd_digest_setting.adapter->mode = mode; +} diff --git a/wd_ecc.c b/wd_ecc.c index 292338a6e..25738350b 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -15,6 +15,7 @@ #include "include/drv/wd_ecc_drv.h" #include "include/wd_ecc_curve.h" +#include "adapter.h" #include "wd_ecc.h" #define ECC_MAX_HW_BITS 521 @@ -49,7 +50,10 @@ struct wd_ecc_sess { __u32 key_size; struct wd_ecc_key key; struct wd_ecc_sess_setup setup; - void *sched_key; + void **sched_key; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; struct wd_ecc_curve_list { @@ -61,12 +65,9 @@ struct wd_ecc_curve_list { static struct wd_ecc_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_ecc_setting; struct wd_env_config wd_ecc_env_config; @@ -106,19 +107,15 @@ static void wd_ecc_close_driver(int init_type) if (!wd_ecc_setting.dlhandle) return; - wd_release_drv(wd_ecc_setting.driver); dlclose(wd_ecc_setting.dlhandle); wd_ecc_setting.dlhandle = NULL; #else - wd_release_drv(wd_ecc_setting.driver); hisi_hpre_remove(); #endif } static int wd_ecc_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "sm2"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -152,15 +149,6 @@ static int wd_ecc_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_ecc_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_ecc_setting.driver = driver; - return WD_SUCCESS; } @@ -179,66 +167,66 @@ static void wd_ecc_clear_status(void) wd_alg_clear_init(&wd_ecc_setting.status); } -static int wd_ecc_common_init(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_ecc_common_init(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; ret = wd_set_epoll_en("WD_ECC_EPOLL_EN", - &wd_ecc_setting.config.epoll_en); + &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_ecc_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_ecc_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; - ret = wd_init_async_request_pool(&wd_ecc_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_ecc_msg)); if (ret < 0) - goto out_clear_sched; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_ecc_setting.config, - wd_ecc_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return WD_SUCCESS; out_clear_pool: - wd_uninit_async_request_pool(&wd_ecc_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_ecc_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_ecc_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } static int wd_ecc_common_uninit(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_ecc_setting.status, &status); if (status == WD_UNINIT) return -WD_EINVAL; - /* uninit async request pool */ - wd_uninit_async_request_pool(&wd_ecc_setting.pool); + for (int i = 0; i < wd_ecc_setting.adapter->workers_nb; i++) { + worker = &wd_ecc_setting.adapter->workers[i]; - /* unset config, sched, driver */ - wd_clear_sched(&wd_ecc_setting.sched); - wd_alg_uninit_driver(&wd_ecc_setting.config, - wd_ecc_setting.driver); + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return WD_SUCCESS; } int wd_ecc_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "sm2"; int ret; pthread_atfork(NULL, NULL, wd_ecc_clear_status); @@ -251,11 +239,24 @@ int wd_ecc_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_ecc_setting.adapter = adapter; + ret = wd_ecc_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_ecc_common_init(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_ecc_common_init(worker, sched); if (ret) goto out_close_driver; @@ -266,6 +267,7 @@ int wd_ecc_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_ecc_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_ecc_setting.status); return ret; } @@ -278,6 +280,7 @@ void wd_ecc_uninit(void) if (ret) return; + free(wd_ecc_setting.adapter); wd_ecc_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_ecc_setting.status); } @@ -286,8 +289,11 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para { struct wd_ctx_nums ecc_ctx_num[WD_EC_OP_MAX] = {0}; struct wd_ctx_params ecc_ctx_params = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; bool flag; + int i; pthread_atfork(NULL, NULL, wd_ecc_clear_status); @@ -307,61 +313,46 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para goto out_clear_init; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + wd_ecc_setting.adapter = adapter; + state = wd_ecc_open_driver(WD_TYPE_V2); if (state) goto out_clear_init; - while (ret) { - memset(&wd_ecc_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlopen; - /* Get alg driver and dev name */ - wd_ecc_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_ecc_setting.driver) { - WD_ERR("failed to bind a valid driver!\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; ecc_ctx_params.ctx_set_num = ecc_ctx_num; - ret = wd_ctx_param_init(&ecc_ctx_params, ctx_params, - wd_ecc_setting.driver, WD_ECC_TYPE, WD_EC_OP_MAX); + ret = wd_ctx_param_init(&ecc_ctx_params, ctx_params, worker->driver, + WD_ECC_TYPE, WD_EC_OP_MAX); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_ecc_setting.driver); - wd_alg_drv_unbind(wd_ecc_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlopen; } wd_ecc_init_attrs.alg = alg; - wd_ecc_init_attrs.sched_type = sched_type; - wd_ecc_init_attrs.driver = wd_ecc_setting.driver; wd_ecc_init_attrs.ctx_params = &ecc_ctx_params; wd_ecc_init_attrs.alg_init = wd_ecc_common_init; - wd_ecc_init_attrs.alg_poll_ctx = wd_ecc_poll_ctx; - ret = wd_alg_attrs_init(&wd_ecc_init_attrs); + wd_ecc_init_attrs.alg_poll_ctx = wd_ecc_poll_ctx_; + ret = wd_alg_attrs_init(worker, &wd_ecc_init_attrs); + wd_ctx_param_uninit(&ecc_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_ecc_setting.driver); - wd_alg_drv_unbind(wd_ecc_setting.driver); - wd_ctx_param_uninit(&ecc_ctx_params); - continue; - } WD_ERR("failed to init alg attrs!\n"); - goto out_params_uninit; + goto out_dlopen; } } wd_alg_set_init(&wd_ecc_setting.status); - wd_ctx_param_uninit(&ecc_ctx_params); return WD_SUCCESS; -out_params_uninit: - wd_ctx_param_uninit(&ecc_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_ecc_setting.driver); out_dlopen: wd_ecc_close_driver(WD_TYPE_V2); out_clear_init: @@ -371,14 +362,19 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para void wd_ecc_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_ecc_common_uninit(); if (ret) return; - wd_alg_attrs_uninit(&wd_ecc_init_attrs); - wd_alg_drv_unbind(wd_ecc_setting.driver); + for (int i = 0; i < wd_ecc_setting.adapter->workers_nb; i++) { + worker = &wd_ecc_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_ecc_setting.adapter); wd_ecc_close_driver(WD_TYPE_V2); wd_ecc_setting.dlh_list = NULL; wd_alg_clear_init(&wd_ecc_setting.status); @@ -1168,22 +1164,28 @@ static void del_sess_key(struct wd_ecc_sess *sess) handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) { + struct uadk_adapter_worker *worker; struct wd_ecc_sess *sess; - int ret; + int nb = wd_ecc_setting.adapter->workers_nb; + int ret, i; if (setup_param_check(setup)) return (handle_t)0; - ret = wd_drv_alg_support(setup->alg, wd_ecc_setting.driver); - if (!ret) { - WD_ERR("failed to support this algorithm: %s!\n", setup->alg); - return (handle_t)0; - } - sess = calloc(1, sizeof(struct wd_ecc_sess)); if (!sess) return (handle_t)0; + worker = sess->worker = &wd_ecc_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; + + ret = wd_drv_alg_support(setup->alg, worker->driver); + if (!ret) { + WD_ERR("failed to support this algorithm: %s!\n", setup->alg); + goto sess_err; + } + memcpy(&sess->setup, setup, sizeof(*setup)); sess->key_size = BITS_TO_BYTES(setup->key_bits); @@ -1193,12 +1195,16 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) goto sess_err; } - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_ecc_setting.sched.sched_init( - wd_ecc_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto sched_err; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_ecc_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto sched_err; + } } return (handle_t)sess; @@ -1206,6 +1212,11 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) sched_err: del_sess_key(sess); sess_err: + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); + free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -1219,8 +1230,11 @@ void wd_ecc_free_sess(handle_t sess) return; } - if (sess_t->sched_key) + if (sess_t->sched_key) { + for (int i = 0; i < wd_ecc_setting.adapter->workers_nb; i++) + free(sess_t->sched_key[i]); free(sess_t->sched_key); + } del_sess_key(sess_t); free(sess_t); } @@ -1557,9 +1571,8 @@ static void msg_pack(char *dst, __u64 *out_len, int wd_do_ecc_sync(handle_t h_sess, struct wd_ecc_req *req) { - struct wd_ctx_config_internal *config = &wd_ecc_setting.config; - handle_t h_sched_ctx = wd_ecc_setting.sched.h_sched_ctx; struct wd_ecc_sess *sess = (struct wd_ecc_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_msg_handle msg_handle; struct wd_ctx_internal *ctx; struct wd_ecc_msg msg; @@ -1571,27 +1584,31 @@ int wd_do_ecc_sync(handle_t h_sess, struct wd_ecc_req *req) return -WD_EINVAL; } - idx = wd_ecc_setting.sched.pick_next_ctx(h_sched_ctx, - sess->sched_key, - CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (ret) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; memset(&msg, 0, sizeof(struct wd_ecc_msg)); ret = fill_ecc_msg(&msg, req, sess); if (unlikely(ret)) return ret; - msg_handle.send = wd_ecc_setting.driver->send; - msg_handle.recv = wd_ecc_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_ecc_setting.driver, &msg_handle, ctx->ctx, &msg, - &balance, wd_ecc_setting.config.epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, &msg, + &balance, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); if (unlikely(ret)) return ret; @@ -2243,11 +2260,10 @@ struct wd_ecc_in *wd_ecdsa_new_verf_in(handle_t sess, return new_verf_in(sess, dgst, r, s, NULL, 1); } -int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) +int wd_do_ecc_async(handle_t h_sess, struct wd_ecc_req *req) { - struct wd_ctx_config_internal *config = &wd_ecc_setting.config; - handle_t h_sched_ctx = wd_ecc_setting.sched.h_sched_ctx; - struct wd_ecc_sess *sess_t = (struct wd_ecc_sess *)sess; + struct wd_ecc_sess *sess = (struct wd_ecc_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_ecc_msg *msg = NULL; struct wd_ctx_internal *ctx; int ret, mid; @@ -2258,16 +2274,20 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) return -WD_EINVAL; } - idx = wd_ecc_setting.sched.pick_next_ctx(h_sched_ctx, - sess_t->sched_key, - CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - mid = wd_get_msg_from_pool(&wd_ecc_setting.pool, idx, (void **)&msg); + mid = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(mid < 0)) { WD_ERR("failed to get msg from pool!\n"); return mid; @@ -2278,7 +2298,7 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) goto fail_with_msg; msg->tag = mid; - ret = wd_alg_driver_send(wd_ecc_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send ecc BD, hw is err!\n"); @@ -2286,7 +2306,7 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_ecc_env_config, idx); if (ret) goto fail_with_msg; @@ -2294,18 +2314,13 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) return WD_SUCCESS; fail_with_msg: - wd_put_msg_to_pool(&wd_ecc_setting.pool, idx, mid); + wd_put_msg_to_pool(&worker->pool, idx, msg->tag); return ret; } -struct wd_ecc_msg *wd_ecc_get_msg(__u32 idx, __u32 tag) +int wd_ecc_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - return wd_find_msg_in_pool(&wd_ecc_setting.pool, idx, tag); -} - -int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) -{ - struct wd_ctx_config_internal *config = &wd_ecc_setting.config; + struct uadk_adapter_worker *worker; struct wd_ecc_msg recv_msg, *msg; struct wd_ctx_internal *ctx; struct wd_ecc_req *req; @@ -2318,28 +2333,33 @@ int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_ecc_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_ecc_setting.driver, ctx->ctx, &recv_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { WD_ERR("failed to async recv, ret = %d!\n", ret); *count = rcv_cnt; - wd_put_msg_to_pool(&wd_ecc_setting.pool, idx, + wd_put_msg_to_pool(&worker->pool, idx, recv_msg.tag); return ret; } rcv_cnt++; - msg = wd_find_msg_in_pool(&wd_ecc_setting.pool, idx, - recv_msg.tag); + msg = wd_find_msg_in_pool(&worker->pool, idx, recv_msg.tag); if (!msg) { WD_ERR("failed to find msg from pool!\n"); return -WD_EINVAL; @@ -2349,23 +2369,47 @@ int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) msg->req.status = recv_msg.result; req = &msg->req; req->cb(req); - wd_put_msg_to_pool(&wd_ecc_setting.pool, idx, recv_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, recv_msg.tag); *count = rcv_cnt; } while (--tmp); return ret; } +int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +{ + return wd_ecc_poll_ctx_(NULL, idx, expt, count); +} + int wd_ecc_poll(__u32 expt, __u32 *count) { - handle_t h_sched_sess = wd_ecc_setting.sched.h_sched_ctx; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!count)) { WD_ERR("invalid: ecc poll param count is NULL!\n"); return -WD_EINVAL; } - return wd_ecc_setting.sched.poll_policy(h_sched_sess, expt, count); + for (int i = 0; i < wd_ecc_setting.adapter->workers_nb; i++) { + worker = &wd_ecc_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + return ret; } static const struct wd_config_variable table[] = { diff --git a/wd_rsa.c b/wd_rsa.c index 366e76603..575244463 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -13,6 +13,7 @@ #include #include "include/drv/wd_rsa_drv.h" +#include "adapter.h" #include "wd_rsa.h" #define RSA_MAX_KEY_SIZE 512 @@ -66,17 +67,17 @@ struct wd_rsa_sess { struct wd_rsa_pubkey *pubkey; struct wd_rsa_prikey *prikey; struct wd_rsa_sess_setup setup; - void *sched_key; + void **sched_key; + struct uadk_adapter_worker *worker; + pthread_spinlock_t worker_lock; + int worker_looptime; }; static struct wd_rsa_setting { enum wd_status status; - struct wd_ctx_config_internal config; - struct wd_sched sched; - struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; + struct uadk_adapter *adapter; } wd_rsa_setting; struct wd_env_config wd_rsa_env_config; @@ -93,19 +94,15 @@ static void wd_rsa_close_driver(int init_type) if (!wd_rsa_setting.dlhandle) return; - wd_release_drv(wd_rsa_setting.driver); dlclose(wd_rsa_setting.dlhandle); wd_rsa_setting.dlhandle = NULL; #else - wd_release_drv(wd_rsa_setting.driver); hisi_hpre_remove(); #endif } static int wd_rsa_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "rsa"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -139,15 +136,6 @@ static int wd_rsa_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_rsa_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support!\n", alg_name); - return -WD_EINVAL; - } - - wd_rsa_setting.driver = driver; - return WD_SUCCESS; } @@ -156,66 +144,65 @@ static void wd_rsa_clear_status(void) wd_alg_clear_init(&wd_rsa_setting.status); } -static int wd_rsa_common_init(struct wd_ctx_config *config, struct wd_sched *sched) +static int wd_rsa_common_init(struct uadk_adapter_worker *worker, struct wd_sched *sched) { int ret; - ret = wd_set_epoll_en("WD_RSA_EPOLL_EN", - &wd_rsa_setting.config.epoll_en); + ret = wd_set_epoll_en("WD_RSA_EPOLL_EN", &worker->config.epoll_en); if (ret < 0) return ret; - ret = wd_init_ctx_config(&wd_rsa_setting.config, config); + ret = wd_init_ctx_config(&worker->config, worker->ctx_config); if (ret < 0) return ret; - ret = wd_init_sched(&wd_rsa_setting.sched, sched); - if (ret < 0) - goto out_clear_ctx_config; + worker->config.pool = &worker->pool; + sched->worker = worker; + worker->sched = sched; - ret = wd_init_async_request_pool(&wd_rsa_setting.pool, - config, WD_POOL_MAX_ENTRIES, + ret = wd_init_async_request_pool(&worker->pool, + worker->ctx_config, WD_POOL_MAX_ENTRIES, sizeof(struct wd_rsa_msg)); if (ret < 0) - goto out_clear_sched; + goto out_clear_ctx_config; - ret = wd_alg_init_driver(&wd_rsa_setting.config, - wd_rsa_setting.driver); + ret = wd_alg_init_driver(&worker->config, worker->driver); if (ret) goto out_clear_pool; return WD_SUCCESS; out_clear_pool: - wd_uninit_async_request_pool(&wd_rsa_setting.pool); -out_clear_sched: - wd_clear_sched(&wd_rsa_setting.sched); + wd_uninit_async_request_pool(&worker->pool); out_clear_ctx_config: - wd_clear_ctx_config(&wd_rsa_setting.config); + wd_clear_ctx_config(&worker->config); return ret; } static int wd_rsa_common_uninit(void) { + struct uadk_adapter_worker *worker; enum wd_status status; wd_alg_get_init(&wd_rsa_setting.status, &status); if (status == WD_UNINIT) return -WD_EINVAL; - /* uninit async request pool */ - wd_uninit_async_request_pool(&wd_rsa_setting.pool); + for (int i = 0; i < wd_rsa_setting.adapter->workers_nb; i++) { + worker = &wd_rsa_setting.adapter->workers[i]; - /* unset config, sched, driver */ - wd_clear_sched(&wd_rsa_setting.sched); - wd_alg_uninit_driver(&wd_rsa_setting.config, - wd_rsa_setting.driver); + wd_uninit_async_request_pool(&worker->pool); + wd_alg_uninit_driver(&worker->config, worker->driver); + } return WD_SUCCESS; } int wd_rsa_init(struct wd_ctx_config *config, struct wd_sched *sched) { + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; + char *alg = "rsa"; int ret; pthread_atfork(NULL, NULL, wd_rsa_clear_status); @@ -228,11 +215,24 @@ int wd_rsa_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + + wd_rsa_setting.adapter = adapter; + ret = wd_rsa_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; - ret = wd_rsa_common_init(config, sched); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_close_driver; + + worker = &adapter->workers[0]; + worker->ctx_config = config; + + ret = wd_rsa_common_init(worker, sched); if (ret) goto out_close_driver; @@ -243,6 +243,7 @@ int wd_rsa_init(struct wd_ctx_config *config, struct wd_sched *sched) out_close_driver: wd_rsa_close_driver(WD_TYPE_V1); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_rsa_setting.status); return ret; } @@ -255,6 +256,7 @@ void wd_rsa_uninit(void) if (ret) return; + free(wd_rsa_setting.adapter); wd_rsa_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_rsa_setting.status); } @@ -263,7 +265,10 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para { struct wd_ctx_nums rsa_ctx_num[WD_RSA_GENKEY] = {0}; struct wd_ctx_params rsa_ctx_params = {0}; + struct uadk_adapter_worker *worker; + struct uadk_adapter *adapter = NULL; int state, ret = -WD_EINVAL; + int i; pthread_atfork(NULL, NULL, wd_rsa_clear_status); @@ -282,78 +287,69 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para goto out_clear_init; } + adapter = calloc(1, sizeof(*adapter)); + if (adapter == NULL) + goto out_clear_init; + wd_rsa_setting.adapter = adapter; + state = wd_rsa_open_driver(WD_TYPE_V2); if (state) goto out_clear_init; - while (ret) { - memset(&wd_rsa_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + ret = uadk_adapter_add_workers(adapter, alg); + if (ret) + goto out_dlopen; - /* Get alg driver and dev name */ - wd_rsa_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_rsa_setting.driver) { - WD_ERR("failed to bind a valid driver!\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } + for (i = 0; i < adapter->workers_nb; i++) { + worker = &adapter->workers[i]; rsa_ctx_params.ctx_set_num = rsa_ctx_num; - ret = wd_ctx_param_init(&rsa_ctx_params, ctx_params, - wd_rsa_setting.driver, WD_RSA_TYPE, WD_RSA_GENKEY); + ret = wd_ctx_param_init(&rsa_ctx_params, ctx_params, worker->driver, + WD_RSA_TYPE, WD_RSA_GENKEY); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_rsa_setting.driver); - wd_alg_drv_unbind(wd_rsa_setting.driver); - continue; - } - goto out_driver; + WD_ERR("fail to init ctx param\n"); + goto out_dlopen; } wd_rsa_init_attrs.alg = alg; - wd_rsa_init_attrs.sched_type = sched_type; - wd_rsa_init_attrs.driver = wd_rsa_setting.driver; wd_rsa_init_attrs.ctx_params = &rsa_ctx_params; wd_rsa_init_attrs.alg_init = wd_rsa_common_init; - wd_rsa_init_attrs.alg_poll_ctx = wd_rsa_poll_ctx; - ret = wd_alg_attrs_init(&wd_rsa_init_attrs); + wd_rsa_init_attrs.alg_poll_ctx = wd_rsa_poll_ctx_; + ret = wd_alg_attrs_init(worker, &wd_rsa_init_attrs); + wd_ctx_param_uninit(&rsa_ctx_params); if (ret) { - if (ret == -WD_ENODEV) { - wd_disable_drv(wd_rsa_setting.driver); - wd_alg_drv_unbind(wd_rsa_setting.driver); - wd_ctx_param_uninit(&rsa_ctx_params); - continue; - } WD_ERR("failed to init alg attrs!\n"); - goto out_params_uninit; + goto out_dlopen; } } wd_alg_set_init(&wd_rsa_setting.status); - wd_ctx_param_uninit(&rsa_ctx_params); return WD_SUCCESS; -out_params_uninit: - wd_ctx_param_uninit(&rsa_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_rsa_setting.driver); out_dlopen: wd_rsa_close_driver(WD_TYPE_V2); out_clear_init: + free(adapter); wd_alg_clear_init(&wd_rsa_setting.status); return ret; } void wd_rsa_uninit2(void) { + struct uadk_adapter_worker *worker; int ret; ret = wd_rsa_common_uninit(); if (ret) return; - wd_alg_attrs_uninit(&wd_rsa_init_attrs); - wd_alg_drv_unbind(wd_rsa_setting.driver); + for (int i = 0; i < wd_rsa_setting.adapter->workers_nb; i++) { + worker = &wd_rsa_setting.adapter->workers[i]; + wd_alg_attrs_uninit(worker); + } + + free(wd_rsa_setting.adapter); wd_rsa_close_driver(WD_TYPE_V2); wd_rsa_setting.dlh_list = NULL; wd_alg_clear_init(&wd_rsa_setting.status); @@ -413,9 +409,8 @@ static int fill_rsa_msg(struct wd_rsa_msg *msg, struct wd_rsa_req *req, int wd_do_rsa_sync(handle_t h_sess, struct wd_rsa_req *req) { - struct wd_ctx_config_internal *config = &wd_rsa_setting.config; - handle_t h_sched_ctx = wd_rsa_setting.sched.h_sched_ctx; struct wd_rsa_sess *sess = (struct wd_rsa_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_msg_handle msg_handle; struct wd_ctx_internal *ctx; struct wd_rsa_msg msg; @@ -427,27 +422,31 @@ int wd_do_rsa_sync(handle_t h_sess, struct wd_rsa_req *req) return -WD_EINVAL; } - idx = wd_rsa_setting.sched.pick_next_ctx(h_sched_ctx, - sess->sched_key, - CTX_MODE_SYNC); - ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_SYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_SYNC, idx); if (ret) return ret; - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); - ctx = config->ctxs + idx; + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); + ctx = worker->config.ctxs + idx; memset(&msg, 0, sizeof(struct wd_rsa_msg)); ret = fill_rsa_msg(&msg, req, sess); if (unlikely(ret)) return ret; - msg_handle.send = wd_rsa_setting.driver->send; - msg_handle.recv = wd_rsa_setting.driver->recv; + msg_handle.send = worker->driver->send; + msg_handle.recv = worker->driver->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_rsa_setting.driver, &msg_handle, ctx->ctx, &msg, - &balance, wd_rsa_setting.config.epoll_en); + ret = wd_handle_msg_sync(worker->driver, &msg_handle, ctx->ctx, &msg, + &balance, worker->config.epoll_en); pthread_spin_unlock(&ctx->lock); if (unlikely(ret)) return ret; @@ -458,11 +457,10 @@ int wd_do_rsa_sync(handle_t h_sess, struct wd_rsa_req *req) return GET_NEGATIVE(msg.result); } -int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) +int wd_do_rsa_async(handle_t h_sess, struct wd_rsa_req *req) { - struct wd_ctx_config_internal *config = &wd_rsa_setting.config; - handle_t h_sched_ctx = wd_rsa_setting.sched.h_sched_ctx; - struct wd_rsa_sess *sess_t = (struct wd_rsa_sess *)sess; + struct wd_rsa_sess *sess = (struct wd_rsa_sess *)h_sess; + struct uadk_adapter_worker *worker; struct wd_rsa_msg *msg = NULL; struct wd_ctx_internal *ctx; int ret, mid; @@ -473,16 +471,20 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) return -WD_EINVAL; } - idx = wd_rsa_setting.sched.pick_next_ctx(h_sched_ctx, - sess_t->sched_key, - CTX_MODE_ASYNC); - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + pthread_spin_lock(&sess->worker_lock); + worker = sess->worker; + pthread_spin_unlock(&sess->worker_lock); + + idx = worker->sched->pick_next_ctx( + worker->sched->h_sched_ctx, + sess->sched_key[worker->idx], CTX_MODE_ASYNC); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; - mid = wd_get_msg_from_pool(&wd_rsa_setting.pool, idx, (void **)&msg); + mid = wd_get_msg_from_pool(&worker->pool, idx, (void **)&msg); if (unlikely(mid < 0)) { WD_ERR("failed to get msg from pool!\n"); return mid; @@ -493,7 +495,7 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) goto fail_with_msg; msg->tag = mid; - ret = wd_alg_driver_send(wd_rsa_setting.driver, ctx->ctx, msg); + ret = wd_alg_driver_send(worker->driver, ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send rsa BD, hw is err!\n"); @@ -501,7 +503,7 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) goto fail_with_msg; } - wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); + wd_dfx_msg_cnt(&worker->config, WD_CTX_CNT_NUM, idx); ret = wd_add_task_to_async_queue(&wd_rsa_env_config, idx); if (ret) goto fail_with_msg; @@ -509,18 +511,13 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) return WD_SUCCESS; fail_with_msg: - wd_put_msg_to_pool(&wd_rsa_setting.pool, idx, mid); + wd_put_msg_to_pool(&worker->pool, idx, mid); return ret; } -struct wd_rsa_msg *wd_rsa_get_msg(__u32 idx, __u32 tag) +int wd_rsa_poll_ctx_(struct wd_sched *sched, __u32 idx, __u32 expt, __u32 *count) { - return wd_find_msg_in_pool(&wd_rsa_setting.pool, idx, tag); -} - -int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) -{ - struct wd_ctx_config_internal *config = &wd_rsa_setting.config; + struct uadk_adapter_worker *worker; struct wd_ctx_internal *ctx; struct wd_rsa_req *req; struct wd_rsa_msg recv_msg, *msg; @@ -533,27 +530,31 @@ int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return -WD_EINVAL; } + /* back-compatible with init1 api */ + if (sched == NULL) + worker = &wd_rsa_setting.adapter->workers[0]; + else + worker = sched->worker; + *count = 0; - ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); + ret = wd_check_ctx(&worker->config, CTX_MODE_ASYNC, idx); if (ret) return ret; - ctx = config->ctxs + idx; + ctx = worker->config.ctxs + idx; do { - ret = wd_alg_driver_recv(wd_rsa_setting.driver, ctx->ctx, &recv_msg); + ret = wd_alg_driver_recv(worker->driver, ctx->ctx, &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { WD_ERR("failed to async recv, ret = %d!\n", ret); - wd_put_msg_to_pool(&wd_rsa_setting.pool, idx, - recv_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, recv_msg.tag); return ret; } rcv_cnt++; - msg = wd_find_msg_in_pool(&wd_rsa_setting.pool, idx, - recv_msg.tag); + msg = wd_find_msg_in_pool(&worker->pool, idx, recv_msg.tag); if (!msg) { WD_ERR("failed to find msg from pool!\n"); return -WD_EINVAL; @@ -563,23 +564,48 @@ int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) msg->req.status = recv_msg.result; req = &msg->req; req->cb(req); - wd_put_msg_to_pool(&wd_rsa_setting.pool, idx, recv_msg.tag); + wd_put_msg_to_pool(&worker->pool, idx, recv_msg.tag); *count = rcv_cnt; } while (--tmp); return ret; } +int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) +{ + return wd_rsa_poll_ctx_(NULL, idx, expt, count); +} + int wd_rsa_poll(__u32 expt, __u32 *count) { - handle_t h_sched_ctx = wd_rsa_setting.sched.h_sched_ctx; + struct uadk_adapter_worker *worker; + __u32 recv = 0; + int ret = WD_SUCCESS; if (unlikely(!count)) { WD_ERR("invalid: rsa poll count is NULL!\n"); return -WD_EINVAL; } - return wd_rsa_setting.sched.poll_policy(h_sched_ctx, expt, count); + for (int i = 0; i < wd_rsa_setting.adapter->workers_nb; i++) { + worker = &wd_rsa_setting.adapter->workers[i]; + + if (worker->valid) { + struct wd_sched *sched = worker->sched; + + ret = worker->sched->poll_policy(sched, expt, &recv); + if (ret) + return ret; + + *count += recv; + expt -= recv; + + if (expt == 0) + break; + } + } + + return ret; } int wd_rsa_kg_in_data(struct wd_rsa_kg_in *ki, char **data) @@ -921,8 +947,10 @@ static void del_sess(struct wd_rsa_sess *c) /* Before initiate this context, we should get a queue from WD */ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) { + struct uadk_adapter_worker *worker; + int nb = wd_rsa_setting.adapter->workers_nb; struct wd_rsa_sess *sess; - int ret; + int ret, i; if (!setup) { WD_ERR("invalid: alloc rsa sess setup NULL!\n"); @@ -944,18 +972,26 @@ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) memcpy(&sess->setup, setup, sizeof(*setup)); sess->key_size = setup->key_bits >> BYTE_BITS_SHIFT; + worker = sess->worker = &wd_rsa_setting.adapter->workers[0]; + worker->valid = true; + sess->worker_looptime = 0; + ret = create_sess_key(setup, sess); if (ret) { WD_ERR("failed to create rsa sess keys!\n"); goto sess_err; } - /* Some simple scheduler don't need scheduling parameters */ - sess->sched_key = (void *)wd_rsa_setting.sched.sched_init( - wd_rsa_setting.sched.h_sched_ctx, setup->sched_param); - if (WD_IS_ERR(sess->sched_key)) { - WD_ERR("failed to init session schedule key!\n"); - goto sched_err; + sess->sched_key = (void **)calloc(nb, sizeof(void *)); + for (i = 0; i < nb; i++) { + worker = &wd_rsa_setting.adapter->workers[i]; + + sess->sched_key[i] = (void *)worker->sched->sched_init( + worker->sched->h_sched_ctx, setup->sched_param); + if (WD_IS_ERR(sess->sched_key[i])) { + WD_ERR("failed to init session schedule key!\n"); + goto sched_err; + } } return (handle_t)sess; @@ -963,6 +999,11 @@ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) sched_err: del_sess_key(sess); sess_err: + if (sess->sched_key) { + for (i = 0; i < nb; i++) + free(sess->sched_key[i]); + free(sess->sched_key); + } free(sess); return (handle_t)0; } @@ -976,8 +1017,11 @@ void wd_rsa_free_sess(handle_t sess) return; } - if (sess_t->sched_key) + if (sess_t->sched_key) { + for (int i = 0; i < wd_rsa_setting.adapter->workers_nb; i++) + free(sess_t->sched_key[i]); free(sess_t->sched_key); + } del_sess_key(sess_t); del_sess(sess_t); } diff --git a/wd_sched.c b/wd_sched.c index aa6c91ec2..28cae6ec8 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -244,9 +244,10 @@ static __u32 session_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, return key->async_ctxid; } -static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, +static int session_poll_region(struct wd_sched *sched, __u32 begin, __u32 end, __u32 expect, __u32 *count) { + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; __u32 poll_num = 0; __u32 i; int ret; @@ -257,7 +258,7 @@ static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, * RR schedule, one time poll one package, * poll_num is always not more than one here. */ - ret = sched_ctx->poll_func(i, 1, &poll_num); + ret = sched_ctx->poll_func(sched, i, 1, &poll_num); if ((ret < 0) && (ret != -EAGAIN)) return ret; else if (ret == -EAGAIN) @@ -270,9 +271,10 @@ static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, return 0; } -static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, +static int session_poll_policy_rr(struct wd_sched *sched, int numa_id, __u32 expect, __u32 *count) { + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; struct sched_ctx_region **region = sched_ctx->sched_info[numa_id].ctx_region; __u32 begin, end; __u32 i; @@ -284,7 +286,7 @@ static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, begin = region[SCHED_MODE_ASYNC][i].begin; end = region[SCHED_MODE_ASYNC][i].end; - ret = session_poll_region(sched_ctx, begin, end, expect, count); + ret = session_poll_region(sched, begin, end, expect, count); if (unlikely(ret)) return ret; } @@ -294,7 +296,7 @@ static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, /* * session_poll_policy - The polling policy matches the pick next ctx. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. + * @sched: Schedule. * @cfg: The global resoure info. * @expect: User expect poll msg num. * @count: The actually poll num. @@ -302,11 +304,10 @@ static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, * The user must init the schedule info through wd_sched_rr_instance, the * func interval will not check the valid, becouse it will affect performance. */ -static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int session_sched_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; struct wd_sched_info *sched_info; - __u32 loop_time = 0; __u32 last_count = 0; __u16 i; int ret; @@ -323,34 +324,27 @@ static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 * sched_info = sched_ctx->sched_info; - /* - * Try different numa's ctx if we can't receive any - * package last time, it is more efficient. In most - * bad situation, poll ends after MAX_POLL_TIMES loop. - */ - while (++loop_time < MAX_POLL_TIMES) { - for (i = 0; i < sched_ctx->numa_num;) { - /* If current numa is not valid, find next. */ - if (!sched_info[i].valid) { - i++; - continue; - } + for (i = 0; i < sched_ctx->numa_num;) { + /* If current numa is not valid, find next. */ + if (!sched_info[i].valid) { + i++; + continue; + } - last_count = *count; - ret = session_poll_policy_rr(sched_ctx, i, expect, count); - if (unlikely(ret)) - return ret; + last_count = *count; + ret = session_poll_policy_rr(sched, i, expect, count); + if (unlikely(ret)) + return ret; - if (expect == *count) - return 0; + if (expect == *count) + return 0; - /* - * If no package is received, find next numa, - * otherwise, keep receiving packets at this node. - */ - if (last_count == *count) - i++; - } + /* + * If no package is received, find next numa, + * otherwise, keep receiving packets at this node. + */ + if (last_count == *count) + i++; } return 0; @@ -367,11 +361,10 @@ static __u32 sched_none_pick_next_ctx(handle_t sched_ctx, return 0; } -static int sched_none_poll_policy(handle_t h_sched_ctx, +static int sched_none_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - __u32 loop_times = MAX_POLL_TIMES + expect; + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; __u32 poll_num = 0; int ret; @@ -380,19 +373,12 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, return -WD_EINVAL; } - while (loop_times > 0) { - /* Default use ctx 0 */ - loop_times--; - ret = sched_ctx->poll_func(0, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) - return ret; - else if (ret == -EAGAIN) - continue; + /* Default use ctx 0 */ + ret = sched_ctx->poll_func(sched, 0, 1, &poll_num); + if (ret < 0) + return ret; - *count += poll_num; - if (*count == expect) - break; - } + *count += poll_num; return 0; } @@ -414,11 +400,10 @@ static __u32 sched_single_pick_next_ctx(handle_t sched_ctx, return CTX_SYNC; } -static int sched_single_poll_policy(handle_t h_sched_ctx, +static int sched_single_poll_policy(struct wd_sched *sched, __u32 expect, __u32 *count) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - __u32 loop_times = MAX_POLL_TIMES + expect; + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; __u32 poll_num = 0; int ret; @@ -427,19 +412,12 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, return -WD_EINVAL; } - while (loop_times > 0) { - /* Default async mode use ctx 1 */ - loop_times--; - ret = sched_ctx->poll_func(1, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) - return ret; - else if (ret == -EAGAIN) - continue; + /* Default async mode use ctx 1 */ + ret = sched_ctx->poll_func(sched, 1, 1, &poll_num); + if (ret < 0) + return ret; - *count += poll_num; - if (*count == expect) - break; - } + *count += poll_num; return 0; } diff --git a/wd_util.c b/wd_util.c index e908dcb44..0e6d8d57d 100644 --- a/wd_util.c +++ b/wd_util.c @@ -13,6 +13,7 @@ #include #include "wd_sched.h" #include "wd_util.h" +#include "adapter.h" #define WD_ASYNC_DEF_POLL_NUM 1 #define WD_ASYNC_DEF_QUEUE_DEPTH 1024 @@ -263,36 +264,6 @@ int wd_init_ctx_config(struct wd_ctx_config_internal *in, return ret; } -int wd_init_sched(struct wd_sched *in, struct wd_sched *from) -{ - if (!from->name || !from->sched_init || - !from->pick_next_ctx || !from->poll_policy) { - WD_ERR("invalid: member of wd_sched is NULL!\n"); - return -WD_EINVAL; - } - - in->h_sched_ctx = from->h_sched_ctx; - in->name = strdup(from->name); - in->sched_init = from->sched_init; - in->pick_next_ctx = from->pick_next_ctx; - in->poll_policy = from->poll_policy; - - return 0; -} - -void wd_clear_sched(struct wd_sched *in) -{ - char *name = (char *)in->name; - - if (name) - free(name); - in->h_sched_ctx = 0; - in->name = NULL; - in->sched_init = NULL; - in->pick_next_ctx = NULL; - in->poll_policy = NULL; -} - void wd_clear_ctx_config(struct wd_ctx_config_internal *in) { __u32 i; @@ -2271,6 +2242,9 @@ void *wd_dlopen_drv(const char *cust_lib_dir) node->dlhandle = dlopen(lib_path, RTLD_NODELETE | RTLD_NOW); if (!node->dlhandle) { + char *error = dlerror(); + + fprintf(stderr, "%s\n", error); free(node); /* there are many other files need to skip */ continue; @@ -2638,9 +2612,9 @@ static int wd_alg_ctx_init(struct wd_init_attrs *attrs) return ret; } -static int wd_alg_ce_ctx_init(struct wd_init_attrs *attrs) +static int wd_alg_ce_ctx_init(struct uadk_adapter_worker *worker) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; + struct wd_ctx_config *ctx_config = worker->ctx_config; ctx_config->ctx_num = 1; ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); @@ -2727,11 +2701,11 @@ static void wd_alg_uninit_sve_ctx(struct wd_ctx_config *ctx_config) free(ctx_config->ctxs); } -int wd_alg_attrs_init(struct wd_init_attrs *attrs) +int wd_alg_attrs_init(struct uadk_adapter_worker *worker, + struct wd_init_attrs *attrs) { wd_alg_poll_ctx alg_poll_func = attrs->alg_poll_ctx; wd_alg_init alg_init_func = attrs->alg_init; - __u32 sched_type = attrs->sched_type; struct wd_ctx_config *ctx_config = NULL; struct wd_sched *alg_sched = NULL; char alg_type[CRYPTO_MAX_ALG_NAME]; @@ -2742,8 +2716,8 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) if (!attrs->ctx_params) return -WD_EINVAL; - if (attrs->driver) - driver_type = attrs->driver->calc_type; + if (worker->driver) + driver_type = worker->driver->calc_type; switch (driver_type) { case UADK_ALG_SOFT: @@ -2753,7 +2727,7 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) WD_ERR("fail to alloc ctx config\n"); return -WD_ENOMEM; } - attrs->ctx_config = ctx_config; + worker->ctx_config = ctx_config; /* Use default sched_type to alloc scheduler */ alg_sched = wd_sched_rr_alloc(SCHED_POLICY_NONE, 1, 1, alg_poll_func); @@ -2762,15 +2736,15 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) goto out_ctx_config; } - attrs->sched = alg_sched; + worker->sched = alg_sched; - ret = wd_alg_ce_ctx_init(attrs); + ret = wd_alg_ce_ctx_init(worker); if (ret) { WD_ERR("fail to init ce ctx\n"); goto out_freesched; } - ret = alg_init_func(ctx_config, alg_sched); + ret = alg_init_func(worker, alg_sched); if (ret) goto out_pre_init; @@ -2782,14 +2756,14 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) WD_ERR("fail to alloc scheduler\n"); return -WD_EINVAL; } - attrs->sched = alg_sched; + worker->sched = alg_sched; ctx_config = calloc(1, sizeof(*ctx_config)); if (!ctx_config) { WD_ERR("fail to alloc ctx config\n"); goto out_freesched; } - attrs->ctx_config = ctx_config; + worker->ctx_config = ctx_config; ret = wd_alg_init_sve_ctx(ctx_config); if (ret) { @@ -2798,7 +2772,7 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) } ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); + ret = alg_init_func(worker, alg_sched); if (ret) { wd_alg_uninit_sve_ctx(ctx_config); goto out_freesched; @@ -2814,14 +2788,17 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) return -WD_ENOMEM; } attrs->ctx_config = ctx_config; + worker->ctx_config = ctx_config; - alg_sched = wd_sched_rr_alloc(sched_type, attrs->ctx_params->op_type_num, + /* Use default sched_type to alloc scheduler */ + alg_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, attrs->ctx_params->op_type_num, numa_max_node() + 1, alg_poll_func); if (!alg_sched) { WD_ERR("fail to instance scheduler\n"); goto out_ctx_config; } attrs->sched = alg_sched; + worker->sched = alg_sched; ret = wd_alg_ctx_init(attrs); if (ret) { @@ -2830,7 +2807,7 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) } ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); + ret = alg_init_func(worker, alg_sched); if (ret) goto out_pre_init; break; @@ -2854,11 +2831,11 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) return ret; } -void wd_alg_attrs_uninit(struct wd_init_attrs *attrs) +void wd_alg_attrs_uninit(struct uadk_adapter_worker *worker) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_sched *alg_sched = attrs->sched; - int driver_type = attrs->driver->calc_type; + struct wd_ctx_config *ctx_config = worker->ctx_config; + struct wd_sched *alg_sched = worker->sched; + int driver_type = worker->driver->calc_type; if (!ctx_config) { wd_sched_rr_release(alg_sched);