|
| 1 | +#include <stdio.h> |
| 2 | +#include <unistd.h> |
| 3 | +#include <malloc.h> |
| 4 | +#include <stdlib.h> |
| 5 | +#include <inttypes.h> |
| 6 | +#include <netinet/in.h> |
| 7 | +#include <arpa/inet.h> |
| 8 | +#include <infiniband/verbs.h> |
| 9 | +#include <infiniband/mlx5dv.h> |
| 10 | +#include <reg_access/reg_access.h> |
| 11 | +#include <dlfcn.h> |
| 12 | +#include <linux/swab.h> |
| 13 | +#include "resource_dump.h" |
| 14 | + |
| 15 | +/* to_mpd always returns the real mlx5_pd object ie the protection domain. */ |
| 16 | +static inline struct mlx5_pd* to_mpd(struct ibv_pd* ibpd) |
| 17 | +{ |
| 18 | + struct mlx5_pd* mpd = to_mxxx(pd, pd); |
| 19 | + |
| 20 | + if (mpd->mprotection_domain) |
| 21 | + return mpd->mprotection_domain; |
| 22 | + |
| 23 | + return mpd; |
| 24 | +} |
| 25 | + |
| 26 | +struct ibv_context* init_my_device_c(char device_name[], void* handler, void* handler_2) |
| 27 | +{ |
| 28 | + struct ibv_device** d_list = NULL; |
| 29 | + int n = 0; |
| 30 | + int i = 0; |
| 31 | + struct ibv_context* context; |
| 32 | + struct mlx5dv_context_attr* my_context_attr; |
| 33 | + my_context_attr = (struct mlx5dv_context_attr*)malloc(sizeof(struct mlx5dv_context_attr)); |
| 34 | + memset(my_context_attr, 0, sizeof(struct mlx5dv_context_attr)); |
| 35 | + |
| 36 | + my_context_attr->flags = MLX5DV_CONTEXT_FLAGS_DEVX; |
| 37 | + my_context_attr->comp_mask = 0; |
| 38 | + f_ibv_get_device_list ibv_get_device_list_func = (f_ibv_get_device_list)dlsym(handler, GET_DEVICE_LIST_FUNC); |
| 39 | + d_list = ibv_get_device_list_func(&n); |
| 40 | + f_ibv_get_device_name ibv_get_device_name_func = (f_ibv_get_device_name)dlsym(handler, GET_DEVICE_NAME_FUNC); |
| 41 | + f_mlx5dv_open_device mlx5dv_open_device_func = (f_mlx5dv_open_device)dlsym(handler_2, OPEN_DEVICE_FUNC); |
| 42 | + |
| 43 | + if (d_list) |
| 44 | + { |
| 45 | + for (i = 0; i < n; i++) |
| 46 | + { |
| 47 | + if (d_list[i] != NULL && strcmp(ibv_get_device_name_func(d_list[i]), device_name) == 0) |
| 48 | + { |
| 49 | + context = mlx5dv_open_device_func(d_list[i], my_context_attr); |
| 50 | + if (!context) |
| 51 | + { |
| 52 | + ibv_get_device_name_func(d_list[i]); |
| 53 | + return NULL; |
| 54 | + } |
| 55 | + return context; |
| 56 | + } |
| 57 | + } |
| 58 | + } |
| 59 | + else |
| 60 | + { |
| 61 | + printf("ibv_get_device_list failed\n"); |
| 62 | + return NULL; |
| 63 | + } |
| 64 | + |
| 65 | + f_ibv_free_device_list ibv_free_device_list_func = (f_ibv_free_device_list)dlsym(handler, FREE_DEVICE_LIST); |
| 66 | + ibv_free_device_list_func(d_list); |
| 67 | + free(my_context_attr); |
| 68 | + |
| 69 | + return NULL; |
| 70 | +} |
| 71 | + |
| 72 | +struct mlx5dv_mkey* create_mkey_c(struct mlx5_mkey* mkey, |
| 73 | + struct ibv_context* context, |
| 74 | + uint32_t pdn, |
| 75 | + uint32_t umem_id, |
| 76 | + uint64_t addr, |
| 77 | + uint32_t buff_size, |
| 78 | + void* handler) |
| 79 | +{ |
| 80 | + uint32_t out[DEVX_ST_SZ_DW(create_mkey_out)] = {}; |
| 81 | + uint32_t in[DEVX_ST_SZ_DW(create_mkey_in)] = {}; |
| 82 | + void* mkc; |
| 83 | + |
| 84 | + DEVX_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); |
| 85 | + mkc = DEVX_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); |
| 86 | + DEVX_SET(mkc, mkc, access_mode_1_0, 0x1); |
| 87 | + DEVX_SET(mkc, mkc, umr_en, 0); |
| 88 | + DEVX_SET(mkc, mkc, pd, pdn); |
| 89 | + DEVX_SET(mkc, mkc, lr, 1); |
| 90 | + DEVX_SET(mkc, mkc, lw, 1); |
| 91 | + DEVX_SET(mkc, mkc, qpn, 0xffffff); |
| 92 | + DEVX_SET64(mkc, mkc, start_addr, addr); |
| 93 | + DEVX_SET64(mkc, mkc, len, buff_size); |
| 94 | + |
| 95 | + DEVX_SET(create_mkey_in, in, mkey_umem_valid, 1); |
| 96 | + DEVX_SET(create_mkey_in, in, mkey_umem_id, umem_id); |
| 97 | + |
| 98 | + f_mlx5dv_devx_obj_create mlx5dv_devx_obj_create_func = |
| 99 | + (f_mlx5dv_devx_obj_create)dlsym(handler, OBJECT_CREATE); |
| 100 | + mkey->devx_obj = mlx5dv_devx_obj_create_func(context, in, sizeof(in), out, sizeof(out)); |
| 101 | + |
| 102 | + if (!mkey->devx_obj) |
| 103 | + { |
| 104 | + printf("mlx5dv_devx_obj_create failed\n"); |
| 105 | + printf("status: %u\n", DEVX_GET(create_mkey_out, out, status)); |
| 106 | + printf("syndrome: %u\n", DEVX_GET(create_mkey_out, out, syndrome)); |
| 107 | + return NULL; |
| 108 | + } |
| 109 | + |
| 110 | + // printf("MKEY_INDEX: %x\n", DEVX_GET(create_mkey_out, out, mkey_index)); |
| 111 | + mkey->dv_mkey.lkey = (DEVX_GET(create_mkey_out, out, mkey_index) << 8) | 0; |
| 112 | + mkey->dv_mkey.rkey = mkey->dv_mkey.lkey; |
| 113 | + |
| 114 | + return &mkey->dv_mkey; |
| 115 | +} |
| 116 | + |
| 117 | +struct ibv_pd* ibv_allocate_pd(struct ibv_context* ctx, void* handler) |
| 118 | +{ |
| 119 | + f_ibv_alloc_pd ibv_alloc_pd_func = (f_ibv_alloc_pd)dlsym(handler, ALLOCATE_PD); |
| 120 | + struct ibv_pd* st = ibv_alloc_pd_func(ctx); |
| 121 | + |
| 122 | + return st; |
| 123 | +} |
| 124 | + |
| 125 | +struct mlx5dv_devx_umem* mlx5dv_devx_umem_register_ex(struct ibv_context* ctx, |
| 126 | + void* buff, |
| 127 | + uint32_t buff_al_size, |
| 128 | + struct mlx5dv_devx_umem_in* umem_in, |
| 129 | + void* handler) |
| 130 | +{ |
| 131 | + umem_in->addr = buff; |
| 132 | + umem_in->size = buff_al_size; |
| 133 | + umem_in->access = IBV_ACCESS_LOCAL_WRITE; |
| 134 | + umem_in->pgsz_bitmap = sysconf(_SC_PAGESIZE); |
| 135 | + f_mlx5dv_devx_umem_reg_ex mlx5dv_devx_umem_reg_ex_func = |
| 136 | + (f_mlx5dv_devx_umem_reg_ex)dlsym(handler, UMEM_REG); |
| 137 | + struct mlx5dv_devx_umem* umem = mlx5dv_devx_umem_reg_ex_func(ctx, umem_in); |
| 138 | + |
| 139 | + return umem; |
| 140 | +} |
| 141 | + |
| 142 | +int generate_lkey(char device_name[], struct result* res) |
| 143 | +{ |
| 144 | + void* buff; |
| 145 | + struct mlx5dv_mkey* mkey; |
| 146 | + struct mlx5dv_devx_umem_in* umem_in; |
| 147 | + umem_in = (struct mlx5dv_devx_umem_in*)malloc(sizeof(struct mlx5dv_devx_umem_in)); |
| 148 | + memset(umem_in, 0, sizeof(struct mlx5dv_devx_umem_in)); |
| 149 | + |
| 150 | + int ret = -1; |
| 151 | + void* handler = dlopen(LIB_VERBS_LINUX_PATH, RTLD_LOCAL | RTLD_LAZY); |
| 152 | + if (!handler) |
| 153 | + { |
| 154 | + handler = dlopen(LIB_VERBS_UBUNTU_PATH, RTLD_LOCAL | RTLD_LAZY); |
| 155 | + if (!handler) |
| 156 | + { |
| 157 | + printf("Failed to load the libibverbs shared library"); |
| 158 | + return ret; |
| 159 | + } |
| 160 | + } |
| 161 | + |
| 162 | + void* handler_2 = dlopen(LIB_MLX5_LINUX_PATH, RTLD_LOCAL | RTLD_LAZY); |
| 163 | + if (!handler_2) |
| 164 | + { |
| 165 | + handler_2 = dlopen(LIB_MLX5_UBUNTU_PATH, RTLD_LOCAL | RTLD_LAZY); |
| 166 | + if (!handler_2) |
| 167 | + { |
| 168 | + printf("Failed to load the libmlx5 shared library"); |
| 169 | + dlclose(handler); |
| 170 | + return ret; |
| 171 | + } |
| 172 | + } |
| 173 | + |
| 174 | + struct ibv_context* context = init_my_device_c(device_name, handler, handler_2); |
| 175 | + if (!context) |
| 176 | + { |
| 177 | + printf("init_my_device_c failed\n"); |
| 178 | + dlclose(handler); |
| 179 | + dlclose(handler_2); |
| 180 | + return ret; |
| 181 | + } |
| 182 | + |
| 183 | + f_ibv_close_device ibv_close_device_func = (f_ibv_close_device)dlsym(handler, CLOSE_DEVICE); |
| 184 | + struct ibv_pd* pd = ibv_allocate_pd(context, handler); |
| 185 | + if (!pd) |
| 186 | + { |
| 187 | + printf("ibv_alloc_pd failed\n"); |
| 188 | + ibv_close_device_func(context); |
| 189 | + dlclose(handler); |
| 190 | + return ret; |
| 191 | + } |
| 192 | + |
| 193 | + f_ibv_dealloc_pd ibv_dealloc_pd_func = (f_ibv_dealloc_pd)dlsym(handler, DEALLOCATE_PD); |
| 194 | + uint32_t buff_al_size = align(BUFF_SIZE, sysconf(_SC_PAGESIZE)); |
| 195 | + if (posix_memalign(&buff, sysconf(_SC_PAGESIZE), buff_al_size)) |
| 196 | + { |
| 197 | + printf("posix_memalign failed\n"); |
| 198 | + ibv_dealloc_pd_func(pd); |
| 199 | + dlclose(handler); |
| 200 | + return ret; |
| 201 | + } |
| 202 | + |
| 203 | + memset(buff, 0, buff_al_size); |
| 204 | + |
| 205 | + struct mlx5dv_devx_umem* umem = mlx5dv_devx_umem_register_ex(context, buff, buff_al_size, umem_in, handler_2); |
| 206 | + if (!umem) |
| 207 | + { |
| 208 | + printf("mlx5dv_devx_umem_reg failed\n"); |
| 209 | + free(buff); |
| 210 | + dlclose(handler); |
| 211 | + return ret; |
| 212 | + } |
| 213 | + |
| 214 | + f_mlx5dv_devx_umem_dereg mlx5dv_devx_umem_dereg_func = |
| 215 | + (f_mlx5dv_devx_umem_dereg)dlsym(handler, UMEM_DEREG); |
| 216 | + struct mlx5_mkey* mmkey = calloc(1, sizeof(*mkey)); |
| 217 | + if (!mmkey) |
| 218 | + { |
| 219 | + printf("calloc Failed\n"); |
| 220 | + mlx5dv_devx_umem_dereg_func(umem); |
| 221 | + dlclose(handler); |
| 222 | + return ret; |
| 223 | + } |
| 224 | + |
| 225 | + mkey = create_mkey_c(mmkey, context, to_mpd(pd)->pdn, umem->umem_id, (uint64_t)buff, buff_al_size, handler_2); |
| 226 | + if (!mkey) |
| 227 | + { |
| 228 | + printf("create_mkey failed\n"); |
| 229 | + free(mmkey); |
| 230 | + dlclose(handler); |
| 231 | + return ret; |
| 232 | + } |
| 233 | + else |
| 234 | + { |
| 235 | + // printf("Mkey: lkey: %u, rkey: %u\n", mkey->lkey, mkey->rkey); |
| 236 | + // printf("%" PRIu64 "buff2-\n", (uint64_t)buff); |
| 237 | + res->lkey = mkey->lkey; |
| 238 | + res->umem_addr = (uint64_t)buff; |
| 239 | + // printf("%d",res->lkey); |
| 240 | + // printf("%" PRIu64 "\n", res->umem_addr); |
| 241 | + |
| 242 | + dlclose(handler); |
| 243 | + } |
| 244 | + |
| 245 | + ret = 0; |
| 246 | + free(umem_in); |
| 247 | + return ret; |
| 248 | +} |
| 249 | + |
| 250 | +int bit_extracted_32(uint64_t add, int offset, struct extract* ex) |
| 251 | +{ |
| 252 | + // FILE *fp; |
| 253 | + // fp = fopen("/tmp/test.log", "w+"); |
| 254 | + // fprintf(fp,"0x%llX\n", (long long unsigned int)add); |
| 255 | + // fprintf(fp, "offset-%d\n",offset); |
| 256 | + // fclose(fp); |
| 257 | + |
| 258 | + uint64_t* p; |
| 259 | + p = (uint64_t*)add; |
| 260 | + // printf("p=%zu\n", (size_t)p); |
| 261 | + |
| 262 | + p += offset / 2; |
| 263 | + |
| 264 | + u_int32_t* data = (u_int32_t*)p; |
| 265 | + // printf("val=0x%016x\n",*p); |
| 266 | + |
| 267 | + // printf("1st-32: 0x%08x\n", __swab32(data[0])); |
| 268 | + // printf("2nd-32: 0x%08x\n", __swab32(data[1])); |
| 269 | + |
| 270 | + uint32_t idx = (offset + 1) % 2 == 0 ? 1 : 0; |
| 271 | + ex->val = __swab32(data[idx]); |
| 272 | + |
| 273 | + return 0; |
| 274 | +} |
| 275 | + |
| 276 | +int extrac_all(uint64_t add, int size, char* list) |
| 277 | +{ |
| 278 | + // int i = 0; |
| 279 | + // u_int8_t *p; |
| 280 | + // p = (u_int8_t*)add; |
| 281 | + |
| 282 | + memcpy(list, (void*)add, sizeof(char) * size); |
| 283 | + |
| 284 | + // for( ; i < size; i++) |
| 285 | + // { |
| 286 | + // memcpy(list + i, p + i, sizeof(char)); |
| 287 | + // } |
| 288 | + |
| 289 | + return 0; |
| 290 | +} |
| 291 | + |
0 commit comments