diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a63439b --- /dev/null +++ b/.gitignore @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (C) 2025 Intel Corporation +# These contents may have been developed with support from one or more +# Intel-operated generative artificial intelligence solutions. +.* +*.o +*.o.* +*.a +*.s +*.ko +*.so +*.so.dbg +*.mod +*.mod.c +*.i +*.lst +*.symtypes +*.order +modules.builtin +*.elf +*.bin +*.gz +*.bz2 +*.lzma +*.xz +*.lz4 +*.lzo +*.patch +*.patch.conv +*.gcno + +# +# Top-level generic files +# +/tags +/TAGS +/linux +/vmlinux +/vmlinuz +/System.map +/Module.markers +Module.symvers + +# stgit generated dirs +patches-* + +# quilt's files +patches +series + +# cscope files +cscope.* +ncscope.* + +# gnu global files +GPATH +GRTAGS +GSYMS +GTAGS + +*.orig +*~ +\#*# diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..4070c17 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,177 @@ +# Installation Guide for Acompress Provider + +This document provides step-by-step instructions for installing and configuring the Acompress Provider for DPUSM and ZFS. + +## Prerequisites + +Before you begin, ensure that you have the following: + +- A system with Linux Kernel support for the Crypto API's acomp interface and zlib-deflate (versions 6.3 to 6.6). +- Administrative access to the system for installation and configuration tasks. +- Setup top level source directory. + + ```bash + export HOME_PATH=/devel/zfs + sudo mkdir -p $HOME_PATH + sudo chown -R $USER:$USER $HOME_PATH + ``` + +## Component 1: DPUSM + +### Download DPUSM + +1. Clone the DPUSM repository: + + ```bash + cd $HOME_PATH + git clone https://github.com/hpc/dpusm + cd dpusm + ``` + +### Build and Install DPUSM + +2. Build DPUSM + + ```bash + make + ``` + +3. Load the module: + + ```bash + sudo insmod dpusm.ko + ``` + +4. Verify module is loaded: + + ```bash + lsmod | grep dpusm + ``` + + Example output: + + ```bash + lsmod | grep dpusm + dpusm 36864 0 + ``` + +## Component 2: ZFS with Z.I.A. + +### Obtain ZFS with Z.I.A. + +To use ZFS with the ZFS Interface for Accelerators (Z.I.A.), you need to obtain the specific branch that includes Z.I.A. support. Follow these steps: + +1. Clone the repository with the Z.I.A.: + + ```bash + cd $HOME_PATH + git clone https://github.com/hpc/zfs.git + cd zfs + ``` + +### Build and Install ZFS + +1. Build ZFS + + Notes: If DPUSM is installed to another directory, update the following command accordingly. + + ```bash + ./autogen.sh + ./configure --with-zia=$HOME_PATH/dpusm + make -j$(nproc) + ``` + +3. Install ZFS: + + ```bash + sudo make install + ``` + +4. Load ZFS modules: + + ```bash + sudo modprobe zfs + ``` + +5. Verify ZFS with Z.I.A was loaded: + + ```bash + sudo dmesg | grep -E "Z\.I\.A|ZFS" + ``` + + The output will look like: + + ```bash + user@server:~/zfs$ sudo dmesg | grep -E "Z\.I\.A|ZFS" + [59203.606953] Z.I.A. initialized (0000000029342ebc) + [59204.818383] ZFS: Loaded module v2.3.99-313_g325a5e241, ZFS pool version 5000, ZFS filesystem version 5 + ``` + +## Component 3: Acompress Provider + +### Clone the Acompress Provider Repository + +1. Clone the Acompress Provider repository: + + ```bash + cd $HOME_PATH + git clone https://github.com/intel/acomp-dpusm.git + cd acomp-dpusm + ``` + +### Build and Install Provider: + +1. Install the Acompress Provider: + + ```bash + make + ``` + +3. Install the Provider: + + ```bash + sudo insmod acomp-dpusm.ko + ``` + +4. Verify provided was loaded: + + ```bash + sudo dmesg | grep acomp_dpusm + ``` + + Example output: + + ```bash + user@server:~/acomp-dpusm$ sudo dmesg | grep acomp_dpusm + [62503.851101] Provider acomp_dpusm supports GZIP Level 1 Compress + [62503.851112] Provider acomp_dpusm supports GZIP Level 2 Compress + [62503.851114] Provider acomp_dpusm supports GZIP Level 3 Compress + [62503.851116] Provider acomp_dpusm supports GZIP Level 4 Compress + [62503.851117] Provider acomp_dpusm supports GZIP Level 5 Compress + [62503.851118] Provider acomp_dpusm supports GZIP Level 6 Compress + [62503.851119] Provider acomp_dpusm supports GZIP Level 7 Compress + [62503.851120] Provider acomp_dpusm supports GZIP Level 8 Compress + [62503.851121] Provider acomp_dpusm supports GZIP Level 9 Compress + [62503.851122] Provider acomp_dpusm supports GZIP Level 1 Decompress + [62503.851123] Provider acomp_dpusm supports GZIP Level 2 Decompress + [62503.851124] Provider acomp_dpusm supports GZIP Level 3 Decompress + [62503.851125] Provider acomp_dpusm supports GZIP Level 4 Decompress + [62503.851126] Provider acomp_dpusm supports GZIP Level 5 Decompress + [62503.851127] Provider acomp_dpusm supports GZIP Level 6 Decompress + [62503.851128] Provider acomp_dpusm supports GZIP Level 7 Decompress + [62503.851129] Provider acomp_dpusm supports GZIP Level 8 Decompress + [62503.851130] Provider acomp_dpusm supports GZIP Level 9 Decompress + [62503.851131] dpusm_provider_register: DPUSM Provider "acomp_dpusm" (00000000863c55ce) added. Now 1 providers registered. + ``` + +## Runtime Configuration + +Ensure the zpools are configured to use the provider and enable compression/decompression. The operations would look like: + + ```bash + sudo $HOME_PATH/zfs/zpool create -f test_zpool /dev/sda + sudo $HOME_PATH/zfs/zpool set zia_provider=acomp_dpusm test_zpool + sudo $HOME_PATH/zfs/zpool zia_compress=on test_zpool + sudo $HOME_PATH/zfs/zpool zia_decompress=on test_zpool + sudo $HOME_PATH/zfs/zfs set compression=gzip test_zpool + ``` diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..22da6f9 --- /dev/null +++ b/Makefile @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (C) 2025 Intel Corporation +# These contents may have been developed with support from one or more +# Intel-operated generative artificial intelligence solutions. + +# Define the kernel source directory +KERNELDIR ?= /lib/modules/$(shell uname -r)/build + +# Define the module name +MODULE_NAME := acomp-dpusm + +# Define the home path for the DPUSM and ZFS source directories +HOME_PATH ?= /devel/zfs + +# Define the include paths for DPUSM +DPUSM_INCLUDE := $(HOME_PATH)/dpusm/include + +# Define the include paths for ZFS +ZFS_INCLUDE := $(HOME_PATH)/zfs/include + +# Define the source files for the module +SRC_FILES := provider.c compress.c + +# Define the object files for the module +OBJ_FILES := $(SRC_FILES:.c=.o) + +# Define the location of the module symvers file for DPUSM +KBUILD_EXTRA_SYMBOLS := $(HOME_PATH)/dpusm/Module.symvers + +ifneq ($(KERNELRELEASE),) +ccflags-y := -I$(DPUSM_INCLUDE) -I$(ZFS_INCLUDE) +obj-m := $(MODULE_NAME).o +$(MODULE_NAME)-y := $(OBJ_FILES) +else +PWD := $(shell pwd) + +default: + $(MAKE) -C $(KERNELDIR) M=$(PWD) W=1 modules + +clean: + $(MAKE) -C $(KERNELDIR) M=$(PWD) clean +endif + +.PHONY: clean diff --git a/README.md b/README.md new file mode 100644 index 0000000..be96ae5 --- /dev/null +++ b/README.md @@ -0,0 +1,57 @@ +# Acompress Provider for DPUSM + +## Table of Contents + +- [Introduction](#introduction) +- [Licensing](#licensing) +- [Software Requirements](#software-requirements) +- [Limitations](#limitations) +- [Installation](#installation) +- [Legal](#legal) + +## Introduction + +The Acompress Provider is a plugin designed to integrate with DPUSM (Data Processing Unit Storage Management) and ZFS, utilizing the Z.I.A (ZFS Interface for Accelerators) framework. It enhances data compression capabilities within ZFS environments by leveraging the Crypto API's acomp interface for efficient data processing and storage management. + +stack diagram + + +## Licensing + +This project is licensed under the GNU General Public License v2.0. Please see the `LICENSE` file contained in the top-level folder for details. + +## Software Requirements + +- Linux Kernel with support for the Crypto API's acomp interface and zlib-deflate (currently covering versions 6.3 to 6.6). +- ZFS installed and operational configured with support for Z.I.A. +- DPUSM installed and configured. +- zpool configured to use this provider and compression offloading enabled + + +## Limitations + +1. The Acompress Provider currently supports specific compression algorithms available through the Crypto API's acomp interface. +2. Ensure compatibility with your existing DPUSM and ZFS setup before integration. +3. Performance may vary based on hardware and system configuration. + +## Installation + +For detailed installation instructions, please refer to the `INSTALL.md` file in this repository. + +## Legal + +Intel® disclaims all express and implied warranties, including without limitation, the implied warranties of merchantability, fitness for a particular purpose, and non-infringement, as well as any warranty arising from course of performance, course of dealing, or usage in trade. + +This document contains information on products, services, and/or processes in development. All information provided here is subject to change without notice. Contact your Intel® representative to obtain the latest forecast, schedule, specifications, and roadmaps. + +The products and services described may contain defects or errors known as errata which may cause deviations from published specifications. Current characterized errata are available on request. + +Copies of documents which have an order number and are referenced in this document may be obtained by calling 1-800-548-4725 or by visiting www.intel.com/design/literature.htm. + +Intel, the Intel logo are trademarks of Intel Corporation in the U.S. and/or other countries. + +\*Other names and brands may be claimed as the property of others + +--- + +Thank you for using the Acompress Provider! We hope it enhances your data processing and storage management experience. diff --git a/compress.c b/compress.c new file mode 100644 index 0000000..00cb1d8 --- /dev/null +++ b/compress.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 Intel Corporation + * These contents may have been developed with support from one or more + * Intel-operated generative artificial intelligence solutions. + */ +#include "compress.h" + +#include +#include +#include +#include +#include +#include +#include + +static struct crypto_acomp *acomp_tfm; +static bool acomp_init_done; + +int acomp_init(void) +{ + acomp_tfm = crypto_alloc_acomp("zlib-deflate", 0, 0); + if (IS_ERR(acomp_tfm)) { + pr_err("Failed to allocate zlib-deflate acomp transform\n"); + return PTR_ERR(acomp_tfm); + } + + acomp_init_done = true; + + return 0; +} + +void acomp_exit(void) +{ + if (!acomp_init_done) + return; + + if (acomp_tfm) { + crypto_free_acomp(acomp_tfm); + acomp_tfm = NULL; + } + acomp_init_done = false; +} + +enum comp_dir { + DECOMPRESS = 0, + COMPRESS = 1, +}; + +static int acomp_comp_decomp_sg(enum comp_dir dir, struct scatterlist *src, int src_len, + struct scatterlist *dst, int dst_len, size_t *c_len) +{ + struct crypto_wait wait; + struct acomp_req *req; + int ret; + + pr_debug("[%s] dir: %s, src_len: %d, dst_len: %d\n", __func__, + dir == COMPRESS ? "COMPRESS" : "DECOMPRESS", src_len, dst_len); + + req = acomp_request_alloc(acomp_tfm); + if (!req) { + pr_err("Failed to allocate acomp request\n"); + return -ENOMEM; + } + + crypto_init_wait(&wait); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + + acomp_request_set_params(req, src, dst, src_len, dst_len); + + if (dir == COMPRESS) + ret = crypto_acomp_compress(req); + else + ret = crypto_acomp_decompress(req); + + ret = crypto_wait_req(ret, &wait); + if (ret) + goto out; + + *c_len = req->dlen; + +out: + acomp_request_free(req); + pr_debug("[%s] ret: %d, dlen=%zu\n", __func__, ret, *c_len); + + return ret; +} + +static int vmalloc_to_sg(const void *vmem, size_t size, struct scatterlist *sgl, int nents) +{ + unsigned long vaddr = (unsigned long)vmem; + unsigned long offset; + size_t remaining = size; + struct page *page; + int sg_count = 0; + + sg_init_table(sgl, nents); + + while (remaining > 0) { + page = vmalloc_to_page((void *)vaddr); + if (!page) { + pr_err("Failed to retrieve page for vaddr: 0x%lx\n", vaddr); + return -EINVAL; + } + + offset = vaddr & (PAGE_SIZE - 1); + size_t len = min((size_t)(PAGE_SIZE - offset), remaining); + + sg_set_page(&sgl[sg_count], page, len, offset); + + vaddr += len; + remaining -= len; + sg_count++; + + if (sg_count >= nents && remaining > 0) { + pr_err("Not enough scatterlist entries for memory size: %zu\n", size); + return -ENOMEM; + } + } + + sg_mark_end(&sgl[sg_count - 1]); + + return 0; +} + +static int count_pages(const void *vmem, size_t size) +{ + unsigned long vaddr = (unsigned long)vmem; + size_t remaining = size; + int page_count = 0; + + while (remaining > 0) { + size_t len = min((size_t)(PAGE_SIZE - (vaddr & (PAGE_SIZE - 1))), remaining); + + vaddr += len; + remaining -= len; + page_count++; + } + + return page_count; +} + +static int alloc_scatterlist(const void *vmem, size_t size, struct scatterlist **sgl) +{ + struct scatterlist *sg = NULL; + int ret = 0; + int pages; + + if (!is_vmalloc_addr(vmem)) { + sg = kmalloc(sizeof(*sg), GFP_KERNEL); + if (!sg) { + *sgl = NULL; + return -ENOMEM; + } + + sg_init_one(sg, vmem, size); + + *sgl = sg; + return 0; + } + + pages = count_pages(vmem, size); + if (pages <= 0) { + pr_err("Failed to count pages for memory\n"); + return -EINVAL; + } + + sg = kmalloc_array(pages, sizeof(struct scatterlist), GFP_KERNEL); + if (!sg) + return -ENOMEM; + + memset(sg, 0, sizeof(struct scatterlist) * pages); + + ret = vmalloc_to_sg(vmem, size, sg, pages); + if (ret) { + pr_err("Failed to map vmalloc memory to scatterlist\n"); + kfree(sg); + *sgl = NULL; + return ret; + } + + *sgl = sg; + return ret; +} + +static int acomp_comp_decomp(enum comp_dir dir, void *src, int src_len, + void *dst, int dst_len, size_t *c_len) +{ + struct scatterlist *src_sg, *dst_sg; + int ret; + + ret = alloc_scatterlist(src, src_len, &src_sg); + if (ret) { + pr_err("Failed to allocate scatterlist for SRC memory\n"); + return ret; + } + + ret = alloc_scatterlist(dst, dst_len, &dst_sg); + if (ret) { + pr_err("Failed to allocate scatterlist for DST memory\n"); + kfree(src_sg); + return ret; + } + + ret = acomp_comp_decomp_sg(dir, src_sg, src_len, dst_sg, dst_len, c_len); + + kfree(src_sg); + kfree(dst_sg); + + return ret; +} + +int acomp_compress(void *src, int src_len, void *dst, int dst_len, size_t *c_len) +{ + return acomp_comp_decomp(COMPRESS, src, src_len, dst, dst_len, c_len); +} + +int acomp_decompress(void *src, int src_len, void *dst, int dst_len, size_t *c_len) +{ + return acomp_comp_decomp(DECOMPRESS, src, src_len, dst, dst_len, c_len); +} diff --git a/compress.h b/compress.h new file mode 100644 index 0000000..aa4b3ce --- /dev/null +++ b/compress.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025 Intel Corporation + * These contents may have been developed with support from one or more + * Intel-operated generative artificial intelligence solutions. + */ +#ifndef _ACOMP_COMPRESS_H +#define _ACOMP_COMPRESS_H + +#include + +int acomp_init(void); +void acomp_exit(void); +int acomp_compress(void *src, int src_len, void *dst, int dst_len, size_t *c_len); +int acomp_decompress(void *src, int src_len, void *dst, int dst_len, size_t *c_len); + +#endif /* _ACOMP_COMPRESS_H */ diff --git a/docs/images/stack.png b/docs/images/stack.png new file mode 100644 index 0000000..06b3193 Binary files /dev/null and b/docs/images/stack.png differ diff --git a/provider.c b/provider.c new file mode 100644 index 0000000..9d0b345 --- /dev/null +++ b/provider.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 Intel Corporation + * These contents may have been developed with support from one or more + * Intel-operated generative artificial intelligence solutions. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "compress.h" +#include + +enum handle_type { + ZQH_REAL, + ZQH_REF, +}; + +struct provider_handle { + enum handle_type type; + void *ptr; + size_t size; +}; + +static void *ptr_start(struct provider_handle *zqh, const size_t offset) +{ + return (char *)zqh->ptr + offset; +} + +static int zfs_acomp_algos(int *compress, int *decompress, int *checksum, + int *checksum_byteorder, int *raid) +{ + *compress = DPUSM_COMPRESS_GZIP_1 | DPUSM_COMPRESS_GZIP_2 | + DPUSM_COMPRESS_GZIP_3 | DPUSM_COMPRESS_GZIP_4 | + DPUSM_COMPRESS_GZIP_5 | DPUSM_COMPRESS_GZIP_6 | + DPUSM_COMPRESS_GZIP_7 | DPUSM_COMPRESS_GZIP_8 | + DPUSM_COMPRESS_GZIP_9; + + *decompress = *compress; + + *checksum = 0; + *checksum_byteorder = 0; + *raid = 0; + + return DPUSM_OK; +} + +static void *zfs_acomp_alloc(size_t size) +{ + struct provider_handle *buf; + + buf = kmalloc(sizeof(*buf), GFP_KERNEL); + if (!buf) + return NULL; + + buf->type = ZQH_REAL; + buf->ptr = kmalloc(size, GFP_KERNEL); + if (!buf->ptr) { + kfree(buf); + return NULL; + } + + buf->size = size; + + return buf; +} + +static void *zfs_acomp_alloc_ref(void *handle, size_t offset, size_t size) +{ + struct provider_handle *src = handle; + struct provider_handle *ref; + + ref = kmalloc(sizeof(*ref), GFP_KERNEL); + if (!ref) + return NULL; + + ref->type = ZQH_REF; + ref->ptr = ptr_start(src, offset); + ref->size = size; + + return ref; +} + +static int zfs_acomp_get_size(void *handle, size_t *size, size_t *actual) +{ + struct provider_handle *buf = handle; + + if (size) + *size = buf->size; + + if (actual) + *actual = buf->size; + + return DPUSM_OK; +} + +static int zfs_acomp_free(void *handle) +{ + struct provider_handle *buf = handle; + + if (buf->type == ZQH_REAL) + kfree(buf->ptr); + + kfree(buf); + + return DPUSM_OK; +} + +static int zfs_acomp_copy_from_generic(dpusm_mv_t *mv, const void *buf, size_t size) +{ + struct provider_handle *dst = mv->handle; + + /* + * If the handle points to a real buffer, free it, + * and convert the handle to a reference to the + * user's pointer. + */ + if (dst->type == ZQH_REAL && mv->offset == 0) { + kfree(dst->ptr); + + dst->type = ZQH_REF; + dst->ptr = (void *)buf; + dst->size = size; + } else { + memcpy(ptr_start(dst, mv->offset), buf, size); + } + + return DPUSM_OK; +} + +static int zfs_acomp_copy_to_generic(dpusm_mv_t *mv, void *buf, size_t size) +{ + struct provider_handle *src = mv->handle; + void *start = ptr_start(src, mv->offset); + + /* + * if the handle is still a real buffer, it is not + * the same as buf, so the contents have to be copied + * into buf. + */ + if (start != buf) + memcpy(buf, start, size); + + return DPUSM_OK; +} + +static int zfs_acomp_compress(dpusm_compress_t alg, int level, void *src, + size_t s_len, void *dst, size_t *d_len) +{ + (void)alg; + (void)level; + + struct provider_handle *s = src; + struct provider_handle *d = dst; + void *s_start; + void *d_start; + int ret; + + if (s_len > s->size || *d_len > d->size) + return DPUSM_ERROR; + + s_start = ptr_start(s, 0); + d_start = ptr_start(d, 0); + + ret = acomp_compress(s_start, s_len, d_start, *d_len, d_len); + if (ret) + return DPUSM_ERROR; + + /* + * TODO: understand what to do in case of overflow. In such case, ret + * is equals to -EOVERFLOW + */ + + return DPUSM_OK; +} + +static int zfs_acomp_decompress(dpusm_compress_t alg, int *level, void *src, + size_t s_len, void *dst, size_t *d_len) +{ + (void)alg; + (void)level; + + struct provider_handle *s = src; + struct provider_handle *d = dst; + void *s_start; + void *d_start; + int ret; + + if (s_len > s->size || *d_len > d->size) + return DPUSM_ERROR; + + s_start = ptr_start(s, 0); + d_start = ptr_start(d, 0); + + ret = acomp_decompress(s_start, s_len, d_start, *d_len, d_len); + if (ret) + return DPUSM_ERROR; + + return DPUSM_OK; +} + +static const dpusm_pf_t zfs_acomp_provider_functions = { + .algorithms = zfs_acomp_algos, + .alloc = zfs_acomp_alloc, + .alloc_ref = zfs_acomp_alloc_ref, + .get_size = zfs_acomp_get_size, + .free = zfs_acomp_free, + .copy = { + .from = { + .generic = zfs_acomp_copy_from_generic, + .ptr = NULL, + .scatterlist = NULL, + }, + .to = { + .generic = zfs_acomp_copy_to_generic, + .ptr = NULL, + .scatterlist = NULL, + }, + }, + .compress = zfs_acomp_compress, + .decompress = zfs_acomp_decompress, + .at_connect = NULL, + .at_disconnect = NULL, + .mem_stats = NULL, +}; + +static int __init zfs_acomp_provider_init(void) +{ + int ret; + + ret = acomp_init(); + if (ret) { + pr_err("%s: Error: Could not initialize provider\n", + module_name(THIS_MODULE)); + return ret; + } + + return dpusm_register_gpl(THIS_MODULE, &zfs_acomp_provider_functions); +} + +static void __exit zfs_acomp_provider_exit(void) +{ + dpusm_unregister_gpl(THIS_MODULE); + + acomp_exit(); +} + +module_init(zfs_acomp_provider_init); +module_exit(zfs_acomp_provider_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("DPUSM ACOMP Provider");