add support for ML-DSA-* etc., various small fixes; OpenSSL 4.0 compat

.github/workflows/OpenSSL_versions.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        branch: [openssl-3.0, openssl-3.1, openssl-3.2, openssl-3.3, openssl-3.4, openssl-3.5, openssl-3.6]
+        branch: [openssl-4.0, openssl-3.6, openssl-3.5, openssl-3.4, openssl-3.0]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -43,37 +43,43 @@ jobs:
           echo "OPENSSL_LIB=${{ env.OPENSSL_DIR }}/lib64" >> $GITHUB_ENV
           echo "${{ env.OPENSSL_DIR }}/bin" >> "$GITHUB_PATH"
           echo "LD_LIBRARY_PATH=${{ env.OPENSSL_DIR }}/lib64:$LD_LIBRARY_PATH" >> "$GITHUB_ENV"
-      - name: cmake
+      - name: cmake build, install, uninstall, Debian
         run: |
           cmake .
           make
           ./cmpClient -help
-          if [[ $(openssl version | cut -d' ' -f2 | cut -d'.' -f1) -lt 4 ]]; then
-            echo "Using OpenSSL version less than 4.0, proceeding build with libcmp"
-            mkdir build-with-libcmp
-            cd build-with-libcmp
-            USE_LIBCMP=1 cmake -S .. -B .
-          fi
-          make clean build
           DESTDIR=tmp make install uninstall
           make deb
+          make clean
+
+      - name: cmake out-of-source build build with libcmp
+        run: |
+          mkdir build-with-libcmp
+          cd build-with-libcmp
+          USE_LIBCMP=1 cmake -S .. -B .
+          make
+          ./cmpClient -help
+          cd ..
+
       - if: ${{ matrix.branch == 'openssl-3.5' }}
-        name: Create mock test certificates and keys with PQC algorithms
+        name: When using OpenSSL 3.5, switch Mock test credentials to PQC algorithms
         run: |
           cd test/recipes/80-test_cmp_http_data/Mock/
-          Algo_used="PQC" ./setup-mock.sh all  # sets up  fresh test certificates and keys for Mock test using PQC algorithms 
+          # use script to overwrite test keys and certificates for Mock test, this time using PQC algorithms
+          Algo_used="PQC" ./setup-mock.sh all
           cd ../../../../ 
-      - name: make
+
+      - name: make (using Makefile_v1) test_Mock
         run: |
-          make -f Makefile_v1
-          ./cmpClient -help
+          make -f Makefile_v1 clean
           make -f Makefile_v1 test_Mock
           make -f Makefile_v1 clean_all
-          if [[ $(openssl version | cut -d' ' -f2 | cut -d'.' -f1) -lt 4 ]]; then
-            USE_LIBCMP=1 STATIC_LIBCMP=1 make -f Makefile_v1
-            ./cmpClient -help
-            make -f Makefile_v1 clean
-          fi
+
+      - name: make (using Makefile_v1) with static libcmp
+        run: |
+          USE_LIBCMP=1 STATIC_LIBCMP=1 make -f Makefile_v1
+          ./cmpClient -help
+          make -f Makefile_v1 clean
 
 
 

CMakeLists.txt

@@ -258,7 +258,8 @@ else()
   add_compile_options(-pedantic) # -Werror is enabled only for development and CI, using Makefile_v1 without NDEBUG
   add_compile_options(
     -Wall -Woverflow -Wextra -Wmissing-prototypes -Wstrict-prototypes -Wswitch
-    -Wsign-compare -Wformat -Wtype-limits -Wundef -Wconversion -Wunused-parameter)
+    -Wsign-compare -Wformat -Wformat-security -Wtype-limits -Wundef -Wconversion
+    -Wpointer-arith -Wunused-parameter -Wshadow)
   add_compile_options(-Wno-c99-extensions -Wno-language-extension-token -Wno-declaration-after-statement -Wno-expansion-to-defined)
 
 endif()

Makefile_src

@@ -84,19 +84,18 @@ OPENSSL_DLLS = *{crypto,ssl}*.dll
 CC ?= gcc
 ifdef NDEBUG
     override DEBUG_FLAGS ?= -O3
-    override DEBUG_FLAGS += -DNDEBUG=1 -Werror
+    override DEBUG_FLAGS += -DNDEBUG=1
 else
-    override DEBUG_FLAGS ?= -g -O0 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all # not every compiler(version) supports -Og
+    override DEBUG_FLAGS ?= -g -O0 -Werror -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all # not every compiler(version) supports -Og
 endif
 override CFLAGS += $(DEBUG_FLAGS) -fstack-protector -fno-omit-frame-pointer
 # override CFLAGS += -std=gnu90  # TODO maybe clean up code and re-enable flag
 override CFLAGS += \
-  -Wall -Woverflow -Wextra -Wswitch -Wmissing-prototypes -Wstrict-prototypes \
-  -Wformat -Wformat-security -Wtype-limits -Wundef -Wconversion \
-  -Wsign-compare -Wpointer-arith -Wunused-parameter -Wshadow \
+  -Wall -Woverflow -Wextra -Wmissing-prototypes -Wstrict-prototypes -Wswitch \
+  -Wsign-compare -Wformat -Wformat-security -Wtype-limits -Wundef -Wconversion \
+  -Wpointer-arith -Wunused-parameter -Wshadow \
   -pedantic -DPEDANTIC
-override CFLAGS +=-Wno-c99-extensions -Wno-language-extension-token -Wno-declaration-after-statement -Wno-expansion-to-defined \
-  -Wno-sign-conversion -Wno-shorten-64-to-32 -Wno-shadow # due to libsecutils
+override CFLAGS +=-Wno-c99-extensions -Wno-language-extension-token -Wno-declaration-after-statement -Wno-expansion-to-defined
 ifeq ($(LPATH),)
   ifndef GENCMP_NO_SECUTILS
     override CFLAGS += -I$(SECUTILS_DIR)/src/libsecutils/include

Makefile_v1

@@ -197,13 +197,13 @@ ifneq ($(filter-out doc start stop doc doc_this doc/cmpClient.md doc/cmpClient.1
         # must use ":=" below to avoid error: Recursive variable `OPENSSL_LIB' references itself (eventually)
         OPENSSL_LIB := $(shell $(GET_LIB) 2>/dev/null)
         ifeq ($(OPENSSL_LIB),)
-            $(shell $(GET_LIB))
+            $(info $(shell $(GET_LIB)))
             $(error Error determining OPENSSL_LIB)
         endif
     endif
     LIB_NAME_PATTERN=libcrypto*$(DLL)*
     ifeq ($(wildcard $(OPENSSL_LIB)/$(LIB_NAME_PATTERN)),)
-        $(shell $(GET_LIB))
+        $(info $(shell $(GET_LIB)))
         $(error Error: cannot find OpenSSL library $(LIB_NAME_PATTERN) at $(OPENSSL_LIB)/)
     endif
     # convert to absolute path
@@ -530,7 +530,8 @@ else
     OCSP_CHECK= # disabled for now: $(OPENSSL) ocsp -url $(EJBCA_OCSP_URL) \
         -CAfile $(EJBCA_CMP_TRUSTED) -issuer $(EJBCA_CMP_ISSUER) \
         -cert creds/operational.crt
-    EJBCA_TLS_HOST_FILE=creds/docker/TLS_ROOTCA-docker-cn.txt
+    EJBCA_TLS_SERVER_CERTS=creds/docker/TLS_server-docker.pem
+    EJBCA_TLS_HOST_FILE=creds/docker/TLS_server-docker-cn.txt
     ifeq ($(EJBCA_TLS_HOST),) # workaround for ephemeral TLS server certificate of ejbca-docker:
         override EXTRA_OPTS += -tls_host `cat $(EJBCA_TLS_HOST_FILE)`
         BOOTSTRAP_CREDS = -cert creds/manufacturer.crt -key creds/manufacturer.pem
@@ -586,9 +587,8 @@ else
 	@echo
 	$(CMPCLIENT) bootstrap -section $(CA_SECTION) -provider default $(EXTRA_OPTS) $(BOOTSTRAP_CREDS)
 	$(GENERATE_OPERATIONAL)
-	$(OPENSSL) x509 -noout -text -in creds/operational.crt
+	$(OPENSSL) x509 -noout -text -in creds/operational.crt | sed -E '/^ *([0-9a-f]{2}:).*/d'
 	@echo :
-	$(OPENSSL) x509 -noout -text -in creds/operational.crt | sed '/^         [0-9a-f].*/d'
     ifneq ($(CA_SECTION),Insta) # on p10cr, Insta responds with an empty message PKIBody
 	@echo
 	@$(SLEEP)
@@ -666,8 +666,8 @@ ifeq ($(EJBCA_CONFIG),)
 	@echo "EJBCA should now be ready to accept requests."
     ifeq ($(EJBCA_TLS_HOST),) # workaround for ephemeral TLS server certificate of ejbca-docker:
 	@$(OPENSSL) s_client -connect $(EJBCA_HOST):$(EJBCA_HTTPS_PORT) 2>/dev/null \
-	  -verify_return_error -showcerts >creds/docker/TLS_ROOTCA-docker.pem || true
-	@echo `grep -E "CN ?= ?" creds/docker/TLS_ROOTCA-docker.pem|head -n 1 | sed -E 's/^.*CN ?= ?//; s/, ?UID ?= ?.*//; s/, O ?= ?.*//;'` >$(EJBCA_TLS_HOST_FILE)
+	  -verify_return_error -showcerts >$(EJBCA_TLS_SERVER_CERTS) || true
+	@echo `grep -E "CN ?= ?" $(EJBCA_TLS_SERVER_CERTS) | head -n 1 | sed -E 's/^.*CN ?= ?//; s/, ?UID ?= ?.*//; s/, O ?= ?.*//;'` >$(EJBCA_TLS_HOST_FILE)
 	@grep -qE '\w' $(EJBCA_TLS_HOST_FILE) || (echo "cannot determine EJBCA docker TLS host name"; false)
     endif
     ifneq ($(EJBCA_STARTED),)

OpenSSL_version.mk

@@ -85,7 +85,7 @@ else
             override OPENSSL_LIB = $(OPENSSL_DIR)
             ifeq ($(wildcard $(OPENSSL_LIB)/$(LIB_NAME_PATTERN)),)
                 ifeq ($(OS),Linux)
-	            ifeq ($(shell echo $(OPENSSL_FULL_DIR) | grep -E $(USERS)),)
+                    ifeq ($(shell echo $(OPENSSL_FULL_DIR) | grep -E $(USERS)),)
                         override OPENSSL_LIB = $(wildcard /lib/$(shell uname -i)-linux-*)
                         $(warning Warning: cannot find OpenSSL libraries at $(OPENSSL_DIR), now trying $(OPENSSL_LIB))
                     endif

README.md

@@ -20,7 +20,7 @@ which is geared towards simple and interoperable industrial certificate manageme
 
 <!--
 Moreover, this software is in internal use for implementing further CMP features
-and experimental additions, for instance as part of ongoing ITEF standardization
+and experimental additions, for instance as part of ongoing IETF standardization
 regarding support for post-quantum cryptography (PQC) and remote attestation.
 --->
 
@@ -123,7 +123,7 @@ The core library can also be built and used natively under Windows.
 
 The following development and network tools are needed or recommended.
 
-* Git (for getting the software, tested versions include 2.7.2, 2.11.0, 2.20, 2.34.1, 2.48.0, 2.53.0)
+* Git (for conveniently getting and updating the software, tested versions include 2.7.2, 2.11.0, 2.20, 2.34.1, 2.48.0, 2.53.0)
 * CMake (for using [`CMakeLists.txt`](CMakeLists.txt), tested versions include 3.18.4, 3.22.1, 3.27.7, 3.31.5)
 * GNU make (tested versions include 3.81, 4.1, 4.2.1, 4.3)
 * GNU C compiler (gcc, tested versions include 5.4.0, 7.3.0, 8.3.0, 10.2.1, 11.4.0, 12.2.0)
@@ -231,6 +231,10 @@ you can execute in a shell on a Unix-like system:
 ```bash
 git clone https://github.com/siemens/gencmpclient.git
 cd genCMPClient
+```
+or using some other way of obtaining the code, then
+
+```bash
 make -f OpenSSL_version.mk
 ```
 
@@ -273,34 +277,37 @@ or an HTTP proxy set up, for instance:
 export https_proxy=http://proxy.example.com:8080
 ```
 
-You can clone the git repository and its submodules with
+When using `git`, you can clone the git repository and its submodules with
 
 ```bash
 git clone https://github.com/siemens/gencmpclient.git
 cd genCMPClient
 make -f Makefile_v1 get_submodules
 ```
 
-This will fetch also the underlying
-[CMPforOpenSSL extension to OpenSSL](https://github.com/mpeylo/cmpossl) and
-the [Security Utilities (libsecutils)](https://github.com/siemens/libsecutils)
-library if needed.
-
-For using the project as a git submodule,
-do for instance the following in the directory where you want to integrate it:
+This will fetch also
+the [Security Utilities (libsecutils)](https://github.com/siemens/libsecutils) library
+and
+the underlying [CMPforOpenSSL extension to OpenSSL](https://github.com/mpeylo/cmpossl)
+as far as needed.
 
-```bash
-git submodule add git@github.com:siemens/gencmpclient.git
-```
+When not using `git`, you need to download and unpack the
+[genCMPClient source tree](https://github.com/siemens/gencmpclient.git)
+by other means,
+as well as the sources of any of the two submodules as far as used,
+in a state (i.e., git commit ID or version) consistent with the genCMPClient tree.
+For instance, place the contents of the
+[libsecutils repository](https://github.com/siemens/libsecutils)
+in the subdirectory `libsecutils`.
 
 When you later want to update your local copy of all relevant repositories
-it is sufficient to invoke
+and `git` is installed, it is sufficient to invoke
 
 ```bash
 make update
 ```
 
-When switching to a certain commit or version, e.g.
+When switching to a certain commit or version, e.g.,
 
 ```bash
 git checkout v2.2
@@ -310,12 +317,22 @@ then also execute
 
 ```bash
 git submodule update
+```
+or update the submodules by other means to a consistent state, then
+
+```bash
 make -f Makefile_v1 clean
 ```
 
 to bring the submodules in a state consistent with that
 and remove any previous possibly outdated artifacts.
 
+The genCMPClient project itself can be used a git submodule as follows:
+
+```bash
+git submodule add git@github.com:siemens/gencmpclient.git
+```
+
 ## Configuring
 
 ### Finding OpenSSL
@@ -613,7 +630,7 @@ OPENSSL_TRACE=HTTP ./cmpClient imprint
 
 ### Demo use with a local EJBCA
 
-The demo uses by default a Docker instance of the EJBCA, which is included in the repo and launched locally on demo startup.
+The demo uses by default a Docker instance of the EJBCA, which is included in the repository and launched locally on demo startup.
 This variant of the demo can be used explicitly as follows:
 
 ```bash
@@ -736,4 +753,5 @@ LocalWords:  util icvutil NDEBUG DCMAKE ln usr libgencmp CC lssl lcmp md bis
 LocalWords:  cmpClient src DESTDIR ROOTFS cmpclient tarball deb rpath
 LocalWords:  debhelper dh devscripts debuild dpkg ecparam FI cr lgencmp cc cnf
 LocalWords:  genkey insta ref cmd newkey certout noout creds Wl ICV GENCMP
+LocalWords:  br cmp gencmp dll DGENCMP libSecUtils mkdir PPKI newkeytype
 -->

config/EJBCA.env

@@ -20,9 +20,9 @@ export EJBCA_CDP_URL_POSTFIX=
 export EJBCA_CDP_URL_POSTFIX_v11=
 export EJBCA_CMP_CLIENT_CERT=creds/manufacturer.crt
 export EJBCA_CMP_CLIENT_KEY=creds/manufacturer.pem
-export EJBCA_TLS_CLIENT=creds/docker/Docker_Playground_TLS.p12
+export EJBCA_TLS_CLIENT=creds/docker/Docker_Playground_TLS.pem
 export EJBCA_CMP_TRUSTED=creds/docker/CMP_ROOTCA.pem
-export EJBCA_TLS_TRUSTED=creds/docker/TLS_ROOTCA-docker.pem
+export EJBCA_TLS_TRUSTED=creds/docker/TLS_ROOTCA.pem,creds/docker/TLS_server-docker.pem
 #export EJBCA_CMP_UNTRUSTED=creds/docker/CMP_ISSUING_CA.pem
 export EJBCA_CMP_ISSUER=creds/docker/CUSTOMER_ISSUING_CA.pem
 export EJBCA_TRUSTED=creds/docker/CUSTOMER_ROOTCA.pem

creds/docker/Docker_Playground_CMP.pem

@@ -0,0 +1,54 @@
+Bag Attributes
+    localKeyID: EF 65 43 CF 4F 35 2E 54 DD 23 80 4A 54 7C FA 25 27 4C 60 9C 
+subject=CN=Docker_Playground_CMP
+issuer=CN=CMP_ISSUING_CA
+-----BEGIN CERTIFICATE-----
+MIIBnDCCAUGgAwIBAgIUVC3571mWwpFPdnSz0LggFGG5wUYwCgYIKoZIzj0EAwIw
+GTEXMBUGA1UEAwwOQ01QX0lTU1VJTkdfQ0EwHhcNMjMwNTA4MDc1NjE2WhcNMjgw
+NTA2MDc1NjE1WjAgMR4wHAYDVQQDDBVEb2NrZXJfUGxheWdyb3VuZF9DTVAwWTAT
+BgcqhkjOPQIBBggqhkjOPQMBBwNCAATQpHc3WLC0W27WpM2W9y7vfuRRENDsRzNA
+nO/1qxsyS41U3ZiHE37fy4h+2jiJaRPq1C1bmM+mEBEhjU748B2Bo2AwXjAMBgNV
+HRMBAf8EAjAAMB8GA1UdIwQYMBaAFBe3WkY1OQbB+HoHOyAKrmdp3pOVMB0GA1Ud
+DgQWBBQo9f4zIuL+JIF0yOdnEztZFMGdQzAOBgNVHQ8BAf8EBAMCB4AwCgYIKoZI
+zj0EAwIDSQAwRgIhALwNgZtLgo4KvhA2dLZlVrBwCblwNrDOjJ1rwaU3eizrAiEA
+x6cfHtAWeh6j4s0OY6Sb+vGk5DCQktrxwkxeIOwgXPE=
+-----END CERTIFICATE-----
+Bag Attributes: <No Attributes>
+subject=CN=CMP_ROOTCA
+issuer=CN=CMP_ROOTCA
+-----BEGIN CERTIFICATE-----
+MIIBjzCCATWgAwIBAgIUbf73QaxpKix6tj9EDwXsfqvPsPQwCgYIKoZIzj0EAwIw
+FTETMBEGA1UEAwwKQ01QX1JPT1RDQTAeFw0yMzAzMjAyMzUzMTJaFw0zODAzMTYy
+MzUzMTFaMBUxEzARBgNVBAMMCkNNUF9ST09UQ0EwWTATBgcqhkjOPQIBBggqhkjO
+PQMBBwNCAAR0ukviHeyaDmbVCyfw9enYFBCxUDu2r4+dhfz9uJ6TDfR2f9ccoE0Z
+EC/8KvNvEM1v5MCkfaOc5PPZmCU7vgNBo2MwYTAPBgNVHRMBAf8EBTADAQH/MB8G
+A1UdIwQYMBaAFPj3VMsyBY5x2442sx4yn/fWZ00/MB0GA1UdDgQWBBT491TLMgWO
+cduONrMeMp/31mdNPzAOBgNVHQ8BAf8EBAMCAYYwCgYIKoZIzj0EAwIDSAAwRQIg
+NdlFaGf2E7vhdfCF49ACIq8eo9myKv6f4znLJ1aa8pMCIQD+JHE6i1m8hsHU6jY0
+2ux5Cvt7BzT8+2ebIbSTTT/j5w==
+-----END CERTIFICATE-----
+Bag Attributes: <No Attributes>
+subject=CN=CMP_ISSUING_CA
+issuer=CN=CMP_ROOTCA
+-----BEGIN CERTIFICATE-----
+MIIBkzCCATmgAwIBAgIUPgNxHZDaL13Is3RbTJ7joZrzS7swCgYIKoZIzj0EAwIw
+FTETMBEGA1UEAwwKQ01QX1JPT1RDQTAeFw0yMzAzMjAyMzU0MDFaFw0zMzAzMTcy
+MzU0MDBaMBkxFzAVBgNVBAMMDkNNUF9JU1NVSU5HX0NBMFkwEwYHKoZIzj0CAQYI
+KoZIzj0DAQcDQgAEf+u8xxjMlp6JD4Hqhn7Wlzse8zsQlWY8PDAFvGOrQsR/4vYC
+ITWaV5hZs7fiBo2jH47n8zzSbAEMQcbf3gXtw6NjMGEwDwYDVR0TAQH/BAUwAwEB
+/zAfBgNVHSMEGDAWgBT491TLMgWOcduONrMeMp/31mdNPzAdBgNVHQ4EFgQUF7da
+RjU5BsH4egc7IAquZ2nek5UwDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMCA0gA
+MEUCIQDvA71BlzTasNO9OsIelwV/DwIaR1Ss9kjv99n2q0+boQIgSjLn0IldzuUt
+wK/hd2Gz4uNk626jI1EqTZC+goHJNaM=
+-----END CERTIFICATE-----
+Bag Attributes
+    localKeyID: EF 65 43 CF 4F 35 2E 54 DD 23 80 4A 54 7C FA 25 27 4C 60 9C 
+Key Attributes: <No Attributes>
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIH0MF8GCSqGSIb3DQEFDTBSMDEGCSqGSIb3DQEFDDAkBBBHXSevPSw+n8K8CE9Z
+T6bBAgIIADAMBggqhkiG9w0CCQUAMB0GCWCGSAFlAwQBKgQQ0LGlPDf7cyqpa7mI
+Bl0VfgSBkFajoqFxhR4IJYcNUWN9yKN6mFGN+1lSHr+H7xpkb66Ju6r+uBBfdBnQ
+y/k9NquDOAZER5LEWD/g7SXnXALyvUZm/hK8/cZOOjHnKot87JD/rLUXyK8xD50b
+x0ntYwk+tAWo//mLWs9ACHvjJ/3H3Q1DX5z9F1YQxjGE8jmcSuW9c3ZfjhTEb0Nf
+uAE0LYG7WQ==
+-----END ENCRYPTED PRIVATE KEY-----