CI: refactor docker-based test runner and add linux container tests

.github/workflows/ruby-unit-tests.yml

@@ -10,25 +10,28 @@ on:
 
 jobs:
   test:
+    env:
+      RUBY_VERSION: '3.2'
+      COMPOSE_CMD: >
+        docker compose -f docker-compose.yml
+        -f dev/compose/linux/${{ matrix.triplestore }}.yml
+        --profile linux
+        --profile ${{ matrix.triplestore }}
     strategy:
       fail-fast: false
       matrix:
-        backend: ['ruby', 'ruby-agraph'] # ruby runs tests with 4store backend and ruby-agraph runs with AllegroGraph backend
+          #triplestore: [ 'fs', 'ag', 'vo', 'gd' ]
+         triplestore: [ 'fs', 'ag' ]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - name: Set up solr configsets
-      run: ./test/solr/generate_ncbo_configsets.sh
-    - name: create config.rb file
-      run: cp config/config.test.rb config/config.rb
-    - name: Build docker compose
-      run: docker compose --profile 4store build # profile flag is set in order to build all containers in this step
+    - name: Build docker container
+      run: >
+        $COMPOSE_CMD build
     - name: Run unit tests
-      # unit tests are run inside a container
-      # http://docs.codecov.io/docs/testing-with-docker
-      run: |
-        ci_env=`bash <(curl -s https://codecov.io/env)`
-        docker compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v'
+      run: >
+        $COMPOSE_CMD run -v "$PWD/coverage:/app/coverage" -e CI=true
+        test-linux bundle exec rake test TESTOPTS="-v"
     - name: Upload coverage reports to Codecov
       uses: codecov/codecov-action@v5
       with:

Dockerfile

@@ -1,4 +1,4 @@
-ARG RUBY_VERSION=3.1
+ARG RUBY_VERSION=3.2
 ARG DISTRO=bullseye
 
 FROM ruby:$RUBY_VERSION-$DISTRO
@@ -8,17 +8,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     git \
     libxml2 \
     libxslt-dev \
+    libxslt1-dev zlib1g-dev \
     openjdk-11-jre-headless \
     raptor2-utils \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
+# set default test config
+COPY config/config.test.rb config/config.rb
+
 COPY Gemfile* *.gemspec ./
 
 # Copy only the `version.rb` file to prevent missing file errors!
 COPY lib/ontologies_linked_data/version.rb lib/ontologies_linked_data/
 
+
 #Install the exact Bundler version from Gemfile.lock (if it exists)
 RUN gem update --system && \
     if [ -f Gemfile.lock ]; then \

dev/compose/linux/ag.yml

@@ -0,0 +1,16 @@
+services:
+  test-linux:
+    environment:
+      GOO_BACKEND_NAME: allegrograph
+      GOO_PORT: 10035
+      GOO_HOST: agraph-ut
+      GOO_PATH_QUERY: /repositories/ontoportal_test
+      GOO_PATH_DATA: /repositories/ontoportal_test/statements
+      GOO_PATH_UPDATE: /repositories/ontoportal_test/statements
+    depends_on:
+      solr-ut:
+        condition: service_healthy
+      redis-ut:
+        condition: service_healthy
+      agraph-ut:
+        condition: service_healthy

dev/compose/linux/docker-compose.ci.yml

@@ -0,0 +1,3 @@
+services:
+  test-linux:
+    image: ontologies_linked_data-test-linux:ci

dev/compose/linux/fs.yml

@@ -0,0 +1,13 @@
+services:
+  test-linux:
+    environment:
+      GOO_BACKEND_NAME: '4store'
+      GOO_HOST: 4store-ut
+      GOO_PORT: 9000
+    depends_on:
+      solr-ut:
+        condition: service_healthy
+      redis-ut:
+        condition: service_healthy
+      4store-ut:
+        condition: service_healthy

dev/compose/linux/gd.yml

@@ -0,0 +1,16 @@
+services:
+  test-linux:
+    environment:
+      GOO_BACKEND_NAME: graphdb
+      GOO_PORT: 7200
+      GOO_HOST: graphdb-ut
+      GOO_PATH_QUERY: /repositories/ontoportal_test
+      GOO_PATH_DATA: /repositories/ontoportal_test/statements
+      GOO_PATH_UPDATE: /repositories/ontoportal_test/statements
+    depends_on:
+      solr-ut:
+        condition: service_healthy
+      redis-ut:
+        condition: service_healthy
+      graphdb-ut:
+        condition: service_healthy

dev/compose/linux/no-ports.yml

@@ -0,0 +1,13 @@
+services:
+  redis-ut:
+    ports: []
+  solr-ut:
+    ports: []
+  agraph-ut:
+    ports: []
+  4store-ut:
+    ports: []
+  virtuoso-ut:
+    ports: []
+  graphdb-ut:
+    ports: []

dev/compose/linux/vo.yml

@@ -0,0 +1,16 @@
+services:
+  test-linux:
+    environment:
+      GOO_BACKEND_NAME: 'virtuoso'
+      GOO_HOST: virtuoso-ut
+      GOO_PORT: 8890
+      GOO_PATH_QUERY: /sparql
+      GOO_PATH_DATA: /sparql
+      GOO_PATH_UPDATE: /sparql
+    depends_on:
+      solr-ut:
+        condition: service_healthy
+      redis-ut:
+        condition: service_healthy
+      virtuoso-ut:
+        condition: service_healthy

docker-compose.yml

@@ -1,140 +1,126 @@
-x-app: &app
+# unit tests in containerased env
+services:
+  test-linux:
     build:
       context: .
       args:
         RUBY_VERSION: '3.2'
-    # Increase the version number in the image tag every time Dockerfile or its arguments is changed
-    image: ontologies_ld-dev:0.0.4
-    environment: &env
+    command: ["bash", "-lc", "bundle exec rake test"]
+    environment:
       COVERAGE: 'true' # enable simplecov code coverage
       REDIS_HOST: redis-ut
-      REDIS_PORT: 6379
-      SOLR_TERM_SEARCH_URL: http://solr-term-ut:8983/solr
-      SOLR_PROP_SEARCH_URL: http://solr-prop-ut:8983/solr
-    stdin_open: true
-    tty: true
-    command: /bin/bash
-    volumes:
-      # bundle volume for hosting gems installed by bundle; it speeds up gem install in local development
-      - bundle:/usr/local/bundle
-      - .:/app
-      # mount directory containing development version of the gems if you need to use 'bundle config local'
-      #- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal
-    depends_on: &depends_on
-      solr-prop-ut:
-        condition: service_healthy
-      solr-term-ut:
+      SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr
+      SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr
+    depends_on:
+      solr-ut:
         condition: service_healthy
       redis-ut:
         condition: service_healthy
-
-services:
-  # environment wtih 4store backend
-  ruby:
-    <<: *app
-    environment:
-      <<: *env
-      GOO_BACKEND_NAME: 4store
-      GOO_PORT: 9000
-      GOO_HOST: 4store-ut
-      GOO_PATH_QUERY: /sparql/
-      GOO_PATH_DATA: /data/
-      GOO_PATH_UPDATE: /update/
     profiles:
-      - 4store
-    depends_on:
-      <<: *depends_on
-      4store-ut:
-        condition: service_started
-
-  # environment with AllegroGraph backend
-  ruby-agraph:
-    <<: *app
-    environment:
-      <<: *env
-      GOO_BACKEND_NAME: ag
-      GOO_PORT: 10035
-      GOO_HOST: agraph-ut
-      GOO_PATH_QUERY: /repositories/ontoportal_test
-      GOO_PATH_DATA: /repositories/ontoportal_test/statements
-      GOO_PATH_UPDATE: /repositories/ontoportal_test/statements
-    profiles:
-      - agraph
-    depends_on:
-      <<: *depends_on
-      agraph-ut:
-        condition: service_healthy
+      - linux
 
   redis-ut:
     image: redis
+    ports:
+      - 6379:6379
     command: ["redis-server", "--save", "", "--appendonly", "no"]
     healthcheck:
       test: redis-cli ping
       interval: 10s
       timeout: 3s
       retries: 10
 
+  solr-ut:
+    image: solr:9
+    command: bin/solr start -cloud -f
+    ports:
+      - 8983:8983
+    healthcheck:
+       test: ["CMD", "curl", "-sf", "http://localhost:8983/solr/admin/info/system?wt=json"]
+       start_period: 5s
+       interval: 10s
+       timeout: 5s
+       retries: 5
+
   4store-ut:
     image: bde2020/4store
     platform: linux/amd64
+    ports:
+      - 9000:9000
     command: >
-      bash -c "4s-backend-setup --segments 4 ontoportal_kb
-      && 4s-backend ontoportal_kb
-      && 4s-httpd -D -s-1 -p 9000 ontoportal_kb"
-    profiles:
-      - 4store
-
-  solr-term-ut:
-    image: solr:8
-    volumes:
-      - ./test/solr/configsets:/configsets:ro
-    # ports:
-    #  - "8983:8983"
-    command:  ["solr-precreate", "term_search_core1", "/configsets/term_search"]
-    healthcheck:
-      test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"]
-      start_period: 5s
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
-  solr-prop-ut:
-    image: solr:8
-    volumes:
-      - ./test/solr/configsets:/configsets:ro
-    # ports:
-    #  - "8984:8983"
-    command:  ["solr-precreate", "prop_search_core1", "/configsets/property_search"]
+      bash -c "4s-backend-setup --segments 4 ontoportal_test
+      && 4s-backend ontoportal_test
+      && 4s-httpd -D -s-1 -p 9000 ontoportal_test"
     healthcheck:
-      test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/prop_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"]
+      test: ["CMD", "4s-backend-info", "ontoportal_test"]
       start_period: 5s
       interval: 10s
-      timeout: 5s
+      timeout: 10s
       retries: 5
+    profiles:
+      - fs
 
   agraph-ut:
-    image: franzinc/agraph:v8.3.1
-    platform: linux/amd64
+    image: franzinc/agraph:v8.4.3
+    platform: linux/amd64 #agraph doesn't provide arm platform 
     environment:
       - AGRAPH_SUPER_USER=test
       - AGRAPH_SUPER_PASSWORD=xyzzy
     shm_size: 1g
-      # ports:
-      #  - 10035:10035
+    ports:
+      - 10035:10035
     command: >
        bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start
        ; agtool repos create --supersede ontoportal_test
        ; agtool users add anonymous
        ; agtool users grant anonymous root:ontoportal_test:rw
        ; tail -f /agraph/data/agraph.log"
     healthcheck:
-      test: ["CMD-SHELL", "agtool storage-report ontoportal_test || exit 1"]
+      test: ["CMD", "agtool", "storage-report", "ontoportal_test"]
       start_period: 30s #AllegroGraph can take a loooooong time to start
       interval: 20s
       timeout: 10s
       retries: 20
     profiles:
-      - agraph
+      - ag
 
-volumes:
-  bundle:
+  virtuoso-ut:
+    image: openlink/virtuoso-opensource-7:7.2.16
+    environment:
+      - SPARQL_UPDATE=true
+      - DBA_PASSWORD=dba
+      - DAV_PASSWORD=dba
+    ports:
+      - 1111:1111
+      - 8890:8890
+    volumes:
+      - ./test/fixtures/backends/virtuoso_initdb_d:/initdb.d
+    healthcheck:
+      test: [ "CMD-SHELL", "echo 'status();' | isql localhost:1111 dba dba || exit 1" ]
+      start_period: 10s
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    profiles:
+      - vo
+
+  graphdb-ut:
+    image: ontotext/graphdb:10.8.12
+    environment:
+      GDB_HEAP_SIZE: 5G
+      GDB_JAVA_OPTS: >-
+        -Xms5g -Xmx5g
+    ports:
+      - 7200:7200
+      - 7300:7300
+    healthcheck:
+      test: [ "CMD", "curl", "-sf", "http://localhost:7200/repositories/ontoportal_test/health" ]
+      start_period: 10s
+      interval: 10s
+    volumes:
+      - ./test/fixtures/backends/graphdb:/opt/graphdb/dist/configs/templates/data
+    entrypoint: >
+      bash -c " importrdf load -f -c /opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt 
+      ; graphdb -Ddefault.min.distinct.threshold=3000 "
+    profiles:
+      - gd