diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index d8eadf4f..63d32263 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -10,25 +10,28 @@ on: jobs: test: + env: + RUBY_VERSION: '3.2' + COMPOSE_CMD: > + docker compose -f docker-compose.yml + -f dev/compose/linux/${{ matrix.triplestore }}.yml + --profile linux + --profile ${{ matrix.triplestore }} strategy: fail-fast: false matrix: - backend: ['ruby', 'ruby-agraph'] # ruby runs tests with 4store backend and ruby-agraph runs with AllegroGraph backend + #triplestore: [ 'fs', 'ag', 'vo', 'gd' ] + triplestore: [ 'fs', 'ag' ] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up solr configsets - run: ./test/solr/generate_ncbo_configsets.sh - - name: create config.rb file - run: cp config/config.test.rb config/config.rb - - name: Build docker compose - run: docker compose --profile 4store build # profile flag is set in order to build all containers in this step + - name: Build docker container + run: > + $COMPOSE_CMD build - name: Run unit tests - # unit tests are run inside a container - # http://docs.codecov.io/docs/testing-with-docker - run: | - ci_env=`bash <(curl -s https://codecov.io/env)` - docker compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v' + run: > + $COMPOSE_CMD run -v "$PWD/coverage:/app/coverage" -e CI=true + test-linux bundle exec rake test TESTOPTS="-v" - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 with: diff --git a/Dockerfile b/Dockerfile index b9529fd1..0b920394 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG RUBY_VERSION=3.1 +ARG RUBY_VERSION=3.2 ARG DISTRO=bullseye FROM ruby:$RUBY_VERSION-$DISTRO @@ -8,17 +8,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ libxml2 \ libxslt-dev \ + libxslt1-dev zlib1g-dev \ openjdk-11-jre-headless \ raptor2-utils \ && rm -rf /var/lib/apt/lists/* WORKDIR /app +# set default test config +COPY config/config.test.rb config/config.rb + COPY Gemfile* *.gemspec ./ # Copy only the `version.rb` file to prevent missing file errors! COPY lib/ontologies_linked_data/version.rb lib/ontologies_linked_data/ + #Install the exact Bundler version from Gemfile.lock (if it exists) RUN gem update --system && \ if [ -f Gemfile.lock ]; then \ diff --git a/dev/compose/linux/ag.yml b/dev/compose/linux/ag.yml new file mode 100644 index 00000000..b56d9c8e --- /dev/null +++ b/dev/compose/linux/ag.yml @@ -0,0 +1,16 @@ +services: + test-linux: + environment: + GOO_BACKEND_NAME: allegrograph + GOO_PORT: 10035 + GOO_HOST: agraph-ut + GOO_PATH_QUERY: /repositories/ontoportal_test + GOO_PATH_DATA: /repositories/ontoportal_test/statements + GOO_PATH_UPDATE: /repositories/ontoportal_test/statements + depends_on: + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + agraph-ut: + condition: service_healthy diff --git a/dev/compose/linux/docker-compose.ci.yml b/dev/compose/linux/docker-compose.ci.yml new file mode 100644 index 00000000..4397c33d --- /dev/null +++ b/dev/compose/linux/docker-compose.ci.yml @@ -0,0 +1,3 @@ +services: + test-linux: + image: ontologies_linked_data-test-linux:ci diff --git a/dev/compose/linux/fs.yml b/dev/compose/linux/fs.yml new file mode 100644 index 00000000..27acf4b3 --- /dev/null +++ b/dev/compose/linux/fs.yml @@ -0,0 +1,13 @@ +services: + test-linux: + environment: + GOO_BACKEND_NAME: '4store' + GOO_HOST: 4store-ut + GOO_PORT: 9000 + depends_on: + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + 4store-ut: + condition: service_healthy diff --git a/dev/compose/linux/gd.yml b/dev/compose/linux/gd.yml new file mode 100644 index 00000000..87576c73 --- /dev/null +++ b/dev/compose/linux/gd.yml @@ -0,0 +1,16 @@ +services: + test-linux: + environment: + GOO_BACKEND_NAME: graphdb + GOO_PORT: 7200 + GOO_HOST: graphdb-ut + GOO_PATH_QUERY: /repositories/ontoportal_test + GOO_PATH_DATA: /repositories/ontoportal_test/statements + GOO_PATH_UPDATE: /repositories/ontoportal_test/statements + depends_on: + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + graphdb-ut: + condition: service_healthy diff --git a/dev/compose/linux/no-ports.yml b/dev/compose/linux/no-ports.yml new file mode 100644 index 00000000..f42191b9 --- /dev/null +++ b/dev/compose/linux/no-ports.yml @@ -0,0 +1,13 @@ +services: + redis-ut: + ports: [] + solr-ut: + ports: [] + agraph-ut: + ports: [] + 4store-ut: + ports: [] + virtuoso-ut: + ports: [] + graphdb-ut: + ports: [] diff --git a/dev/compose/linux/vo.yml b/dev/compose/linux/vo.yml new file mode 100644 index 00000000..c47dd654 --- /dev/null +++ b/dev/compose/linux/vo.yml @@ -0,0 +1,16 @@ +services: + test-linux: + environment: + GOO_BACKEND_NAME: 'virtuoso' + GOO_HOST: virtuoso-ut + GOO_PORT: 8890 + GOO_PATH_QUERY: /sparql + GOO_PATH_DATA: /sparql + GOO_PATH_UPDATE: /sparql + depends_on: + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + virtuoso-ut: + condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index df84ea47..2aec73dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,72 +1,28 @@ -x-app: &app +# unit tests in containerased env +services: + test-linux: build: context: . args: RUBY_VERSION: '3.2' - # Increase the version number in the image tag every time Dockerfile or its arguments is changed - image: ontologies_ld-dev:0.0.4 - environment: &env + command: ["bash", "-lc", "bundle exec rake test"] + environment: COVERAGE: 'true' # enable simplecov code coverage REDIS_HOST: redis-ut - REDIS_PORT: 6379 - SOLR_TERM_SEARCH_URL: http://solr-term-ut:8983/solr - SOLR_PROP_SEARCH_URL: http://solr-prop-ut:8983/solr - stdin_open: true - tty: true - command: /bin/bash - volumes: - # bundle volume for hosting gems installed by bundle; it speeds up gem install in local development - - bundle:/usr/local/bundle - - .:/app - # mount directory containing development version of the gems if you need to use 'bundle config local' - #- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal - depends_on: &depends_on - solr-prop-ut: - condition: service_healthy - solr-term-ut: + SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr + SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr + depends_on: + solr-ut: condition: service_healthy redis-ut: condition: service_healthy - -services: - # environment wtih 4store backend - ruby: - <<: *app - environment: - <<: *env - GOO_BACKEND_NAME: 4store - GOO_PORT: 9000 - GOO_HOST: 4store-ut - GOO_PATH_QUERY: /sparql/ - GOO_PATH_DATA: /data/ - GOO_PATH_UPDATE: /update/ profiles: - - 4store - depends_on: - <<: *depends_on - 4store-ut: - condition: service_started - - # environment with AllegroGraph backend - ruby-agraph: - <<: *app - environment: - <<: *env - GOO_BACKEND_NAME: ag - GOO_PORT: 10035 - GOO_HOST: agraph-ut - GOO_PATH_QUERY: /repositories/ontoportal_test - GOO_PATH_DATA: /repositories/ontoportal_test/statements - GOO_PATH_UPDATE: /repositories/ontoportal_test/statements - profiles: - - agraph - depends_on: - <<: *depends_on - agraph-ut: - condition: service_healthy + - linux redis-ut: image: redis + ports: + - 6379:6379 command: ["redis-server", "--save", "", "--appendonly", "no"] healthcheck: test: redis-cli ping @@ -74,53 +30,45 @@ services: timeout: 3s retries: 10 + solr-ut: + image: solr:9 + command: bin/solr start -cloud -f + ports: + - 8983:8983 + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8983/solr/admin/info/system?wt=json"] + start_period: 5s + interval: 10s + timeout: 5s + retries: 5 + 4store-ut: image: bde2020/4store platform: linux/amd64 + ports: + - 9000:9000 command: > - bash -c "4s-backend-setup --segments 4 ontoportal_kb - && 4s-backend ontoportal_kb - && 4s-httpd -D -s-1 -p 9000 ontoportal_kb" - profiles: - - 4store - - solr-term-ut: - image: solr:8 - volumes: - - ./test/solr/configsets:/configsets:ro - # ports: - # - "8983:8983" - command: ["solr-precreate", "term_search_core1", "/configsets/term_search"] - healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] - start_period: 5s - interval: 10s - timeout: 5s - retries: 5 - - solr-prop-ut: - image: solr:8 - volumes: - - ./test/solr/configsets:/configsets:ro - # ports: - # - "8984:8983" - command: ["solr-precreate", "prop_search_core1", "/configsets/property_search"] + bash -c "4s-backend-setup --segments 4 ontoportal_test + && 4s-backend ontoportal_test + && 4s-httpd -D -s-1 -p 9000 ontoportal_test" healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/prop_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] + test: ["CMD", "4s-backend-info", "ontoportal_test"] start_period: 5s interval: 10s - timeout: 5s + timeout: 10s retries: 5 + profiles: + - fs agraph-ut: - image: franzinc/agraph:v8.3.1 - platform: linux/amd64 + image: franzinc/agraph:v8.4.3 + platform: linux/amd64 #agraph doesn't provide arm platform environment: - AGRAPH_SUPER_USER=test - AGRAPH_SUPER_PASSWORD=xyzzy shm_size: 1g - # ports: - # - 10035:10035 + ports: + - 10035:10035 command: > bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start ; agtool repos create --supersede ontoportal_test @@ -128,13 +76,51 @@ services: ; agtool users grant anonymous root:ontoportal_test:rw ; tail -f /agraph/data/agraph.log" healthcheck: - test: ["CMD-SHELL", "agtool storage-report ontoportal_test || exit 1"] + test: ["CMD", "agtool", "storage-report", "ontoportal_test"] start_period: 30s #AllegroGraph can take a loooooong time to start interval: 20s timeout: 10s retries: 20 profiles: - - agraph + - ag -volumes: - bundle: + virtuoso-ut: + image: openlink/virtuoso-opensource-7:7.2.16 + environment: + - SPARQL_UPDATE=true + - DBA_PASSWORD=dba + - DAV_PASSWORD=dba + ports: + - 1111:1111 + - 8890:8890 + volumes: + - ./test/fixtures/backends/virtuoso_initdb_d:/initdb.d + healthcheck: + test: [ "CMD-SHELL", "echo 'status();' | isql localhost:1111 dba dba || exit 1" ] + start_period: 10s + interval: 10s + timeout: 5s + retries: 3 + profiles: + - vo + + graphdb-ut: + image: ontotext/graphdb:10.8.12 + environment: + GDB_HEAP_SIZE: 5G + GDB_JAVA_OPTS: >- + -Xms5g -Xmx5g + ports: + - 7200:7200 + - 7300:7300 + healthcheck: + test: [ "CMD", "curl", "-sf", "http://localhost:7200/repositories/ontoportal_test/health" ] + start_period: 10s + interval: 10s + volumes: + - ./test/fixtures/backends/graphdb:/opt/graphdb/dist/configs/templates/data + entrypoint: > + bash -c " importrdf load -f -c /opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt + ; graphdb -Ddefault.min.distinct.threshold=3000 " + profiles: + - gd diff --git a/rakelib/docker_based_test.rake b/rakelib/docker_based_test.rake new file mode 100644 index 00000000..c3cd32aa --- /dev/null +++ b/rakelib/docker_based_test.rake @@ -0,0 +1,240 @@ +# Docker compose driven unit test orchestration +# +# Notes: +# - Backend names match compose profile names (ag, fs, vo, gd). +# - Hostnames are NOT set here. The app defaults them (localhost for host runs). +# - Linux container env is provided via compose override files: +# dev/compose/linux/ag.yml +# dev/compose/linux/fs.yml +# dev/compose/linux/vo.yml +# dev/compose/linux/gd.yml +namespace :test do + namespace :docker do + BASE_COMPOSE = 'docker-compose.yml' + LINUX_OVERRIDE_DIR = 'dev/compose/linux' + LINUX_NO_PORTS_OVERRIDE = "#{LINUX_OVERRIDE_DIR}/no-ports.yml" + TIMEOUT = (ENV['OP_TEST_DOCKER_TIMEOUT'] || '600').to_i + DEFAULT_BACKEND = (ENV['OP_TEST_DOCKER_BACKEND'] || 'fs').to_sym + + # Minimal per-backend config for host runs only. + # Do not set hostnames here. The app defaults them. + BACKENDS = { + ag: { + host_env: { + 'GOO_BACKEND_NAME' => 'allegrograph', + 'GOO_PORT' => '10035', + 'GOO_PATH_QUERY' => '/repositories/ontoportal_test', + 'GOO_PATH_DATA' => '/repositories/ontoportal_test/statements', + 'GOO_PATH_UPDATE' => '/repositories/ontoportal_test/statements' + } + }, + fs: { + host_env: { + 'GOO_BACKEND_NAME' => '4store', + 'GOO_PORT' => '9000', + 'GOO_PATH_QUERY' => '/sparql/', + 'GOO_PATH_DATA' => '/data/', + 'GOO_PATH_UPDATE' => '/update/' + } + }, + vo: { + host_env: { + 'GOO_BACKEND_NAME' => 'virtuoso', + 'GOO_PORT' => '8890', + 'GOO_PATH_QUERY' => '/sparql', + 'GOO_PATH_DATA' => '/sparql', + 'GOO_PATH_UPDATE' => '/sparql' + } + }, + gd: { + host_env: { + 'GOO_BACKEND_NAME' => 'graphdb', + 'GOO_PORT' => '7200', + 'GOO_PATH_QUERY' => '/repositories/ontoportal_test', + 'GOO_PATH_DATA' => '/repositories/ontoportal_test/statements', + 'GOO_PATH_UPDATE' => '/repositories/ontoportal_test/statements' + } + } + }.freeze + + def abort_with(msg) + warn(msg) + exit(1) + end + + def shell!(cmd) + system(cmd) || abort_with("Command failed: #{cmd}") + end + + def cfg!(key) + cfg = BACKENDS[key] + abort_with("Unknown backend key: #{key}. Supported: #{BACKENDS.keys.join(', ')}") unless cfg + cfg + end + + def compose_files(*files) + files.flatten.map { |f| "-f #{f}" }.join(' ') + end + + def linux_override_for(key) + "#{LINUX_OVERRIDE_DIR}/#{key}.yml" + end + + def compose_up(key, files:) + # Host tests use only the backend profile. Linux tests add the linux profile. + # `docker compose up --wait` only applies to services started by `up`, + # so linux runs still call `run` separately after this wait completes. + shell!("docker compose #{compose_files(files)} --profile #{key} up -d --wait --wait-timeout #{TIMEOUT}") + end + + def compose_down(files:) + return puts('OP_KEEP_CONTAINERS=1 set, skipping docker compose down') if ENV['OP_KEEP_CONTAINERS'] == '1' + + shell!( + "docker compose #{compose_files(files)} " \ + '--profile ag --profile fs --profile vo --profile gd --profile linux down' + ) + end + + def apply_host_env(key) + cfg!(key)[:host_env].each { |k, v| ENV[k] = v } + end + + def run_host_tests(key) + apply_host_env(key) + files = [BASE_COMPOSE] + + compose_up(key, files: files) + Rake::Task['test'].invoke + end + + def run_linux_tests(key) + override = linux_override_for(key) + abort_with("Missing compose override file: #{override}") unless File.exist?(override) + abort_with("Missing compose override file: #{LINUX_NO_PORTS_OVERRIDE}") unless File.exist?(LINUX_NO_PORTS_OVERRIDE) + + files = [BASE_COMPOSE, override, LINUX_NO_PORTS_OVERRIDE] + # docker compose is handleling wait_for_healthy + compose_up(key, files: files) + + shell!( + "docker compose #{compose_files(files)} --profile linux --profile #{key} " \ + 'run --rm --build test-linux bundle exec rake test TESTOPTS="-v"' + ) + end + + def run_linux_shell(key) + override = linux_override_for(key) + abort_with("Missing compose override file: #{override}") unless File.exist?(override) + abort_with("Missing compose override file: #{LINUX_NO_PORTS_OVERRIDE}") unless File.exist?(LINUX_NO_PORTS_OVERRIDE) + + files = [BASE_COMPOSE, override, LINUX_NO_PORTS_OVERRIDE] + compose_up(key, files: files) + + shell!( + "docker compose #{compose_files(files)} --profile linux --profile #{key} " \ + 'run --rm --build test-linux bash' + ) + end + + # + # Public tasks + # + + desc 'Run unit tests with AllegroGraph backend (docker deps, host Ruby)' + task :ag do + run_host_tests(:ag) + ensure + Rake::Task['test'].reenable + compose_down(files: [BASE_COMPOSE]) + end + + desc 'Run unit tests with AllegroGraph backend (docker deps, Linux container)' + task 'ag:linux' do + files = [BASE_COMPOSE, linux_override_for(:ag)] + begin + run_linux_tests(:ag) + ensure + compose_down(files: files) + end + end + + desc 'Run unit tests with 4store backend (docker deps, host Ruby)' + task :fs do + run_host_tests(:fs) + ensure + Rake::Task['test'].reenable + compose_down(files: [BASE_COMPOSE]) + end + + desc 'Run unit tests with 4store backend (docker deps, Linux container)' + task 'fs:linux' do + files = [BASE_COMPOSE, linux_override_for(:fs)] + begin + run_linux_tests(:fs) + ensure + compose_down(files: files) + end + end + + desc 'Run unit tests with Virtuoso backend (docker deps, host Ruby)' + task :vo do + run_host_tests(:vo) + ensure + Rake::Task['test'].reenable + compose_down(files: [BASE_COMPOSE]) + end + + desc 'Run unit tests with Virtuoso backend (docker deps, Linux container)' + task 'vo:linux' do + files = [BASE_COMPOSE, linux_override_for(:vo)] + begin + run_linux_tests(:vo) + ensure + compose_down(files: files) + end + end + + desc 'Run unit tests with GraphDB backend (docker deps, host Ruby)' + task :gd do + run_host_tests(:gd) + ensure + Rake::Task['test'].reenable + compose_down(files: [BASE_COMPOSE]) + end + + desc 'Run unit tests with GraphDB backend (docker deps, Linux container)' + task 'gd:linux' do + files = [BASE_COMPOSE, linux_override_for(:gd)] + begin + run_linux_tests(:gd) + ensure + compose_down(files: files) + end + end + + desc 'Start a shell in the Linux test container (default backend: fs)' + task :shell, [:backend] do |_t, args| + key = (args[:backend] || DEFAULT_BACKEND).to_sym + cfg!(key) + files = [BASE_COMPOSE, linux_override_for(key), LINUX_NO_PORTS_OVERRIDE] + begin + run_linux_shell(key) + ensure + compose_down(files: files) + end + end + + desc 'Start backend services for development (default backend: fs)' + task :up, [:backend] do |_t, args| + key = (args[:backend] || DEFAULT_BACKEND).to_sym + cfg!(key) + compose_up(key, files: [BASE_COMPOSE]) + end + + desc 'Stop backend services for development (default backend: fs)' + task :down, [:backend] do |_t, args| + compose_down(files: [BASE_COMPOSE]) + end + end +end diff --git a/test/fixtures/backends/graphdb/graphdb-repo-config.ttl b/test/fixtures/backends/graphdb/graphdb-repo-config.ttl new file mode 100644 index 00000000..84032a0b --- /dev/null +++ b/test/fixtures/backends/graphdb/graphdb-repo-config.ttl @@ -0,0 +1,33 @@ +@prefix rdfs: . +@prefix rep: . +@prefix sail: . +@prefix xsd: . + +<#ontoportal_test> a rep:Repository; + rep:repositoryID "ontoportal_test"; + rep:repositoryImpl [ + rep:repositoryType "graphdb:SailRepository"; + [ + "http://example.org/owlim#"; + "false"; + ""; + "true"; + "false"; + "true"; + "true"; + "32"; + "10000000"; + ""; + "true"; + ""; + "0"; + "0"; + "false"; + "file-repository"; + "rdfsplus-optimized"; + "storage"; + "false"; + sail:sailType "owlim:Sail" + ] + ]; + rdfs:label "" . diff --git a/test/fixtures/backends/graphdb/graphdb-test-load.nt b/test/fixtures/backends/graphdb/graphdb-test-load.nt new file mode 100644 index 00000000..e69de29b diff --git a/test/fixtures/backends/virtuoso_initdb_d/virtuoso-grant-write-sparql-access.sql b/test/fixtures/backends/virtuoso_initdb_d/virtuoso-grant-write-sparql-access.sql new file mode 100644 index 00000000..d509c6fb --- /dev/null +++ b/test/fixtures/backends/virtuoso_initdb_d/virtuoso-grant-write-sparql-access.sql @@ -0,0 +1,3 @@ +GRANT EXECUTE ON DB.DBA.SPARQL_INSERT_DICT_CONTENT TO "SPARQL"; +GRANT SPARQL_UPDATE TO "SPARQL"; +DB.DBA.RDF_DEFAULT_USER_PERMS_SET ('nobody', 7);