diff --git a/.DS_Store b/.DS_Store index b3390a6..b4fd886 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.github/workflows/plugin-integration.yml b/.github/workflows/plugin-integration.yml index 245aa3a..ae16155 100644 --- a/.github/workflows/plugin-integration.yml +++ b/.github/workflows/plugin-integration.yml @@ -7,12 +7,10 @@ on: jobs: build: runs-on: ${{ matrix.operating-system }} - environment: - name: ${{ github.ref_name }} strategy: matrix: operating-system: [ubuntu-latest] - versions: [ { jdk: 17, mapping-service: v1.0.5 }, { jdk: 21, mapping-service: latest } ] + versions: [ { jdk: 17, mapping-service: v2.0.2 }, { jdk: 21, mapping-service: latest } ] steps: - name: Checkout repository @@ -34,8 +32,9 @@ jobs: JAR_VERSION=$(./mappingservice-plugin/gradlew printVersion -q -p ./mappingservice-plugin/) JAR_VERSION=${JAR_VERSION##*$'\n'} ./mappingservice-plugin/gradlew clean jar -p ./mappingservice-plugin/ + ls -ll ./mappingservice-plugin/build/libs/ echo "JAR_VERSION=${JAR_VERSION}" - mv -v ./mappingservice-plugin/build/libs/ApeHePlugin-$JAR_VERSION-plain.jar ./mappingservice-plugin/build/libs/ApeHeplugin.jar + mv -v ./mappingservice-plugin/build/libs/ApeHePlugin-$JAR_VERSION.jar ./mappingservice-plugin/build/libs/ApeHeplugin.jar env: VERSION_OVERRIDE_BY_BRANCH: ${{ steps.extract_branch.outputs.branch }} @@ -47,13 +46,11 @@ jobs: test: runs-on: ${{ matrix.operating-system }} - environment: - name: ${{ github.ref_name }} strategy: fail-fast: false matrix: operating-system: [ubuntu-latest] - versions: [ { jdk: 17, mapping-service: v1.0.5 }, { jdk: 21, mapping-service: latest } ] + versions: [ { jdk: 17, mapping-service: v2.0.2 }, { jdk: 21, mapping-service: latest } ] needs: build steps: @@ -73,7 +70,7 @@ jobs: - name: Run Docker Container run: | - docker run -d -p 8095:8095 -e PIP_BREAK_SYSTEM_PACKAGES=1 -v ./plugins/ApeHeplugin.jar:/spring/mapping-service/plugins/ApeHeplugin.jar --name mapping4docker ghcr.io/kit-data-manager/mapping-service:${{ matrix.versions.mapping-service }} + docker run -d -p 8095:8095 -e PIP_BREAK_SYSTEM_PACKAGES=1 -e SPRING_APPLICATION_JSON='{"mapping-service.executionTimeout":300}' -v ./plugins/ApeHeplugin.jar:/spring/mapping-service/plugins/ApeHeplugin.jar --name mapping4docker ghcr.io/kit-data-manager/mapping-service:${{ matrix.versions.mapping-service }} echo "Waiting for mapping service to be healthy..." while true; do if ! docker ps | grep -q mapping4docker; then @@ -84,35 +81,20 @@ jobs: echo "Service is running." break fi - echo "Waiting..." + echo "Waiting for the service to be ready ..." docker logs --tail 20 mapping4docker sleep 5 done - - name: Install Hurl & Prepare JSON - run: | - curl -LO https://github.com/Orange-OpenSource/hurl/releases/download/6.0.0/hurl_6.0.0_amd64.deb - sudo dpkg -i hurl_6.0.0_amd64.deb - sudo apt install -y dos2unix - - # Fetch mappingType dynamically - mappingType=$(curl -s http://localhost:8095/api/v1/mappingAdministration/types | jq -r '.[0].id') - - echo "Using mappingType: $mappingType" - - echo "{\"mappingId\":\"96\",\"mappingType\":\"$mappingType\",\"title\":\"apeHe from CI test\",\"description\":\"\",\"acl\":[]}" > record.json - echo '{"entry.title.value": "entry.title"}' > document.json - - unix2dos -n ./mappingservice-plugin/integrationtests/basic.hurl ./mappingservice-plugin/integrationtests/basic_crlf.hurl - - name: Run Tests with Hurl run: | - hurl --variable host=http://localhost:8095 --test ./mappingservice-plugin/integrationtests/basic_crlf.hurl --verbose --file-root . + curl --location --remote-name https://github.com/Orange-OpenSource/hurl/releases/download/6.0.0/hurl_6.0.0_amd64.deb + sudo dpkg -i hurl_6.0.0_amd64.deb + sudo apt install -y dos2unix + unix2dos -n ./mappingservice-plugin/integrationtests/basic.hurl ./mappingservice-plugin/integrationtests/basic_crlf.hurl + hurl --variable host=http://localhost:8095 --test ./mappingservice-plugin/integrationtests/basic_crlf.hurl --verbose --file-root . env: VERSION_OVERRIDE_BY_BRANCH: ${{ steps.extract_branch.outputs.branch }} - - - name: Clean up temp files - run: rm -f record.json document.json - + - name: Stop Docker Container - run: docker stop mapping4docker + run: docker stop mapping4docker \ No newline at end of file diff --git a/CITATION.cff b/CITATION.cff index c8c6d15..38abc79 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -4,7 +4,7 @@ type: software title: APE-HE mapper abstract: APE-HE Mapper is a tool designed for mapping APE-HE (Advanced Photoelectric Effect - High Energy) metadata to a uniform, schema-compliant json format. -version: v1.0.0 +version: v1.1.0rc keywords: - APE-HE - neXus diff --git a/codemeta.json b/codemeta.json index 8087cda..6575089 100644 --- a/codemeta.json +++ b/codemeta.json @@ -25,7 +25,7 @@ ], "name": "APE-HE mapper", "description": "APE-HE Mapper is a tool designed for mapping APE-HE (Advanced Photoelectric Effect - High Energy) metadata to a uniform, schema-compliant json format.", - "version": "v1.0.0", + "version": "v1.1.0rc", "keywords": [ "APE-HE", "neXus", diff --git a/mappingservice-plugin/build.gradle b/mappingservice-plugin/build.gradle index 29abdad..119f5bc 100644 --- a/mappingservice-plugin/build.gradle +++ b/mappingservice-plugin/build.gradle @@ -1,12 +1,20 @@ -import org.tomlj.Toml +buildscript { + // Buildscript repositories + repositories { + mavenLocal() + mavenCentral() + } + + dependencies { + classpath("org.tomlj:tomlj:1.1.1") + } +} plugins { - id 'io.spring.dependency-management' version "1.1.7" - id 'org.springframework.boot' version "3.4.4" id 'java' } -group 'edu.kit.datamanager' +import org.tomlj.Toml repositories { mavenLocal() @@ -14,9 +22,7 @@ repositories { } dependencies { - implementation 'org.springframework:spring-core:6.2.5' - implementation 'org.slf4j:slf4j-api:2.0.17' - implementation files("src/main/lib/mapping-service-plain.jar") + implementation 'edu.kit.datamanager:mapping-plugin-core:2.0.0' } if (System.getenv('VERSION_OVERRIDE_BY_BRANCH')) { @@ -32,22 +38,24 @@ tasks.register('printVersion') { println project.version } +group 'edu.kit.datamanager' + //Task for creating a resource file with the version info tasks.register("generateVersionProps", WriteProperties) { t -> def generatedResourcesDir = project.layout.buildDirectory.dir(["resources", "main"].join(File.separator)) - def outputFile = generatedResourcesDir.map { it.file("apeHepluginversion.properties") } + def outputFile = generatedResourcesDir.map { it.file("apehe_nxs2json.properties") } t.destinationFile = outputFile.get().asFile t.property("version", project.version) } -resolveMainClassName.dependsOn("generateVersionProps") +//resolveMainClassName.dependsOn("generateVersionProps") jar { dependsOn(generateVersionProps) archiveFileName } -bootJar { - enabled = false -} \ No newline at end of file +//bootJar { +// enabled = false +//} \ No newline at end of file diff --git a/mappingservice-plugin/integrationtests/basic.hurl b/mappingservice-plugin/integrationtests/basic.hurl index 71e2e5a..d4ceace 100644 --- a/mappingservice-plugin/integrationtests/basic.hurl +++ b/mappingservice-plugin/integrationtests/basic.hurl @@ -8,25 +8,34 @@ HTTP 200 # Get mapping types from the mapping service # to fetch the first available id # ---------------------------------------------------------- -GET {{host}}/api/v1/mappingAdministration/types +GET {{host}}/api/v1/mappingAdministration/plugins HTTP 200 [Captures] -mappingType: jsonpath "$[0].id" +mappingType: regex "(?s)\"id\"\\s*:\\s*\"([^\"]+)\"\\s*,\\s*[^}]*?\"name\"\\s*:\\s*\"ApeHe_nxs2JSON\"" # ---------------- BEGIN Test Mapping for Ape-He ----------- # ---------------------------------------------------------- -# Register a dummy mapping with created files: -# - record.json (containing the mapping scheme metadata) -# - document.json (containing the dummy map file) +# Register a dummy mapping with embedded JSON # ---------------------------------------------------------- POST {{host}}/api/v1/mappingAdministration/ +Content-Type: multipart/form-data; boundary=boundary [Options] -variable: id1=96 -[MultipartFormData] -record: file,record.json; type=application/json -document: file,document.json; type=application/json -HTTP 201 # Expect plugin to be successfully created +variable: id1=0 +``` +--boundary +Content-Disposition: form-data; name="record"; filename="blob" +Content-Type: application/json + +{"mappingId":"{{id1}}","mappingType": "{{mappingType}}","title":"APE-HE from CI test","description":"","acl":[]} +--boundary +Content-Disposition: form-data; name="document"; filename="blob" +Content-Type: application/json + +{"entry.title.value": "entry.title"} +--boundary-- +``` +HTTP 201 # ---------------------------------------------------------- # Execute mapping with test data diff --git a/mappingservice-plugin/src/main/java/edu/kit/datamanager/apeHeplugin/ApeHePlugin.java b/mappingservice-plugin/src/main/java/edu/kit/datamanager/apeHeplugin/ApeHePlugin.java index 6ed86d5..db645fa 100644 --- a/mappingservice-plugin/src/main/java/edu/kit/datamanager/apeHeplugin/ApeHePlugin.java +++ b/mappingservice-plugin/src/main/java/edu/kit/datamanager/apeHeplugin/ApeHePlugin.java @@ -1,60 +1,15 @@ package edu.kit.datamanager.apeHeplugin; -import edu.kit.datamanager.mappingservice.plugins.*; -import edu.kit.datamanager.mappingservice.util.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.MimeType; -import org.springframework.util.MimeTypeUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; +import edu.kit.datamanager.mappingservice.plugins.AbstractPythonMappingPlugin; import java.nio.file.Path; -import java.util.Properties; - -public class ApeHePlugin implements IMappingPlugin{ - private static String version; +public class ApeHePlugin extends AbstractPythonMappingPlugin{ - private final Logger LOGGER = LoggerFactory.getLogger(ApeHePlugin.class); - private final String REPOSITORY = "https://github.com/kit-data-manager/ape_he_mapper"; - private String TAG; - private Path dir; + private static final String REPOSITORY = "https://github.com/kit-data-manager/ape_he_mapper"; - //private String pluginVenv = "venv/PluginVenv"; - //private String venvInterpreter; public ApeHePlugin() { - try { - // Get the context class loader - ClassLoader classLoader = this.getClass().getClassLoader(); - // TODO: do we need to make sure that the resource path is somehow related to the current plugin to avoid loading the wrong property file in case of identical property names? - URL resource = classLoader.getResource("apeHepluginversion.properties"); - LOGGER.info("Resource file: {}", resource); - if (resource != null) { - // Load the properties file - try (InputStream input = resource.openStream()) { - Properties properties = new Properties(); - properties.load(input); - version = properties.getProperty("version"); - TAG = version; - } - } else { - System.err.println("Properties file not found!"); - version = "unavailable"; - TAG = "unavailable"; - } - - //if (System.getProperty("os.name").startsWith("Windows")) { - // venvInterpreter = pluginVenv + "/Scripts/python.exe"; - //} else { - // venvInterpreter = pluginVenv + "/bin/python3"; - //} - - } catch (IOException e) { - e.printStackTrace(); - } + super("ApeHe_nxs2JSON", REPOSITORY); } @Override @@ -68,86 +23,32 @@ public String description() { } @Override - public String version() { - return version; - } - - @Override - public String uri() { - return REPOSITORY; - } - - @Override - public MimeType[] inputTypes() { - return new MimeType[]{ - MimeTypeUtils.parseMimeType("application/octet-stream"), - MimeTypeUtils.parseMimeType("application/x-hdf5"), - MimeTypeUtils.parseMimeType("application/zip") + public String[] inputTypes() { + return new String[]{ + "application/octet-stream", + "application/x-hdf5", + "application/zip" }; } @Override - public MimeType[] outputTypes() { - return new MimeType[]{ - MimeTypeUtils.APPLICATION_JSON, - MimeTypeUtils.parseMimeType("application/zip") + public String[] outputTypes() { + return new String[]{ + "application/json", + "application/zip" }; } @Override - public void setup() { - LOGGER.info("Checking and installing dependencies for the tool: "); - //TODO: test for minimal python version? - try { - LOGGER.info("Cloning git repository {}, Tag {}", REPOSITORY, TAG); - dir = FileUtil.cloneGitRepository(REPOSITORY, TAG); - // Install Python dependencies - - ProcessBuilder pb = new ProcessBuilder("python3", "-m", "pip", "install", "-r", dir + "/requirements.dist.txt"); - pb.inheritIO(); - Process p = pb.start(); - - int exitCode = p.waitFor(); - if (exitCode != 0) { - LOGGER.error("Failed to install Python packages"); - } - - } catch (Exception e) { - e.printStackTrace(); - } - } - /* public void setup() { - LOGGER.trace("Setting up mapping plugin {} {}", name(), version()); - //TODO: test for minimal python version? - try { - LOGGER.info("Cloning git repository {}, Tag {}", REPOSITORY, TAG); - dir = FileUtil.cloneGitRepository(REPOSITORY, TAG); - // Install Python dependencies - - MappingPluginState venvState = PythonRunnerUtil.runPythonScript("-m", "venv", "--system-site-packages", dir + "/" + pluginVenv); - if (MappingPluginState.SUCCESS().getState().equals(venvState.getState())) { - LOGGER.info("Venv for plugin installed successfully."); - LOGGER.info("Installing packages"); - ShellRunnerUtil.run(dir + "/" + venvInterpreter, "-m", "pip", "install", "-r", dir + "/" + "requirements.dist.txt"); - } else { - LOGGER.error("venv installation was not successful"); - } - - } catch (Exception e) { - e.printStackTrace(); - } - } */ - - @Override - public MappingPluginState mapFile(Path mappingFile, Path inputFile, Path outputFile) throws MappingPluginException { - long startTime = System.currentTimeMillis(); - LOGGER.trace("Run ApeHe-Mapping-Tool on '{}' with mapping '{}' -> '{}'", mappingFile, inputFile, outputFile); - //MappingPluginState result = ShellRunnerUtil.run(dir + "/" + venvInterpreter, dir + "/plugin_wrapper.py", "-m", mappingFile.toString(), "-i", inputFile.toString(), "-o", outputFile.toString()); - String[] args = {"-m", mappingFile.toString(), "-i", inputFile.toString(), "-o", outputFile.toString()}; - MappingPluginState result = PythonRunnerUtil.runPythonScript(dir + "/plugin_wrapper.py", args); - long endTime = System.currentTimeMillis(); - long totalTime = endTime - startTime; - LOGGER.info("Execution time of mapFile: {} milliseconds", totalTime); - return result; + public String[] getCommandArray(Path workingDir, Path mappingFile, Path inputFile, Path outputFile) { + return new String[]{ + workingDir + "/plugin_wrapper.py", + "-m", + mappingFile.toString(), + "-i", + inputFile.toString(), + "-o", + outputFile.toString() + }; } } diff --git a/mappingservice-plugin/src/main/lib/mapping-service-plain.jar b/mappingservice-plugin/src/main/lib/mapping-service-plain.jar deleted file mode 100644 index de0f428..0000000 Binary files a/mappingservice-plugin/src/main/lib/mapping-service-plain.jar and /dev/null differ diff --git a/somesy.toml b/somesy.toml index f93193e..3b46566 100644 --- a/somesy.toml +++ b/somesy.toml @@ -1,6 +1,6 @@ [project] name = "APE-HE mapper" -version = "v1.0.0" +version = "v1.1.0rc" description = "APE-HE Mapper is a tool designed for mapping APE-HE (Advanced Photoelectric Effect - High Energy) metadata to a uniform, schema-compliant json format." keywords = ["APE-HE", "neXus", "metadata", "extraction", "schema"] diff --git a/src/.DS_Store b/src/.DS_Store index 8f4d54c..1adbe2b 100644 Binary files a/src/.DS_Store and b/src/.DS_Store differ diff --git a/src/model/SchemaConcepts/codegen/__pycache__/SchemaClasses_APE_HE.cpython-312.pyc b/src/model/SchemaConcepts/codegen/__pycache__/SchemaClasses_APE_HE.cpython-312.pyc index 626a3e9..cf232cc 100644 Binary files a/src/model/SchemaConcepts/codegen/__pycache__/SchemaClasses_APE_HE.cpython-312.pyc and b/src/model/SchemaConcepts/codegen/__pycache__/SchemaClasses_APE_HE.cpython-312.pyc differ diff --git a/src/model/SchemaConcepts/codegen/__pycache__/__init__.cpython-312.pyc b/src/model/SchemaConcepts/codegen/__pycache__/__init__.cpython-312.pyc index 5fcde19..a7c319a 100644 Binary files a/src/model/SchemaConcepts/codegen/__pycache__/__init__.cpython-312.pyc and b/src/model/SchemaConcepts/codegen/__pycache__/__init__.cpython-312.pyc differ diff --git a/tests/.DS_Store b/tests/.DS_Store index 68c253f..4c865a4 100644 Binary files a/tests/.DS_Store and b/tests/.DS_Store differ