icat.lucene support for InvestigationSample

.github/workflows/ci-build.yml

@@ -37,7 +37,7 @@ jobs:
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4.0.0
         with:
           repository: icatproject-contrib/icat-ansible
-          ref: no-icat-install
+          ref: icat.lucene_5.0.0
           path: icat-ansible
       - name: Install Ansible
         run: pip install -r icat-ansible/requirements.txt

src/main/config/run.properties.example

@@ -59,7 +59,7 @@ search.maxSearchTimeSeconds = 5
 # Configure this option to prevent certain entities being indexed
 # For example, remove Datafile and DatafileParameter if these are not of interest
 # Note then when commented out, the full set of all possible entities will be indexed - to disable all search functionality, instead comment out search.engine or search.urls
-!search.entitiesToIndex = Datafile DatafileFormat DatafileParameter Dataset DatasetParameter DatasetType DatasetTechnique Facility Instrument InstrumentScientist Investigation InvestigationInstrument InvestigationParameter InvestigationType InvestigationUser ParameterType Sample SampleType SampleParameter User
+!search.entitiesToIndex = Datafile DatafileFormat DatafileParameter Dataset DatasetParameter DatasetType DatasetTechnique Facility Instrument InstrumentScientist Investigation InvestigationInstrument InvestigationParameter InvestigationSample InvestigationType InvestigationUser ParameterType Sample SampleType SampleParameter User
 
 # List members of cluster
 !cluster = http://vm200.nubes.stfc.ac.uk:8080 https://smfisher:8181				

src/main/java/org/icatproject/core/entity/Investigation.java

@@ -383,6 +383,13 @@ public static Map<String, Relationship[]> getDocumentFields() throws IcatExcepti
 			Relationship[] parameterTypeRelationships = {
 					EntityInfoHandler.getRelationshipsByName(Investigation.class).get("parameters"),
 					EntityInfoHandler.getRelationshipsByName(InvestigationParameter.class).get("type") };
+			Relationship[] sampleRelationships = {
+					EntityInfoHandler.getRelationshipsByName(Investigation.class).get("samples"),
+					EntityInfoHandler.getRelationshipsByName(InvestigationSample.class).get("sample") };
+			Relationship[] sampleTypeRelationships = {
+					EntityInfoHandler.getRelationshipsByName(Investigation.class).get("samples"),
+					EntityInfoHandler.getRelationshipsByName(InvestigationSample.class).get("sample"),
+					EntityInfoHandler.getRelationshipsByName(Sample.class).get("type") };
 			documentFields.put("name", null);
 			documentFields.put("visitId", null);
 			documentFields.put("title", null);
@@ -406,6 +413,9 @@ public static Map<String, Relationship[]> getDocumentFields() throws IcatExcepti
 			documentFields.put("InvestigationParameter stringValue", parameterRelationships);
 			documentFields.put("InvestigationParameter numericValue", parameterRelationships);
 			documentFields.put("InvestigationParameter dateTimeValue", parameterRelationships);
+			documentFields.put("InvestigationSample sample.id", sampleRelationships);
+			documentFields.put("InvestigationSample sample.name", sampleRelationships);
+			documentFields.put("InvestigationSample type.name", sampleTypeRelationships);
 		}
 		return documentFields;
 	}

src/main/java/org/icatproject/core/entity/InvestigationSample.java

@@ -2,7 +2,12 @@
 
 import java.io.Serializable;
 
+import org.icatproject.core.IcatException;
+import org.icatproject.core.manager.search.SearchApi;
+
+import jakarta.json.stream.JsonGenerator;
 import jakarta.persistence.Entity;
+import jakarta.persistence.EntityManager;
 import jakarta.persistence.FetchType;
 import jakarta.persistence.JoinColumn;
 import jakarta.persistence.ManyToOne;
@@ -38,4 +43,14 @@ public void setInvestigation(Investigation investigation) {
 	public void setSample(Sample sample) {
 		this.sample = sample;
 	}
+
+	@Override
+	public void getDoc(EntityManager entityManager, JsonGenerator gen) throws IcatException {
+		if (sample.getName() == null) {
+			sample = entityManager.find(sample.getClass(), sample.id);
+		}
+		sample.getDoc(entityManager, gen);
+		SearchApi.encodeLong(gen, "investigation.id", investigation.id);
+		SearchApi.encodeLong(gen, "id", id);
+	}
 }

src/main/java/org/icatproject/core/entity/Sample.java

@@ -48,8 +48,7 @@ public class Sample extends EntityBaseBean implements Serializable {
 	@ManyToOne(fetch = FetchType.LAZY)
 	private SampleType type;
 
-	public static Set<String> docFields = new HashSet<>(
-			Arrays.asList("sample.name", "sample.id", "sample.investigation.id"));
+	public static Set<String> docFields = new HashSet<>(Arrays.asList("sample.name", "sample.id"));
 
 	/* Needed for JPA */
 	public Sample() {

src/main/java/org/icatproject/core/manager/EntityBeanManager.java

@@ -1617,6 +1617,8 @@ private JsonObject buildFacetQuery(Class<? extends EntityBaseBean> klass, String
 				return SearchManager.buildFacetQuery(results, resultIdField, "investigation.id", jsonFacet);
 			} else if (target.contains("Parameter")) {
 				relationship = EntityInfoHandler.getRelationshipsByName(klass).get("parameters");
+			} else if (target.contains("InvestigationSample")) {
+				relationship = EntityInfoHandler.getRelationshipsByName(klass).get("samples");
 			} else {
 				relationship = EntityInfoHandler.getRelationshipsByName(klass).get(target.toLowerCase() + "s");
 			}

src/main/java/org/icatproject/core/manager/PropertyHandler.java

@@ -408,8 +408,8 @@ private void init() {
 				entitiesToIndex.addAll(Arrays.asList("Datafile", "DatafileFormat", "DatafileParameter",
 						"Dataset", "DatasetParameter", "DatasetType", "DatasetTechnique", "Facility", "Instrument",
 						"InstrumentScientist", "Investigation", "InvestigationInstrument", "InvestigationParameter",
-						"InvestigationType", "InvestigationUser", "ParameterType", "Sample", "SampleType",
-						"SampleParameter", "User"));
+						"InvestigationSample", "InvestigationType", "InvestigationUser", "ParameterType", "Sample",
+						"SampleType", "SampleParameter", "User"));
 				logger.info("search.entitiesToIndex not set. Defaulting to: {}", entitiesToIndex.toString());
 			}
 			formattedProps.add("search.entitiesToIndex " + entitiesToIndex.toString());

src/test/java/org/icatproject/core/manager/TestEntityInfo.java

@@ -49,8 +49,8 @@ public void testHasSearchDoc() throws Exception {
 		Set<String> docdbeans = new HashSet<>(Arrays.asList("Datafile", "DatafileFormat", "DatafileParameter",
 				"Dataset", "DatasetParameter", "DatasetTechnique", "DatasetType", "Facility", "Instrument",
 				"InstrumentScientist", "Investigation", "InvestigationFacilityCycle", "InvestigationInstrument",
-				"InvestigationParameter", "InvestigationType", "InvestigationUser", "ParameterType", "Sample",
-				"SampleType", "SampleParameter", "Technique", "User"));
+				"InvestigationParameter", "InvestigationSample", "InvestigationType", "InvestigationUser",
+				"ParameterType", "Sample", "SampleType", "SampleParameter", "Technique", "User"));
 		for (String beanName : EntityInfoHandler.getEntityNamesList()) {
 			@SuppressWarnings("unchecked")
 			Class<? extends EntityBaseBean> bean = EntityInfoHandler.getClass(beanName);

src/test/java/org/icatproject/core/manager/TestSearchApi.java

@@ -42,6 +42,7 @@
 import org.icatproject.core.entity.InvestigationFacilityCycle;
 import org.icatproject.core.entity.InvestigationInstrument;
 import org.icatproject.core.entity.InvestigationParameter;
+import org.icatproject.core.entity.InvestigationSample;
 import org.icatproject.core.entity.InvestigationType;
 import org.icatproject.core.entity.InvestigationUser;
 import org.icatproject.core.entity.Parameter;
@@ -505,15 +506,19 @@ private ParameterType parameterType(long id, String name, String units) {
 		return parameterType;
 	}
 
-	private Sample sample(long id, String name) {
+	private InvestigationSample investigationSample(long id, String name, Investigation investigation) {
 		SampleType sampleType = new SampleType();
 		sampleType.setId(0L);
 		sampleType.setName("test");
 		Sample sample = new Sample();
 		sample.setId(id);
 		sample.setName(name);
 		sample.setType(sampleType);
-		return sample;
+		InvestigationSample investigationSample = new InvestigationSample();
+		investigationSample.setId(0L);
+		investigationSample.setInvestigation(investigation);
+		investigationSample.setSample(sample);
+		return investigationSample;
 	}
 
 	private void modify(String... operations) throws IcatException {
@@ -623,9 +628,9 @@ private void populate() throws IcatException {
 
 				if (datasetId < NUMSAMP) {
 					word = word("SType ", datasetId);
-					Sample sample = sample(datasetId, word);
-					queue.add(SearchApi.encodeOperation(entityManager, "create", sample));
-					dataset.setSample(sample);
+					InvestigationSample investigationSample = investigationSample(datasetId, word, investigation);
+					queue.add(SearchApi.encodeOperation(entityManager, "create", investigationSample));
+					dataset.setSample(investigationSample.getSample());
 				}
 
 				queue.add(SearchApi.encodeOperation(entityManager, "create", dataset));
@@ -1165,52 +1170,48 @@ public void investigations(SearchApi searchApi) throws Exception {
 		checkResults(lsr);
 
 		// Target sample.name
-		// FIXME: this test is broken
-		// query = buildQuery("Investigation", null, "sample.name:ddd", null, null, null, null);
-		// lsr = searchApi.getResults(query, 100, null);
-		// checkResults(lsr, 3L);
+		query = buildQuery("Investigation", null, "sample.name:ddd", null, null, null, null);
+		lsr = searchApi.getResults(query, 100, null);
+		checkResults(lsr, 3L);
 
 		// Multiple samples associated with investigation 3
-		// FIXME: this test is broken
-		// query = buildQuery("Investigation", null, "ddd nnn", null, null, null, null);
-		// lsr = searchApi.getResults(query, 100, null);
-		// checkResults(lsr, 3L);
+		query = buildQuery("Investigation", null, "ddd nnn", null, null, null, null);
+		lsr = searchApi.getResults(query, 100, null);
+		checkResults(lsr, 3L);
 
 		// By default, sample ddd OR sample mmm gives two investigations
-		// FIXME: this test is broken
-		// query = buildQuery("Investigation", null, "ddd mmm", null, null, null, null);
-		// lsr = searchApi.getResults(query, 100, null);
-		// checkResults(lsr, 2L, 3L);
+		query = buildQuery("Investigation", null, "ddd mmm", null, null, null, null);
+		lsr = searchApi.getResults(query, 100, null);
+		checkResults(lsr, 2L, 3L);
 
 		// AND logic should not return any results
 		query = buildQuery("Investigation", null, "+ddd +mmm", null, null, null, null);
 		lsr = searchApi.getResults(query, 100, null);
 		checkResults(lsr);
 
 		// Fields on Investigation and Sample
-		// FIXME: this test is broken
-		// query = buildQuery("Investigation", null, "visitId ddd", null, null, null, null);
-		// lsr = searchApi.getResults(query, 100, null);
-		// checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L);
+		query = buildQuery("Investigation", null, "visitId ddd", null, null, null, null);
+		lsr = searchApi.getResults(query, 100, null);
+		checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L);
 		// ID 3 should be most relevant since it matches both terms
-		// lsr = searchApi.getResults(query, 1, null);
-		// checkResults(lsr, 3L);
+		lsr = searchApi.getResults(query, 1, null);
+		checkResults(lsr, 3L);
 		// Specifying fields should not alter behaviour
-		// query = buildQuery("Investigation", null, "visitId:visitId sample.name:ddd", null, null, null, null);
-		// lsr = searchApi.getResults(query, 100, null);
-		// checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L);
+		query = buildQuery("Investigation", null, "visitId:visitId sample.name:ddd", null, null, null, null);
+		lsr = searchApi.getResults(query, 100, null);
+		checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L);
 		// Individual MUST should work when applied to either an Investigation or Sample
-		// query = buildQuery("Investigation", null, "+visitId:visitId", null, null, null, null);
-		// lsr = searchApi.getResults(query, 100, null);
-		// checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L);
-		// query = buildQuery("Investigation", null, "+sample.name:ddd", null, null, null, null);
-		// lsr = searchApi.getResults(query, 100, null);
-		// checkResults(lsr, 3L);
+		query = buildQuery("Investigation", null, "+visitId:visitId", null, null, null, null);
+		lsr = searchApi.getResults(query, 100, null);
+		checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L);
+		query = buildQuery("Investigation", null, "+sample.name:ddd", null, null, null, null);
+		lsr = searchApi.getResults(query, 100, null);
+		checkResults(lsr, 3L);
 		// This query is expected to fail, as we apply both terms to Investigation and
 		// Sample (since we have no fields) and neither possesses both terms.
-		// query = buildQuery("Investigation", null, "+visitId +ddd", null, null, null, null);
-		// lsr = searchApi.getResults(query, 100, null);
-		// checkResults(lsr);
+		query = buildQuery("Investigation", null, "+visitId +ddd", null, null, null, null);
+		lsr = searchApi.getResults(query, 100, null);
+		checkResults(lsr);
 
 		pojos = new ArrayList<>();
 		pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9"));
@@ -1220,12 +1221,11 @@ public void investigations(SearchApi searchApi) throws Exception {
 		checkResults(lsr, 3L);
 
 		// Sample filtering
-		// FIXME: this test is broken
-		// query = buildQuery("Investigation", null, null, null, null, null, null, new Filter("sample.sample.type.name", "test"));
-		// lsr = searchApi.getResults(query, 5, null);
-		// checkResults(lsr, 0L, 1L, 2L, 3L, 4L);
+		query = buildQuery("Investigation", null, null, null, null, null, null, new Filter("investigationsample.sample.type.name", "test"));
+		lsr = searchApi.getResults(query, 5, null);
+		checkResults(lsr, 0L, 1L, 2L, 3L, 4L);
 
-		query = buildQuery("Investigation", null, null, null, null, null, null, new Filter("sample.sample.type.name", "fail"));
+		query = buildQuery("Investigation", null, null, null, null, null, null, new Filter("investigationsample.sample.type.name", "fail"));
 		lsr = searchApi.getResults(query, 5, null);
 		checkResults(lsr);
 
@@ -1523,7 +1523,8 @@ public void sampleParameters(SearchApi searchApi) throws IcatException {
 		Investigation investigation = investigation(0, "investigation", date, date);
 		Dataset dataset = dataset(1, "dataset", date, date, investigation);
 		Datafile datafile = datafile(2, "datafile", "datafile.txt", date, dataset);
-		Sample sample = sample(3, "sample");
+		InvestigationSample investigationSample = investigationSample(3, "sample", investigation);
+		Sample sample = investigationSample.getSample();
 		ParameterType parameterType = parameterType(4, "parameter", "K");
 		SampleParameter parameter = (SampleParameter) parameter(5, "stringValue", parameterType, sample);
 		dataset.setSample(sample);
@@ -1534,7 +1535,7 @@ public void sampleParameters(SearchApi searchApi) throws IcatException {
 		JsonArrayBuilder dimensions = Json.createArrayBuilder().add(dimension);
 		JsonObject sampleParameterFacetQuery = Json.createObjectBuilder().add("query", sampleQuery).add("dimensions", dimensions).build();
 
-		JsonObjectBuilder sampleInvestigationQuery = Json.createObjectBuilder().add("sample.investigation.id", Json.createArrayBuilder().add(0));
+		JsonObjectBuilder sampleInvestigationQuery = Json.createObjectBuilder().add("investigation.id", Json.createArrayBuilder().add(0));
 		JsonObjectBuilder sampleTypeDimension = Json.createObjectBuilder().add("dimension", "sample.type.name");
 		JsonArrayBuilder sampleTypeDimensions = Json.createArrayBuilder().add(sampleTypeDimension);
 		JsonObject sampleTypeFacetQuery = Json.createObjectBuilder().add("query", sampleInvestigationQuery).add("dimensions", sampleTypeDimensions).build();
@@ -1564,14 +1565,13 @@ public void sampleParameters(SearchApi searchApi) throws IcatException {
 		modify(SearchApi.encodeOperation(entityManager, "create", investigation),
 				SearchApi.encodeOperation(entityManager, "create", dataset),
 				SearchApi.encodeOperation(entityManager, "create", datafile),
-				SearchApi.encodeOperation(entityManager, "create", sample),
+				SearchApi.encodeOperation(entityManager, "create", investigationSample),
 				SearchApi.encodeOperation(entityManager, "create", parameterType),
 				SearchApi.encodeOperation(entityManager, "create", parameter));
 
 		// Test
 		checkFacets(searchApi.facetSearch("SampleParameter", sampleParameterFacetQuery, 5, 5), sampleParemeterFacet);
-		// FIXME: this test is broken
-		// checkFacets(searchApi.facetSearch("Sample", sampleTypeFacetQuery, 5, 5), sampleTypeFacet);
+		checkFacets(searchApi.facetSearch("InvestigationSample", sampleTypeFacetQuery, 5, 5), sampleTypeFacet);
 		checkFacets(searchApi.facetSearch("Dataset", sparseRequest, 5, 5), datasetTypeFacet, sampleTypeFacet);
 		checkFacets(searchApi.facetSearch("Datafile", sparseRequest, 5, 5), sampleTypeFacet);