diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 996e6a12..25549876 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -37,7 +37,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4.0.0 with: repository: icatproject-contrib/icat-ansible - ref: no-icat-install + ref: icat.lucene_5.0.0 path: icat-ansible - name: Install Ansible run: pip install -r icat-ansible/requirements.txt diff --git a/src/main/config/run.properties.example b/src/main/config/run.properties.example index 8b4fdc8b..8763691a 100644 --- a/src/main/config/run.properties.example +++ b/src/main/config/run.properties.example @@ -59,7 +59,7 @@ search.maxSearchTimeSeconds = 5 # Configure this option to prevent certain entities being indexed # For example, remove Datafile and DatafileParameter if these are not of interest # Note then when commented out, the full set of all possible entities will be indexed - to disable all search functionality, instead comment out search.engine or search.urls -!search.entitiesToIndex = Datafile DatafileFormat DatafileParameter Dataset DatasetParameter DatasetType DatasetTechnique Facility Instrument InstrumentScientist Investigation InvestigationInstrument InvestigationParameter InvestigationType InvestigationUser ParameterType Sample SampleType SampleParameter User +!search.entitiesToIndex = Datafile DatafileFormat DatafileParameter Dataset DatasetParameter DatasetType DatasetTechnique Facility Instrument InstrumentScientist Investigation InvestigationInstrument InvestigationParameter InvestigationSample InvestigationType InvestigationUser ParameterType Sample SampleType SampleParameter User # List members of cluster !cluster = http://vm200.nubes.stfc.ac.uk:8080 https://smfisher:8181 diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java index 7797a161..c526eb44 100644 --- a/src/main/java/org/icatproject/core/entity/Investigation.java +++ b/src/main/java/org/icatproject/core/entity/Investigation.java @@ -383,6 +383,13 @@ public static Map getDocumentFields() throws IcatExcepti Relationship[] parameterTypeRelationships = { EntityInfoHandler.getRelationshipsByName(Investigation.class).get("parameters"), EntityInfoHandler.getRelationshipsByName(InvestigationParameter.class).get("type") }; + Relationship[] sampleRelationships = { + EntityInfoHandler.getRelationshipsByName(Investigation.class).get("samples"), + EntityInfoHandler.getRelationshipsByName(InvestigationSample.class).get("sample") }; + Relationship[] sampleTypeRelationships = { + EntityInfoHandler.getRelationshipsByName(Investigation.class).get("samples"), + EntityInfoHandler.getRelationshipsByName(InvestigationSample.class).get("sample"), + EntityInfoHandler.getRelationshipsByName(Sample.class).get("type") }; documentFields.put("name", null); documentFields.put("visitId", null); documentFields.put("title", null); @@ -406,6 +413,9 @@ public static Map getDocumentFields() throws IcatExcepti documentFields.put("InvestigationParameter stringValue", parameterRelationships); documentFields.put("InvestigationParameter numericValue", parameterRelationships); documentFields.put("InvestigationParameter dateTimeValue", parameterRelationships); + documentFields.put("InvestigationSample sample.id", sampleRelationships); + documentFields.put("InvestigationSample sample.name", sampleRelationships); + documentFields.put("InvestigationSample type.name", sampleTypeRelationships); } return documentFields; } diff --git a/src/main/java/org/icatproject/core/entity/InvestigationSample.java b/src/main/java/org/icatproject/core/entity/InvestigationSample.java index ecf312d4..3462b353 100644 --- a/src/main/java/org/icatproject/core/entity/InvestigationSample.java +++ b/src/main/java/org/icatproject/core/entity/InvestigationSample.java @@ -2,7 +2,12 @@ import java.io.Serializable; +import org.icatproject.core.IcatException; +import org.icatproject.core.manager.search.SearchApi; + +import jakarta.json.stream.JsonGenerator; import jakarta.persistence.Entity; +import jakarta.persistence.EntityManager; import jakarta.persistence.FetchType; import jakarta.persistence.JoinColumn; import jakarta.persistence.ManyToOne; @@ -38,4 +43,14 @@ public void setInvestigation(Investigation investigation) { public void setSample(Sample sample) { this.sample = sample; } + + @Override + public void getDoc(EntityManager entityManager, JsonGenerator gen) throws IcatException { + if (sample.getName() == null) { + sample = entityManager.find(sample.getClass(), sample.id); + } + sample.getDoc(entityManager, gen); + SearchApi.encodeLong(gen, "investigation.id", investigation.id); + SearchApi.encodeLong(gen, "id", id); + } } diff --git a/src/main/java/org/icatproject/core/entity/Sample.java b/src/main/java/org/icatproject/core/entity/Sample.java index d28008ea..4ceb7254 100644 --- a/src/main/java/org/icatproject/core/entity/Sample.java +++ b/src/main/java/org/icatproject/core/entity/Sample.java @@ -48,8 +48,7 @@ public class Sample extends EntityBaseBean implements Serializable { @ManyToOne(fetch = FetchType.LAZY) private SampleType type; - public static Set docFields = new HashSet<>( - Arrays.asList("sample.name", "sample.id", "sample.investigation.id")); + public static Set docFields = new HashSet<>(Arrays.asList("sample.name", "sample.id")); /* Needed for JPA */ public Sample() { diff --git a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java index 5acaeb27..7ed2b84f 100644 --- a/src/main/java/org/icatproject/core/manager/EntityBeanManager.java +++ b/src/main/java/org/icatproject/core/manager/EntityBeanManager.java @@ -1617,6 +1617,8 @@ private JsonObject buildFacetQuery(Class klass, String return SearchManager.buildFacetQuery(results, resultIdField, "investigation.id", jsonFacet); } else if (target.contains("Parameter")) { relationship = EntityInfoHandler.getRelationshipsByName(klass).get("parameters"); + } else if (target.contains("InvestigationSample")) { + relationship = EntityInfoHandler.getRelationshipsByName(klass).get("samples"); } else { relationship = EntityInfoHandler.getRelationshipsByName(klass).get(target.toLowerCase() + "s"); } diff --git a/src/main/java/org/icatproject/core/manager/PropertyHandler.java b/src/main/java/org/icatproject/core/manager/PropertyHandler.java index 91d9538d..8b43fa0b 100644 --- a/src/main/java/org/icatproject/core/manager/PropertyHandler.java +++ b/src/main/java/org/icatproject/core/manager/PropertyHandler.java @@ -408,8 +408,8 @@ private void init() { entitiesToIndex.addAll(Arrays.asList("Datafile", "DatafileFormat", "DatafileParameter", "Dataset", "DatasetParameter", "DatasetType", "DatasetTechnique", "Facility", "Instrument", "InstrumentScientist", "Investigation", "InvestigationInstrument", "InvestigationParameter", - "InvestigationType", "InvestigationUser", "ParameterType", "Sample", "SampleType", - "SampleParameter", "User")); + "InvestigationSample", "InvestigationType", "InvestigationUser", "ParameterType", "Sample", + "SampleType", "SampleParameter", "User")); logger.info("search.entitiesToIndex not set. Defaulting to: {}", entitiesToIndex.toString()); } formattedProps.add("search.entitiesToIndex " + entitiesToIndex.toString()); diff --git a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java index fea15f16..8c8c6a8c 100644 --- a/src/test/java/org/icatproject/core/manager/TestEntityInfo.java +++ b/src/test/java/org/icatproject/core/manager/TestEntityInfo.java @@ -49,8 +49,8 @@ public void testHasSearchDoc() throws Exception { Set docdbeans = new HashSet<>(Arrays.asList("Datafile", "DatafileFormat", "DatafileParameter", "Dataset", "DatasetParameter", "DatasetTechnique", "DatasetType", "Facility", "Instrument", "InstrumentScientist", "Investigation", "InvestigationFacilityCycle", "InvestigationInstrument", - "InvestigationParameter", "InvestigationType", "InvestigationUser", "ParameterType", "Sample", - "SampleType", "SampleParameter", "Technique", "User")); + "InvestigationParameter", "InvestigationSample", "InvestigationType", "InvestigationUser", + "ParameterType", "Sample", "SampleType", "SampleParameter", "Technique", "User")); for (String beanName : EntityInfoHandler.getEntityNamesList()) { @SuppressWarnings("unchecked") Class bean = EntityInfoHandler.getClass(beanName); diff --git a/src/test/java/org/icatproject/core/manager/TestSearchApi.java b/src/test/java/org/icatproject/core/manager/TestSearchApi.java index 9062ba08..78ba3de2 100644 --- a/src/test/java/org/icatproject/core/manager/TestSearchApi.java +++ b/src/test/java/org/icatproject/core/manager/TestSearchApi.java @@ -42,6 +42,7 @@ import org.icatproject.core.entity.InvestigationFacilityCycle; import org.icatproject.core.entity.InvestigationInstrument; import org.icatproject.core.entity.InvestigationParameter; +import org.icatproject.core.entity.InvestigationSample; import org.icatproject.core.entity.InvestigationType; import org.icatproject.core.entity.InvestigationUser; import org.icatproject.core.entity.Parameter; @@ -505,7 +506,7 @@ private ParameterType parameterType(long id, String name, String units) { return parameterType; } - private Sample sample(long id, String name) { + private InvestigationSample investigationSample(long id, String name, Investigation investigation) { SampleType sampleType = new SampleType(); sampleType.setId(0L); sampleType.setName("test"); @@ -513,7 +514,11 @@ private Sample sample(long id, String name) { sample.setId(id); sample.setName(name); sample.setType(sampleType); - return sample; + InvestigationSample investigationSample = new InvestigationSample(); + investigationSample.setId(0L); + investigationSample.setInvestigation(investigation); + investigationSample.setSample(sample); + return investigationSample; } private void modify(String... operations) throws IcatException { @@ -623,9 +628,9 @@ private void populate() throws IcatException { if (datasetId < NUMSAMP) { word = word("SType ", datasetId); - Sample sample = sample(datasetId, word); - queue.add(SearchApi.encodeOperation(entityManager, "create", sample)); - dataset.setSample(sample); + InvestigationSample investigationSample = investigationSample(datasetId, word, investigation); + queue.add(SearchApi.encodeOperation(entityManager, "create", investigationSample)); + dataset.setSample(investigationSample.getSample()); } queue.add(SearchApi.encodeOperation(entityManager, "create", dataset)); @@ -1165,22 +1170,19 @@ public void investigations(SearchApi searchApi) throws Exception { checkResults(lsr); // Target sample.name - // FIXME: this test is broken - // query = buildQuery("Investigation", null, "sample.name:ddd", null, null, null, null); - // lsr = searchApi.getResults(query, 100, null); - // checkResults(lsr, 3L); + query = buildQuery("Investigation", null, "sample.name:ddd", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 3L); // Multiple samples associated with investigation 3 - // FIXME: this test is broken - // query = buildQuery("Investigation", null, "ddd nnn", null, null, null, null); - // lsr = searchApi.getResults(query, 100, null); - // checkResults(lsr, 3L); + query = buildQuery("Investigation", null, "ddd nnn", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 3L); // By default, sample ddd OR sample mmm gives two investigations - // FIXME: this test is broken - // query = buildQuery("Investigation", null, "ddd mmm", null, null, null, null); - // lsr = searchApi.getResults(query, 100, null); - // checkResults(lsr, 2L, 3L); + query = buildQuery("Investigation", null, "ddd mmm", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 2L, 3L); // AND logic should not return any results query = buildQuery("Investigation", null, "+ddd +mmm", null, null, null, null); @@ -1188,29 +1190,28 @@ public void investigations(SearchApi searchApi) throws Exception { checkResults(lsr); // Fields on Investigation and Sample - // FIXME: this test is broken - // query = buildQuery("Investigation", null, "visitId ddd", null, null, null, null); - // lsr = searchApi.getResults(query, 100, null); - // checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); + query = buildQuery("Investigation", null, "visitId ddd", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); // ID 3 should be most relevant since it matches both terms - // lsr = searchApi.getResults(query, 1, null); - // checkResults(lsr, 3L); + lsr = searchApi.getResults(query, 1, null); + checkResults(lsr, 3L); // Specifying fields should not alter behaviour - // query = buildQuery("Investigation", null, "visitId:visitId sample.name:ddd", null, null, null, null); - // lsr = searchApi.getResults(query, 100, null); - // checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); + query = buildQuery("Investigation", null, "visitId:visitId sample.name:ddd", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); // Individual MUST should work when applied to either an Investigation or Sample - // query = buildQuery("Investigation", null, "+visitId:visitId", null, null, null, null); - // lsr = searchApi.getResults(query, 100, null); - // checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); - // query = buildQuery("Investigation", null, "+sample.name:ddd", null, null, null, null); - // lsr = searchApi.getResults(query, 100, null); - // checkResults(lsr, 3L); + query = buildQuery("Investigation", null, "+visitId:visitId", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); + query = buildQuery("Investigation", null, "+sample.name:ddd", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr, 3L); // This query is expected to fail, as we apply both terms to Investigation and // Sample (since we have no fields) and neither possesses both terms. - // query = buildQuery("Investigation", null, "+visitId +ddd", null, null, null, null); - // lsr = searchApi.getResults(query, 100, null); - // checkResults(lsr); + query = buildQuery("Investigation", null, "+visitId +ddd", null, null, null, null); + lsr = searchApi.getResults(query, 100, null); + checkResults(lsr); pojos = new ArrayList<>(); pojos.add(new ParameterPOJO("Snm ddd", "u iii", "v9")); @@ -1220,12 +1221,11 @@ public void investigations(SearchApi searchApi) throws Exception { checkResults(lsr, 3L); // Sample filtering - // FIXME: this test is broken - // query = buildQuery("Investigation", null, null, null, null, null, null, new Filter("sample.sample.type.name", "test")); - // lsr = searchApi.getResults(query, 5, null); - // checkResults(lsr, 0L, 1L, 2L, 3L, 4L); + query = buildQuery("Investigation", null, null, null, null, null, null, new Filter("investigationsample.sample.type.name", "test")); + lsr = searchApi.getResults(query, 5, null); + checkResults(lsr, 0L, 1L, 2L, 3L, 4L); - query = buildQuery("Investigation", null, null, null, null, null, null, new Filter("sample.sample.type.name", "fail")); + query = buildQuery("Investigation", null, null, null, null, null, null, new Filter("investigationsample.sample.type.name", "fail")); lsr = searchApi.getResults(query, 5, null); checkResults(lsr); @@ -1523,7 +1523,8 @@ public void sampleParameters(SearchApi searchApi) throws IcatException { Investigation investigation = investigation(0, "investigation", date, date); Dataset dataset = dataset(1, "dataset", date, date, investigation); Datafile datafile = datafile(2, "datafile", "datafile.txt", date, dataset); - Sample sample = sample(3, "sample"); + InvestigationSample investigationSample = investigationSample(3, "sample", investigation); + Sample sample = investigationSample.getSample(); ParameterType parameterType = parameterType(4, "parameter", "K"); SampleParameter parameter = (SampleParameter) parameter(5, "stringValue", parameterType, sample); dataset.setSample(sample); @@ -1534,7 +1535,7 @@ public void sampleParameters(SearchApi searchApi) throws IcatException { JsonArrayBuilder dimensions = Json.createArrayBuilder().add(dimension); JsonObject sampleParameterFacetQuery = Json.createObjectBuilder().add("query", sampleQuery).add("dimensions", dimensions).build(); - JsonObjectBuilder sampleInvestigationQuery = Json.createObjectBuilder().add("sample.investigation.id", Json.createArrayBuilder().add(0)); + JsonObjectBuilder sampleInvestigationQuery = Json.createObjectBuilder().add("investigation.id", Json.createArrayBuilder().add(0)); JsonObjectBuilder sampleTypeDimension = Json.createObjectBuilder().add("dimension", "sample.type.name"); JsonArrayBuilder sampleTypeDimensions = Json.createArrayBuilder().add(sampleTypeDimension); JsonObject sampleTypeFacetQuery = Json.createObjectBuilder().add("query", sampleInvestigationQuery).add("dimensions", sampleTypeDimensions).build(); @@ -1564,14 +1565,13 @@ public void sampleParameters(SearchApi searchApi) throws IcatException { modify(SearchApi.encodeOperation(entityManager, "create", investigation), SearchApi.encodeOperation(entityManager, "create", dataset), SearchApi.encodeOperation(entityManager, "create", datafile), - SearchApi.encodeOperation(entityManager, "create", sample), + SearchApi.encodeOperation(entityManager, "create", investigationSample), SearchApi.encodeOperation(entityManager, "create", parameterType), SearchApi.encodeOperation(entityManager, "create", parameter)); // Test checkFacets(searchApi.facetSearch("SampleParameter", sampleParameterFacetQuery, 5, 5), sampleParemeterFacet); - // FIXME: this test is broken - // checkFacets(searchApi.facetSearch("Sample", sampleTypeFacetQuery, 5, 5), sampleTypeFacet); + checkFacets(searchApi.facetSearch("InvestigationSample", sampleTypeFacetQuery, 5, 5), sampleTypeFacet); checkFacets(searchApi.facetSearch("Dataset", sparseRequest, 5, 5), datasetTypeFacet, sampleTypeFacet); checkFacets(searchApi.facetSearch("Datafile", sparseRequest, 5, 5), sampleTypeFacet); diff --git a/src/test/java/org/icatproject/integration/TestRS.java b/src/test/java/org/icatproject/integration/TestRS.java index 124fe378..0e13ecbc 100644 --- a/src/test/java/org/icatproject/integration/TestRS.java +++ b/src/test/java/org/icatproject/integration/TestRS.java @@ -648,9 +648,9 @@ public void testLuceneInvestigations() throws Exception { parameters.add(new ParameterForLucene("colour", "name", "green")); // This does not work any more as there is no direct relation between investigation and samples now - // JsonArray array = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, - // null, "Professor", 20, 1); - // checkResultFromLuceneSearch(session, "one", array, "Investigation", "visitId"); + JsonArray array = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, parameters, + null, "Professor", 20, 1); + checkResultFromLuceneSearch(session, "one", array, "Investigation", "visitId"); // change user searchInvestigations(session, "db/fred", textAnd, null, null, parameters, null, null, 20, 0); @@ -659,7 +659,7 @@ public void testLuceneInvestigations() throws Exception { searchInvestigations(session, "db/tr", textTwo, null, null, parameters, null, null, 20, 0); // Only working to a minute - JsonArray array = searchInvestigations(session, "db/tr", textAnd, lowerSecond, upperOrigin, parameters, null, null, 20, + array = searchInvestigations(session, "db/tr", textAnd, lowerSecond, upperOrigin, parameters, null, null, 20, 1); checkResultFromLuceneSearch(session, "one", array, "Investigation", "visitId"); @@ -999,13 +999,12 @@ public void testSearchInvestigations() throws Exception { null, 10, null, null, 0); // Change samples - // FIXME: this test is broken - // searchInvestigations(session, "db/tr", samplesSingular, lowerOrigin, upperOrigin, parameters, null, null, - // 10, null, null, 1); - // searchInvestigations(session, "db/tr", samplesMultiple, lowerOrigin, upperOrigin, parameters, null, null, - // 10, null, null, 1); - // searchInvestigations(session, "db/tr", samplesBad, lowerOrigin, upperOrigin, parameters, null, null, - // 10, null, null, 0); + searchInvestigations(session, "db/tr", samplesSingular, lowerOrigin, upperOrigin, parameters, null, null, + 10, null, null, 1); + searchInvestigations(session, "db/tr", samplesMultiple, lowerOrigin, upperOrigin, parameters, null, null, + 10, null, null, 1); + searchInvestigations(session, "db/tr", samplesBad, lowerOrigin, upperOrigin, parameters, null, null, + 10, null, null, 0); // Change userFullName searchInvestigations(session, "db/tr", textPlus, lowerOrigin, upperOrigin, parameters, "Doctor", @@ -1081,21 +1080,20 @@ public void testSearchInvestigations() throws Exception { Arrays.asList(1L)); // Test no facets match on Sample due to lack of READ access - // FIXME: this test is broken - // facets = buildFacetRequest("Sample", "sample.type.name"); - // responseObject = searchInvestigations(session, null, null, null, null, null, null, null, 10, null, facets, - // 3); - // assertFalse(responseObject.containsKey("search_after")); - // assertFalse(responseObject.containsKey("dimensions"), NO_DIMENSIONS); + facets = buildFacetRequest("InvestigationSample", "sample.type.name"); + responseObject = searchInvestigations(session, null, null, null, null, null, null, null, 10, null, facets, + 3); + assertFalse(responseObject.containsKey("search_after")); + assertFalse(responseObject.containsKey("dimensions"), NO_DIMENSIONS); // Test facets match on Sample - // FIXME: this test is broken - // wSession.addRule(null, "Sample", "R"); - // responseObject = searchInvestigations(session, null, null, null, null, null, null, null, 10, null, facets, - // 3); - // assertFalse(responseObject.containsKey("search_after")); - // checkFacets(responseObject, "Sample.sample.type.name", Arrays.asList("diamond", "rust"), - // Arrays.asList(1L, 1L)); + wSession.addRule(null, "InvestigationSample", "R"); + wSession.addRule(null, "Sample", "R"); + responseObject = searchInvestigations(session, null, null, null, null, null, null, null, 10, null, facets, + 3); + assertFalse(responseObject.containsKey("search_after")); + checkFacets(responseObject, "InvestigationSample.sample.type.name", Arrays.asList("diamond", "rust"), + Arrays.asList(1L, 1L)); } @Test